From 0f4d43ef470d3f254aff72211b2476403af8f52a Mon Sep 17 00:00:00 2001 From: David Zollo Date: Thu, 11 Jun 2026 21:53:52 +0800 Subject: [PATCH 001/375] [Fix][Transform-V2] Avoid eager zeta udf open and fix null table context (#10760) --- .../transform/sql/zeta/ZetaSQLEngine.java | 27 +++- .../transform/sql/zeta/ZetaUDFContext.java | 5 +- .../transform/sql/zeta/ZetaSQLEngineTest.java | 116 ++++++++++++++++++ .../sql/zeta/ZetaUDFContextTest.java | 43 +++++++ 4 files changed, 187 insertions(+), 4 deletions(-) create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaUDFContextTest.java diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngine.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngine.java index 86a12e6126be..c9985826657f 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngine.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngine.java @@ -73,6 +73,7 @@ public class ZetaSQLEngine implements SQLEngine { private ZetaUDFContext udfContext; private Integer allColumnsCount = null; + private boolean udfOpened; public ZetaSQLEngine() {} @@ -95,7 +96,6 @@ public void init( this.zetaSQLFilter = new ZetaSQLFilter(zetaSQLFunction, zetaSQLType); parseSQL(); - openUDFs(); } protected List loadUDFs() { @@ -111,6 +111,14 @@ private void openUDFs() { try { udf.open(); } catch (Exception e) { + try { + udf.close(); + } catch (Exception closeException) { + log.warn( + "Best-effort close failed for udf {}", + udf.functionName(), + closeException); + } closeUDFs(i - 1); log.error("Open udf {} failed", udf.functionName(), e); throw new TransformException( @@ -121,6 +129,19 @@ private void openUDFs() { } } + private void ensureUdfOpened() { + if (udfOpened || CollectionUtils.isEmpty(udfList)) { + return; + } + synchronized (this) { + if (udfOpened) { + return; + } + openUDFs(); + udfOpened = true; + } + } + private void parseSQL() { try { Statement statement = CCJSqlParserUtil.parse(sql); @@ -262,6 +283,7 @@ private static String cleanEscape(String columnName) { @Override public List transformBySQL(SeaTunnelRow inputRow, SeaTunnelRowType outRowType) { + ensureUdfOpened(); // ------Physical Query Plan Execution------ // Scan Table Object[] inputFields = scanTable(inputRow); @@ -343,10 +365,11 @@ private int countColumnsSize(List> selectItems) { @Override public void close() { - if (udfList == null || udfList.isEmpty()) { + if (CollectionUtils.isEmpty(udfList) || !udfOpened) { return; } closeUDFs(udfList.size() - 1); + udfOpened = false; } private void closeUDFs(int lastIndex) { diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaUDFContext.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaUDFContext.java index b42dc9473c52..89fde0967baa 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaUDFContext.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaUDFContext.java @@ -51,11 +51,12 @@ public ZetaUDFContext update(Object[] fields, SeaTunnelRow row) { } private void updateTableId(String tableId) { - if (Objects.equals(this.rawTableId, tableId)) { + boolean isNullTableId = tableId == null; + if (Objects.equals(this.rawTableId, tableId) && this.tableIdIsNull == isNullTableId) { return; } this.rawTableId = tableId; - this.tableIdIsNull = tableId == null; + this.tableIdIsNull = isNullTableId; this.database = null; this.schema = null; this.table = null; diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngineTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngineTest.java index 4513fdba4bdd..e955638b2633 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngineTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngineTest.java @@ -27,6 +27,8 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; public class ZetaSQLEngineTest { @@ -95,4 +97,118 @@ public void testInvalidSqlThrowsTransformException() { rowType, "insert into test(id, name, age) values (1, 'bad', 10)")); } + + @Test + public void testSchemaInferenceShouldNotOpenUdf() { + TrackingUdf trackingUdf = new TrackingUdf("tracking", false); + ZetaSQLEngine engine = new TestableZetaSQLEngine(Collections.singletonList(trackingUdf)); + engine.init("test", "test", simpleRowType(), "select id, name from test"); + + SeaTunnelRowType outType = engine.typeMapping(new ArrayList<>()); + + Assertions.assertNotNull(outType); + Assertions.assertEquals(0, trackingUdf.getOpenCount()); + Assertions.assertEquals(0, trackingUdf.getCloseCount()); + } + + @Test + public void testOpenUdfWhenExecuteAndCloseOnEngineClose() { + TrackingUdf trackingUdf = new TrackingUdf("tracking", false); + ZetaSQLEngine engine = new TestableZetaSQLEngine(Collections.singletonList(trackingUdf)); + engine.init("test", "test", simpleRowType(), "select id from test"); + SeaTunnelRowType outType = engine.typeMapping(new ArrayList<>()); + + SeaTunnelRow inputRow = new SeaTunnelRow(new Object[] {1, "Alice", 20}); + engine.transformBySQL(inputRow, outType); + engine.transformBySQL(inputRow, outType); + + Assertions.assertEquals(1, trackingUdf.getOpenCount()); + Assertions.assertEquals(0, trackingUdf.getCloseCount()); + + engine.close(); + + Assertions.assertEquals(1, trackingUdf.getCloseCount()); + } + + @Test + public void testOpenFailureShouldCloseFailedAndOpenedUdfs() { + TrackingUdf firstUdf = new TrackingUdf("first", false); + TrackingUdf failedUdf = new TrackingUdf("failed", true); + ZetaSQLEngine engine = new TestableZetaSQLEngine(Arrays.asList(firstUdf, failedUdf)); + engine.init("test", "test", simpleRowType(), "select id from test"); + SeaTunnelRowType outType = engine.typeMapping(new ArrayList<>()); + + Assertions.assertThrows( + TransformException.class, + () -> + engine.transformBySQL( + new SeaTunnelRow(new Object[] {1, "Alice", 20}), outType)); + + Assertions.assertEquals(1, firstUdf.getOpenCount()); + Assertions.assertEquals(1, firstUdf.getCloseCount()); + Assertions.assertEquals(1, failedUdf.getOpenCount()); + Assertions.assertEquals(1, failedUdf.getCloseCount()); + } + + private static final class TestableZetaSQLEngine extends ZetaSQLEngine { + + private final List testUdfs; + + private TestableZetaSQLEngine(List testUdfs) { + this.testUdfs = testUdfs; + } + + @Override + protected List loadUDFs() { + return new ArrayList<>(testUdfs); + } + } + + private static final class TrackingUdf implements ZetaUDF { + private final String functionName; + private final boolean failOnOpen; + private int openCount; + private int closeCount; + + private TrackingUdf(String functionName, boolean failOnOpen) { + this.functionName = functionName; + this.failOnOpen = failOnOpen; + } + + @Override + public String functionName() { + return functionName; + } + + @Override + public SeaTunnelDataType resultType(List> argsType) { + return BasicType.STRING_TYPE; + } + + @Override + public Object evaluate(List args) { + return null; + } + + @Override + public void open() throws Exception { + openCount++; + if (failOnOpen) { + throw new Exception("open failed"); + } + } + + @Override + public void close() { + closeCount++; + } + + private int getOpenCount() { + return openCount; + } + + private int getCloseCount() { + return closeCount; + } + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaUDFContextTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaUDFContextTest.java new file mode 100644 index 000000000000..9df2a53d310e --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaUDFContextTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.sql.zeta; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ZetaUDFContextTest { + + @Test + public void testNullTableIdShouldNotTriggerTablePathResolve() { + ZetaUDFContext context = new ZetaUDFContext(); + SeaTunnelRow row = new SeaTunnelRow(new Object[] {1}); + row.setTableId(null); + + context.update(row); + + Assertions.assertNull(context.getRawTableId()); + Assertions.assertDoesNotThrow(context::getDatabase); + Assertions.assertDoesNotThrow(context::getSchema); + Assertions.assertDoesNotThrow(context::getTable); + Assertions.assertNull(context.getDatabase()); + Assertions.assertNull(context.getSchema()); + Assertions.assertNull(context.getTable()); + } +} From 9aad1cb93bd8fe7e66405c4199924ce37ace862b Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Thu, 11 Jun 2026 21:57:02 +0800 Subject: [PATCH 002/375] [Docs] Fix Http source code fence (#11064) Co-authored-by: Shenghang --- docs/en/connectors/source/Http.md | 2 +- docs/zh/connectors/source/Http.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/connectors/source/Http.md b/docs/en/connectors/source/Http.md index 1af464b2bb22..b0752c72a800 100644 --- a/docs/en/connectors/source/Http.md +++ b/docs/en/connectors/source/Http.md @@ -590,7 +590,7 @@ source { the `pageing.page_type` parameter must be set to `Cursor`. `cursor_field` is the field name of the cursor in the request parameters. `cursor_response_field` is the field name denotes the name of the pagination token field in the response data, we should add this to add pageing fields into request. -````hocon +```hocon source { Http { diff --git a/docs/zh/connectors/source/Http.md b/docs/zh/connectors/source/Http.md index 5ba01bca5601..a3e5dcf8ad45 100644 --- a/docs/zh/connectors/source/Http.md +++ b/docs/zh/connectors/source/Http.md @@ -579,7 +579,7 @@ source { `pageing.page_type` 参数必须设置为 `Cursor`。 `cursor_field` 是请求参数中游标的字段名称。 `cursor_response_field` 是响应数据中分页令牌字段的名称，我们应该将其添加到请求的分页字段中。 -````hocon +```hocon source { Http { From 9fdfc38142fb696f0306b8e0bc5afa7d6bfdc0dc Mon Sep 17 00:00:00 2001 From: Jast Date: Fri, 12 Jun 2026 14:57:19 +0800 Subject: [PATCH 003/375] [Fix][Connector-V2] Fix Cassandra null timestamp conversion (#11068) --- .../cassandra/util/TypeConvertUtil.java | 6 +-- .../cassandra/util/TypeConvertUtilTest.java | 51 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-cassandra/src/test/java/org/apache/seatunnel/connectors/seatunnel/cassandra/util/TypeConvertUtilTest.java diff --git a/seatunnel-connectors-v2/connector-cassandra/src/main/java/org/apache/seatunnel/connectors/seatunnel/cassandra/util/TypeConvertUtil.java b/seatunnel-connectors-v2/connector-cassandra/src/main/java/org/apache/seatunnel/connectors/seatunnel/cassandra/util/TypeConvertUtil.java index 974eb605b6a4..1753c97df288 100644 --- a/seatunnel-connectors-v2/connector-cassandra/src/main/java/org/apache/seatunnel/connectors/seatunnel/cassandra/util/TypeConvertUtil.java +++ b/seatunnel-connectors-v2/connector-cassandra/src/main/java/org/apache/seatunnel/connectors/seatunnel/cassandra/util/TypeConvertUtil.java @@ -44,6 +44,7 @@ import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.sql.Timestamp; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -181,9 +182,8 @@ public static SeaTunnelRow buildSeaTunnelRow(Row row) { fields[i] = row.getLocalDate(i); break; case ProtocolConstants.DataType.TIMESTAMP: - fields[i] = - Timestamp.from(Objects.requireNonNull(row.getInstant(i))) - .toLocalDateTime(); + Instant instant = row.getInstant(i); + fields[i] = instant == null ? null : Timestamp.from(instant).toLocalDateTime(); break; case ProtocolConstants.DataType.BLOB: fields[i] = diff --git a/seatunnel-connectors-v2/connector-cassandra/src/test/java/org/apache/seatunnel/connectors/seatunnel/cassandra/util/TypeConvertUtilTest.java b/seatunnel-connectors-v2/connector-cassandra/src/test/java/org/apache/seatunnel/connectors/seatunnel/cassandra/util/TypeConvertUtilTest.java new file mode 100644 index 000000000000..149087782899 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cassandra/src/test/java/org/apache/seatunnel/connectors/seatunnel/cassandra/util/TypeConvertUtilTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cassandra.util; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import com.datastax.oss.driver.api.core.cql.ColumnDefinition; +import com.datastax.oss.driver.api.core.cql.ColumnDefinitions; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.type.DataTypes; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TypeConvertUtilTest { + + @Test + void testBuildSeaTunnelRowKeepsNullTimestamp() { + Row row = mock(Row.class); + ColumnDefinitions columnDefinitions = mock(ColumnDefinitions.class); + ColumnDefinition columnDefinition = mock(ColumnDefinition.class); + + when(row.size()).thenReturn(1); + when(row.getColumnDefinitions()).thenReturn(columnDefinitions); + when(columnDefinitions.get(0)).thenReturn(columnDefinition); + when(columnDefinition.getType()).thenReturn(DataTypes.TIMESTAMP); + when(row.getInstant(0)).thenReturn(null); + + SeaTunnelRow seaTunnelRow = TypeConvertUtil.buildSeaTunnelRow(row); + + Assertions.assertNull(seaTunnelRow.getField(0)); + } +} From 343796d3922261223e748c10c86ec2dab8a731dc Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Fri, 12 Jun 2026 14:58:09 +0800 Subject: [PATCH 004/375] [Docs] Fix zeta engine jar storage typos (#11066) Co-authored-by: DanielCarter-stack <254644355+DanielCarter-stack@users.noreply.github.com> --- docs/en/engines/zeta/engine-jar-storage-mode.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/zeta/engine-jar-storage-mode.md b/docs/en/engines/zeta/engine-jar-storage-mode.md index 37ec099ac23f..095448db34f4 100644 --- a/docs/en/engines/zeta/engine-jar-storage-mode.md +++ b/docs/en/engines/zeta/engine-jar-storage-mode.md @@ -11,7 +11,7 @@ We are committed to ongoing efforts to enhance and stabilize this functionality, ::: -We can enable the optimization job submission process, which is configured in the `seatunel.yaml`. After enabling the optimization of the Seatunnel job submission process configuration item, +We can enable the optimization job submission process, which is configured in the `seatunnel.yaml`. After enabling the optimization of the Seatunnel job submission process configuration item, users can use the Seatunnel engine(Zeta) as the execution engine without placing the connector jar packages required for task execution or the third-party jar packages that the connector relies on in each engine `connector` directory. Users only need to place all the jar packages for task execution on the client that submits the job, and the client will automatically upload the jars required for task execution to the Zeta engine. It is necessary to enable this configuration item when submitting jobs in Docker or k8s mode, which can fundamentally solve the problem of large container images caused by the heavy weight of the Seatunnel Zeta engine. In the image, only the core framework package of the Zeta engine needs to be provided, @@ -45,7 +45,7 @@ Two different storage strategies provide a more flexible storage mode for jar fi ## IsolatedConnectorJarStorageStrategy -Before the job is submitted, the connector Jjr package will be uploaded to an independent file storage path on the Master node. +Before the job is submitted, the connector jar package will be uploaded to an independent file storage path on the Master node. The connector jar packages of different jobs are in different storage paths, so the connector jar packages of different jobs are isolated from each other. The jar package files required for the execution of a job have no influence on other jobs. When the current job execution ends, the jar package file in the storage path generated based on the JobId will be deleted. @@ -89,6 +89,5 @@ Detailed explanation of configuration parameters: - connector-jar-storage-enable: Enable uploading the connector jar package before executing the job. - connector-jar-storage-mode: Connector jar package storage mode, two storage modes are available: shared mode (SHARED) and isolation mode (ISOLATED). - connector-jar-storage-path: The local storage path of the user-defined connector jar package on the Zeta engine. -- connector-jar-cleanup-task-interval: Zeta engine connector Jjr package scheduled cleanup task interval, the default is 3600 seconds. +- connector-jar-cleanup-task-interval: Zeta engine connector jar package scheduled cleanup task interval, the default is 3600 seconds. - connector-jar-expiry-time: The expiration time of the connector jar package. The default is 600 seconds. - From 67fd886372b31cec525cb456cc4ae50eeed90d93 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 12 Jun 2026 15:10:58 +0800 Subject: [PATCH 005/375] [Docs] localize zh transform titles and summaries (#11070) --- docs/zh/transforms/data-validator.md | 8 +++---- docs/zh/transforms/define-sink-type.md | 18 +++++++------- docs/zh/transforms/dynamic-compile.md | 13 +++++------ docs/zh/transforms/embedding.md | 8 +++---- docs/zh/transforms/encrypt.md | 6 ++--- docs/zh/transforms/field-rename.md | 5 ++-- docs/zh/transforms/filter-rowkind.md | 5 ++-- docs/zh/transforms/jsonpath.md | 9 ++++--- docs/zh/transforms/llm.md | 6 ++--- docs/zh/transforms/metadata.md | 6 ++--- docs/zh/transforms/rowkind-extractor.md | 6 ++--- docs/zh/transforms/table-filter.md | 26 ++++++++++----------- docs/zh/transforms/table-merge.md | 22 ++++++++--------- docs/zh/transforms/table-rename.md | 5 ++-- docs/zh/transforms/transform-multi-table.md | 8 +++---- 15 files changed, 73 insertions(+), 78 deletions(-) diff --git a/docs/zh/transforms/data-validator.md b/docs/zh/transforms/data-validator.md index c35f47825f73..a4c02a9c6b4b 100644 --- a/docs/zh/transforms/data-validator.md +++ b/docs/zh/transforms/data-validator.md @@ -1,10 +1,10 @@ -# DataValidator +# 数据验证 -> 数据验证转换插件 +> DataValidator：按规则校验字段值，并处理不符合要求的数据 ## 描述 -DataValidator 转换插件根据配置的规则验证字段值，并基于指定的错误处理策略处理验证失败的情况。它支持多种验证规则类型，包括空值检查、范围验证、长度验证和正则表达式模式匹配。 +DataValidator 转换插件会根据配置规则校验字段值，并按照指定的错误处理策略处理验证失败的数据。它支持空值检查、范围验证、长度验证和正则表达式匹配等多种校验方式。 ## 选项 @@ -21,7 +21,7 @@ DataValidator 转换插件根据配置的规则验证字段值，并基于指定 - `SKIP`: 跳过无效行并继续处理 - `ROUTE_TO_TABLE`: 将无效数据路由到指定的错误表 -**注意**: `ROUTE_TO_TABLE` 模式仅适用于支持多表的 sink 连接器。sink 必须具备处理路由到不同表目标的数据的能力。 +**注意**：`ROUTE_TO_TABLE` 模式仅适用于支持多表写入的 sink 连接器。sink 必须具备把数据路由到不同目标表的能力。 ### row_error_handle_way.error_table [string] diff --git a/docs/zh/transforms/define-sink-type.md b/docs/zh/transforms/define-sink-type.md index 83ab268b3ff6..1346e7e1a602 100644 --- a/docs/zh/transforms/define-sink-type.md +++ b/docs/zh/transforms/define-sink-type.md @@ -1,18 +1,18 @@ -# Define Sink Type +# 定义写入字段类型 -> Define sink type transform plugin +> DefineSinkType：为 sink 建表或写入阶段显式指定字段类型 -## Description +## 描述 -用于定义 sink 字段存储类型，对于 savemode 开启自动建表时有效 +DefineSinkType 转换插件用于定义 sink 字段的目标存储类型，适用于开启 `savemode` 自动建表的场景。 -## Options +## 参数 -| name | type | required | default value | Description | -|:-------:|---------------------------|----------|---------------|--------------------| -| columns | list> | yes | | 需要定义的列，必须设置列的名称和类型 | +| 参数名 | 类型 | 是否必填 | 默认值 | 说明 | +|:------:|---------------------------|----------|--------|------| +| columns | list> | 是 | | 需要定义的列，必须为每一列指定名称和类型 | -## Examples +## 示例 ### 指定部分字段的建表类型 diff --git a/docs/zh/transforms/dynamic-compile.md b/docs/zh/transforms/dynamic-compile.md index 8a9e7dfe0caf..56ec41992821 100644 --- a/docs/zh/transforms/dynamic-compile.md +++ b/docs/zh/transforms/dynamic-compile.md @@ -1,17 +1,17 @@ -# DynamicCompile +# 动态编译转换 -> 动态编译插件 +> DynamicCompile：在运行时编译并执行自定义代码，完成灵活的数据处理 ## 描述 :::tip -特别申明 +特别声明 您需要确保服务的安全性，并防止攻击者上传破坏性代码 ::: -提供一种可编程的方式来处理行，允许用户自定义任何业务行为，甚至基于现有行字段作为参数的RPC请求，或者通过从其他数据源检索相关数据来扩展字段。为了区分业务，您还可以定义多个转换进行组合， +DynamicCompile 转换插件提供一种可编程的方式来处理行，允许用户自定义任何业务行为，甚至基于现有行字段作为参数的 RPC 请求，或者通过从其他数据源检索相关数据来扩展字段。为了区分业务，您还可以定义多个转换进行组合， 如果转换过于复杂，可能会影响性能 ## 属性 @@ -24,7 +24,7 @@ | absolute_path | string | no | | -### common options [string] +### 通用选项 [string] 转换插件的常见参数, 请参考 [Transform Plugin](common-options/common-options.md) 了解详情。 @@ -62,7 +62,7 @@ SOURCE_CODE,ABSOLUTE_PATH 你需要重启集群服务，才能重新加载这些依赖。 -## Example +## 示例 源端数据读取的表格如下： @@ -224,4 +224,3 @@ transform { https://github.com/apache/seatunnel/tree/dev/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/dynamic_compile/conf ## Changelog - diff --git a/docs/zh/transforms/embedding.md b/docs/zh/transforms/embedding.md index 415dae4766ab..9422181449c5 100644 --- a/docs/zh/transforms/embedding.md +++ b/docs/zh/transforms/embedding.md @@ -1,10 +1,10 @@ -# Embedding +# 向量化嵌入 -> Embedding Transform Plugin +> Embedding：将文本、图片或视频等内容转换为向量表示 ## 描述 -`Embedding` 转换插件利用 embedding 模型将文本和多模态数据转换为向量化表示。此转换可以应用于各种字段，包括文本、图片和视频。该插件支持多种模型提供商，并且可以与不同的API集成。 +Embedding 转换插件利用 embedding 模型将文本和多模态数据转换为向量化表示。此转换可以应用于各种字段，包括文本、图片和视频。该插件支持多种模型提供商，并且可以与不同的 API 集成。 > **重要提示：** 当前 embedding 精确度仅支持 float32 @@ -682,4 +682,4 @@ sink { } } -``` \ No newline at end of file +``` diff --git a/docs/zh/transforms/encrypt.md b/docs/zh/transforms/encrypt.md index 0d60ff120d16..4829ae253768 100644 --- a/docs/zh/transforms/encrypt.md +++ b/docs/zh/transforms/encrypt.md @@ -1,10 +1,10 @@ -# Encrypt +# 字段加解密 -> 加密 Transform 插件 +> FieldEncrypt：对指定字段执行加密或解密 ## 描述 -Encrypt Transform 插件用于使用对称加密算法，对记录中指定的字段进行加密或解密。 +FieldEncrypt 转换插件用于使用对称加密算法，对记录中的指定字段进行加密或解密。 ## 参数说明 diff --git a/docs/zh/transforms/field-rename.md b/docs/zh/transforms/field-rename.md index 1e0577b346ba..b705b26ccb91 100644 --- a/docs/zh/transforms/field-rename.md +++ b/docs/zh/transforms/field-rename.md @@ -1,10 +1,10 @@ # 字段重命名 -> FieldRename 转换插件 +> FieldRename：批量重命名字段名，统一输出字段命名 ## 描述 -FieldRename 用于批量重命名字段名。 +FieldRename 转换插件用于批量重命名字段名。 ## 选项 @@ -146,4 +146,3 @@ sink { } } ``` - diff --git a/docs/zh/transforms/filter-rowkind.md b/docs/zh/transforms/filter-rowkind.md index 58e625b232fd..0c8febb9a841 100644 --- a/docs/zh/transforms/filter-rowkind.md +++ b/docs/zh/transforms/filter-rowkind.md @@ -1,10 +1,10 @@ # 行类型过滤 -> 行类型转换插件 +> FilterRowKind：按插入、更新、删除等行类型筛选数据 ## 描述 -按行类型过滤数据 +FilterRowKind 转换插件用于按 RowKind 过滤数据。 ## 操作 @@ -65,4 +65,3 @@ sink { } } ``` - diff --git a/docs/zh/transforms/jsonpath.md b/docs/zh/transforms/jsonpath.md index c2984a82d22d..02f7110db78a 100644 --- a/docs/zh/transforms/jsonpath.md +++ b/docs/zh/transforms/jsonpath.md @@ -1,10 +1,10 @@ -# JsonPath +# JSON 路径提取 -> JSONPath 转换插件 +> JsonPath：使用 JSONPath 从 JSON 数据中提取字段 ## 描述 -> 支持使用 JSONPath 选择数据 +JsonPath 转换插件支持使用 JSONPath 选择数据。 ## 属性 @@ -13,7 +13,7 @@ | columns | Array | Yes | | | row_error_handle_way | Enum | No | FAIL | -### common options [string] +### 通用选项 [string] 转换插件的常见参数, 请参考 [Transform Plugin](common-options/common-options.md) 了解详情 @@ -323,4 +323,3 @@ transform { ## 更新日志 * 添加 JsonPath 转换 - diff --git a/docs/zh/transforms/llm.md b/docs/zh/transforms/llm.md index e59492e65c5e..4362dcccd59a 100644 --- a/docs/zh/transforms/llm.md +++ b/docs/zh/transforms/llm.md @@ -1,10 +1,10 @@ -# LLM +# 大语言模型处理 -> LLM 转换插件 +> LLM：调用大语言模型完成清洗、标注、推理或数据丰富 ## 描述 -利用大型语言模型 (LLM) 的强大功能来处理数据，方法是将数据发送到 LLM 并接收生成的结果。利用 LLM 的功能来标记、清理、丰富数据、执行数据推理等。 +LLM 转换插件利用大型语言模型（LLM）的能力处理数据，将输入内容发送到 LLM 并接收生成结果，可用于标记、清理、丰富数据以及执行数据推理等场景。 ## 属性 diff --git a/docs/zh/transforms/metadata.md b/docs/zh/transforms/metadata.md index fb7c2f79b0ce..552ea73ad921 100644 --- a/docs/zh/transforms/metadata.md +++ b/docs/zh/transforms/metadata.md @@ -1,10 +1,10 @@ -# Metadata +# 元数据提取 -> Metadata 转换插件 +> Metadata：把库名、表名、RowKind 等元数据提取为普通字段 ## 描述 -Metadata 转换插件用于将数据行中的元数据信息提取并转换为普通字段，方便后续处理和分析。 +Metadata 转换插件用于将数据行中的元数据信息提取为普通字段，方便后续处理和分析。 **核心功能：** - 将元数据（如数据库名、表名、行类型等）提取为可见字段 diff --git a/docs/zh/transforms/rowkind-extractor.md b/docs/zh/transforms/rowkind-extractor.md index 7813b27f6d69..5fb90b03cd9b 100644 --- a/docs/zh/transforms/rowkind-extractor.md +++ b/docs/zh/transforms/rowkind-extractor.md @@ -1,10 +1,10 @@ -# RowKindExtractor +# 行变更类型提取 -> RowKindExtractor 转换插件 +> RowKindExtractor：将 CDC 的 RowKind 提取为字段，并转换为 Append-Only 输出 ## 描述 -RowKindExtractor 转换插件用于将 CDC（Change Data Capture）数据流转换为 Append-Only（仅追加）模式，同时将原始的 RowKind 信息提取为一个新的字段。 +RowKindExtractor 转换插件用于将 CDC（Change Data Capture）数据流改写为 Append-Only（仅追加）模式，同时把原始的 RowKind 信息提取为一个新的字段。 **核心功能：** - 将所有数据行的 RowKind 统一改为 `+I`（INSERT），实现 Append-Only 模式 diff --git a/docs/zh/transforms/table-filter.md b/docs/zh/transforms/table-filter.md index 92e34720cc22..ab77eb6726fb 100644 --- a/docs/zh/transforms/table-filter.md +++ b/docs/zh/transforms/table-filter.md @@ -1,21 +1,21 @@ -# TableFilter +# 表过滤 -> TableFilter transform plugin +> TableFilter：按库名、schema 或表名规则筛选需要处理的表 -## Description +## 描述 -表过滤 transform，用于正向或者反向过滤部分表 +TableFilter 转换插件用于按表名、库名或 schema 规则，正向或反向过滤部分表。 -## Options +## 参数 -| name | type | required | default value | Description | -|:----------------:|--------|----------|---------------|--------------------------------------------------------| -| database_pattern | string | no | | 指定数据库过滤模式，默认值为 null，表示不过滤。如果要过滤数据库名称，请将其设置为正则表达式。 | -| schema_pattern | string | no | | 指定 schema 过滤模式，默认值为 null，表示不过滤。如果要过滤架构名称，请将其设置为正则表达式。 | -| table_pattern | string | no | | 指定表过滤模式，默认值为 null，表示不过滤。如果要过滤表名称，请将其设置为正则表达式。 | -| pattern_mode | string | no | INCLUDE | 指定过滤模式，默认值为 INCLUDE，表示包含匹配的表。如果要排除匹配的表，请将其设置为 EXCLUDE。 | +| 参数名 | 类型 | 是否必填 | 默认值 | 说明 | +|:----------------:|--------|----------|--------|------| +| database_pattern | string | 否 | | 数据库过滤规则。默认不过滤；如需过滤数据库名称，请填写正则表达式。 | +| schema_pattern | string | 否 | | schema 过滤规则。默认不过滤；如需过滤 schema 名称，请填写正则表达式。 | +| table_pattern | string | 否 | | 表过滤规则。默认不过滤；如需过滤表名称，请填写正则表达式。 | +| pattern_mode | string | 否 | INCLUDE | 过滤模式。`INCLUDE` 表示保留匹配的表，`EXCLUDE` 表示排除匹配的表。 | -## Examples +## 示例 ### 包含表过滤 @@ -48,4 +48,4 @@ transform { pattern_mode = "EXCLUDE" } } -``` \ No newline at end of file +``` diff --git a/docs/zh/transforms/table-merge.md b/docs/zh/transforms/table-merge.md index 1f796853f9b2..04d6aac8d433 100644 --- a/docs/zh/transforms/table-merge.md +++ b/docs/zh/transforms/table-merge.md @@ -1,20 +1,20 @@ -# TableMerge +# 多表合并 -> TableMerge transform plugin +> TableMerge：将分库分表或多张来源表合并为统一输出表 -## Description +## 描述 -表合并插件，用于分库分表合并为一个表。 +TableMerge 转换插件用于将分库分表或多张来源表合并为统一输出表。 -## Options +## 参数 -| name | type | required | default value | Description | -|:--------:|--------|----------|---------------|------------------| -| database | string | no | | 指定新的 database 名称 | -| schema | string | no | | 指定新的 schema 名称 | -| table | string | yes | | 指定新的 table 名称 | +| 参数名 | 类型 | 是否必填 | 默认值 | 说明 | +|:--------:|--------|----------|--------|------| +| database | string | 否 | | 合并后新的 database 名称 | +| schema | string | 否 | | 合并后新的 schema 名称 | +| table | string | 是 | | 合并后新的 table 名称 | -## Examples +## 示例 ### 合并分库分表为一个表 diff --git a/docs/zh/transforms/table-rename.md b/docs/zh/transforms/table-rename.md index 96035f5234c6..5402061d11aa 100644 --- a/docs/zh/transforms/table-rename.md +++ b/docs/zh/transforms/table-rename.md @@ -1,6 +1,6 @@ # 表重命名 -> TableRename 转换插件 +> TableRename：按规则重命名输出表名 ## 描述 @@ -20,6 +20,7 @@ TableRename 转换插件用于重命名表名。 ### 将表名转为大写 ``` + env { parallelism = 1 job.mode = "STREAMING" @@ -130,5 +131,3 @@ sink { } } ``` - - diff --git a/docs/zh/transforms/transform-multi-table.md b/docs/zh/transforms/transform-multi-table.md index 6517fc51b911..8fa346a42245 100644 --- a/docs/zh/transforms/transform-multi-table.md +++ b/docs/zh/transforms/transform-multi-table.md @@ -2,10 +2,11 @@ sidebar_position: 2 --- -# Transform的多表转换 +# 多表转换 -SeaTunnel transform支持多表转换，在上游插件输出多个表的时候特别有用，能够在一个transform中完成所有的转换操作。目前SeaTunnel很多Connectors支持多表输出，比如`JDBCSource`、`MySQL-CDC` -等。所有的Transform都可以通过如下配置实现多表转换。 +多表转换（Multi-Table Transform）允许你在一个 transform 中为多张来源表分别定义处理规则，适合上游一次输出多表的场景。 + +SeaTunnel 的 Multi-Table Transform 支持在上游插件输出多个表时，在一个 transform 中完成所有转换操作。目前很多 connector 支持多表输出，比如 `JDBCSource`、`MySQL-CDC` 等。所有 transform 都可以通过如下配置实现多表转换。 :::tip @@ -121,4 +122,3 @@ transform { | id | name | age | 我们使用了Copy Transform作为了示例，实际上所有的Transform都支持多表转换，只需要在对应的Transform中配置即可。 - From e2951f2064e568d7880097d2b5d37f72caf5d610 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Fri, 12 Jun 2026 16:56:07 +0800 Subject: [PATCH 006/375] [Feature][API] Add EXTENSION operator for pluggable validation in ConditionOperator (#11048) --- .../configuration-and-option-system.md | 68 ++ docs/en/engines/zeta/rest-api-v2.md | 29 +- .../configuration-and-option-system.md | 68 ++ docs/zh/engines/zeta/rest-api-v2.md | 29 +- .../api/configuration/util/Condition.java | 20 +- .../util/ConditionEvaluators.java | 8 + .../util/ConditionExtension.java | 63 ++ .../configuration/util/ConditionOperator.java | 15 +- .../api/configuration/util/Conditions.java | 9 +- .../util/ConfigValidatorTest.java | 706 +++++++++++++++++- .../configuration/util/OptionRuleTest.java | 30 + .../command/MetadataExportCommand.java | 10 +- .../command/MetadataExportCommandTest.java | 91 +++ .../rest/service/OptionRulesService.java | 6 +- .../rest/service/OptionRulesServiceTest.java | 43 ++ 15 files changed, 1175 insertions(+), 20 deletions(-) create mode 100644 seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionExtension.java create mode 100644 seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/MetadataExportCommandTest.java diff --git a/docs/en/architecture/configuration-and-option-system.md b/docs/en/architecture/configuration-and-option-system.md index c63fe1fb2602..5ba5cc782803 100644 --- a/docs/en/architecture/configuration-and-option-system.md +++ b/docs/en/architecture/configuration-and-option-system.md @@ -131,6 +131,7 @@ Available operators (all accessed via the `Conditions` factory class): | Cross-field | `lessOrEqualField(option, other)` | value <= another option's value | | Cross-field | `greaterThanField(option, other)` | value > another option's value | | Cross-field | `greaterOrEqualField(option, other)` | value >= another option's value | +| Extension | `Conditions.extension(option, ext)` | delegates to a `ConditionExtension` implementation | :::tip Multiple conditions can be chained with `.and(...)` or `.or(...)` to form compound constraints. AND binds tighter than OR, so `A.or(B).and(C)` evaluates as `A || (B && C)`. @@ -206,6 +207,7 @@ Quick reference: | Validate value only when trigger matches | `.conditional(trigger, value, condition...)` | | Optional field with value check when present | `.optional(opt, condition...)` | | Cross-field comparisons | `Conditions.lessThanField/greaterThanField(...)` | +| Custom / structural validation | `Conditions.extension(opt, ext)` | ### Required fields @@ -395,6 +397,72 @@ When two optional fields are provided together, their values must satisfy a cros Conditions.lessThanField(START_TS, END_TS)) ``` +### Custom validation with Extension + +When built-in operators are not expressive enough — for example, validating the internal structure of a `List` or enforcing cross-key constraints inside nested configs — use the `EXTENSION` operator. + +Implement `ConditionExtension` and wire it via `Conditions.extension(option, ext)`. The extension plugs into the same `valueConstraints` pipeline as all built-in operators, so it works with `.and()` / `.or()`, `required`, `optional`, and `conditional` rules. + +Inline anonymous class: + +```java +.optional(API_KEY_ENCODED, Conditions.extension(API_KEY_ENCODED, + new ConditionExtension() { + @Override + public String description() { + return "must be Base64-encoded 'id:api_key'"; + } + + @Override + public boolean evaluate(ReadonlyConfig cfg, String v) + throws OptionValidationException { + try { + return new String(Base64.getDecoder().decode(v)).contains(":"); + } catch (IllegalArgumentException e) { + return false; + } + } + })) +``` + +Static inner class for complex types: + +```java +static class TableConfigsValidator + implements ConditionExtension>> { + @Override + public String description() { + return "each entry must contain a non-empty 'table_name', and all table names must be unique"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List> value) throws OptionValidationException { + if (value.isEmpty()) { + return false; + } + Set seen = new HashSet<>(); + for (Map entry : value) { + Object name = entry.get("table_name"); + if (!(name instanceof String) || ((String) name).isEmpty()) { + return false; + } + if (!seen.add((String) name)) { + return false; + } + } + return true; + } +} + +.exclusive(TABLE_CONFIGS, SCHEMA) +.optional(TABLE_CONFIGS, + Conditions.extension(TABLE_CONFIGS, new TableConfigsValidator())) +``` + +:::caution +`ConditionExtension.evaluate()` runs during job submission validation only. REST metadata queries only serialize `description()` and do not invoke `evaluate()`. Implementations should avoid I/O (database connections, HTTP calls, file access) and only validate structure and values. +::: + ## Why It Matters For Operators This architecture is also what makes the `option-rules` REST endpoint useful. Tools can inspect the runtime metadata of installed connectors and dynamically understand: diff --git a/docs/en/engines/zeta/rest-api-v2.md b/docs/en/engines/zeta/rest-api-v2.md index ccde0b04cb39..672e3f962dbb 100644 --- a/docs/en/engines/zeta/rest-api-v2.md +++ b/docs/en/engines/zeta/rest-api-v2.md @@ -110,6 +110,10 @@ Please refer [security](security.md) ] }, "expectValue": "TEMPLATE", + "compareOperator": null, + "compareOption": null, + "conditionOperator": "EQUAL", + "conditionOperatorCategory": "EQUALITY", "operator": null, "next": null }, @@ -139,6 +143,26 @@ Please refer [security](security.md) "operator": null, "next": null } + }, + { + "expression": "'port' must be between 1 and 65535", + "conditionTree": { + "option": { + "key": "port", + "type": "java.lang.Integer", + "defaultValue": null, + "description": "Server port", + "fallbackKeys": [], + "optionValues": null + }, + "expectValue": "must be between 1 and 65535", + "compareOperator": "extension", + "compareOption": null, + "conditionOperator": "EXTENSION", + "conditionOperatorCategory": "EXTENSION", + "operator": null, + "next": null + } } ] } @@ -151,8 +175,9 @@ Please refer [security](security.md) - `optionRule.conditionRules` recursively exposes nested conditional option rules and is an empty array when the connector does not define nested rules. - For conditional rules, both `expression` and `expressionTree` are returned for dynamic form rendering. - `optionRule.valueConstraints` describes value-level validation rules such as numeric ranges, string patterns, and cross-field comparisons. Each entry provides a human-readable `expression` string alongside a structured `conditionTree` for programmatic use. This array is empty when the connector does not define any value constraints. -- Within `conditionTree`, the `compareOperator` field (e.g. `>=`, `<`, `>`) and `compareOption` field are populated for numeric and cross-field comparisons. For equality checks and other non-comparison conditions, these fields are `null`. -- The `conditionOperator` field provides a stable, machine-readable operator identifier (e.g. `GREATER_OR_EQUAL`, `NOT_BLANK`, `FIELD_LESS_THAN`), while `conditionOperatorCategory` indicates the operator's category (e.g. `NUMERIC`, `STRING`, `COLLECTION`, `EQUALITY`). These two fields are designed for programmatic consumption by frontend applications and automation tools. +- Within `conditionTree`, the `compareOperator` field is `null` for `EQUAL` and otherwise uses the operator symbol exposed by the runtime rule (for example `>=`, `is not blank`, or `extension`). The `compareOption` field is populated only for cross-field comparisons. +- `conditionOperator` is a stable operator identifier. Possible values include `EQUAL`, `GREATER_OR_EQUAL`, `NOT_BLANK`, `FIELD_LESS_THAN`, `EXTENSION`, etc. `conditionOperatorCategory` indicates the operator category, such as `NUMERIC`, `STRING`, `COLLECTION`, `EQUALITY`, `EXTENSION`, etc. +- For `EXTENSION` conditions, `expectValue` carries the rule description text returned by `ConditionExtension.description()`. diff --git a/docs/zh/architecture/configuration-and-option-system.md b/docs/zh/architecture/configuration-and-option-system.md index e9b956a998ed..7f437ff246d8 100644 --- a/docs/zh/architecture/configuration-and-option-system.md +++ b/docs/zh/architecture/configuration-and-option-system.md @@ -131,6 +131,7 @@ public OptionRule optionRule() { | 跨字段 | `lessOrEqualField(option, other)` | 值 <= 另一个配置项的值 | | 跨字段 | `greaterThanField(option, other)` | 值 > 另一个配置项的值 | | 跨字段 | `greaterOrEqualField(option, other)` | 值 >= 另一个配置项的值 | +| 扩展 | `Conditions.extension(option, ext)` | 委托给 `ConditionExtension` 实现类执行自定义校验 | :::tip 多个条件可以通过 `.and(...)` 或 `.or(...)` 链式组合成复合约束。AND 优先级高于 OR，因此 `A.or(B).and(C)` 等价于 `A || (B && C)`。 @@ -206,6 +207,7 @@ Option validation failed (4 errors): | 条件触发的值校验 | `.conditional(trigger, value, condition...)` | | 可选字段（提供时校验） | `.optional(opt, condition...)` | | 跨字段比较 | `Conditions.lessThanField/greaterThanField(...)` | +| 自定义 / 结构化校验 | `Conditions.extension(opt, ext)` | ### 必填字段 @@ -395,6 +397,72 @@ AND 优先级高于 OR，因此 `A.or(B.and(C))` 等价于 `A || (B && C)`。适 Conditions.lessThanField(START_TS, END_TS)) ``` +### 自定义校验（Extension 扩展算子） + +当内置算子无法满足需求时 — 例如校验 `List` 内部结构，或对嵌套配置做跨 key 约束 — 使用 `EXTENSION` 算子。 + +实现 `ConditionExtension` 接口，通过 `Conditions.extension(option, ext)` 接入。扩展算子复用与内置算子完全相同的 `valueConstraints` 管线，支持 `.and()` / `.or()` 链式组合，适用于 `required`、`optional`、`conditional` 等所有规则类型。 + +匿名内部类写法： + +```java +.optional(API_KEY_ENCODED, Conditions.extension(API_KEY_ENCODED, + new ConditionExtension() { + @Override + public String description() { + return "must be Base64-encoded 'id:api_key'"; + } + + @Override + public boolean evaluate(ReadonlyConfig cfg, String v) + throws OptionValidationException { + try { + return new String(Base64.getDecoder().decode(v)).contains(":"); + } catch (IllegalArgumentException e) { + return false; + } + } + })) +``` + +静态内部类写法（适合复杂类型）： + +```java +static class TableConfigsValidator + implements ConditionExtension>> { + @Override + public String description() { + return "each entry must contain a non-empty 'table_name', and all table names must be unique"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List> value) throws OptionValidationException { + if (value.isEmpty()) { + return false; + } + Set seen = new HashSet<>(); + for (Map entry : value) { + Object name = entry.get("table_name"); + if (!(name instanceof String) || ((String) name).isEmpty()) { + return false; + } + if (!seen.add((String) name)) { + return false; + } + } + return true; + } +} + +.exclusive(TABLE_CONFIGS, SCHEMA) +.optional(TABLE_CONFIGS, + Conditions.extension(TABLE_CONFIGS, new TableConfigsValidator())) +``` + +:::caution +`ConditionExtension.evaluate()` 仅在作业提交校验时执行，REST 元数据查询只序列化 `description()`，不会调用 `evaluate()`。实现时应避免 I/O 操作（如数据库连接、HTTP 请求、文件读写），只做结构和值校验。 +::: + ## 为什么这对运维也重要 这套设计也是 `option-rules` REST 接口能够成立的原因。运维平台或 UI 可以通过运行时元数据动态获知： diff --git a/docs/zh/engines/zeta/rest-api-v2.md b/docs/zh/engines/zeta/rest-api-v2.md index 7cbc0e43f279..b5f3d1365829 100644 --- a/docs/zh/engines/zeta/rest-api-v2.md +++ b/docs/zh/engines/zeta/rest-api-v2.md @@ -108,6 +108,10 @@ seatunnel: ] }, "expectValue": "TEMPLATE", + "compareOperator": null, + "compareOption": null, + "conditionOperator": "EQUAL", + "conditionOperatorCategory": "EQUALITY", "operator": null, "next": null }, @@ -137,6 +141,26 @@ seatunnel: "operator": null, "next": null } + }, + { + "expression": "'port' must be between 1 and 65535", + "conditionTree": { + "option": { + "key": "port", + "type": "java.lang.Integer", + "defaultValue": null, + "description": "Server port", + "fallbackKeys": [], + "optionValues": null + }, + "expectValue": "must be between 1 and 65535", + "compareOperator": "extension", + "compareOption": null, + "conditionOperator": "EXTENSION", + "conditionOperatorCategory": "EXTENSION", + "operator": null, + "next": null + } } ] } @@ -149,8 +173,9 @@ seatunnel: - `optionRule.conditionRules` 会递归返回嵌套条件规则；当 connector 未定义嵌套规则时，该字段返回空数组。 - 对于条件规则，会同时返回 `expression` 和 `expressionTree`，便于 Web 做动态表单渲染。 - `optionRule.valueConstraints` 描述值级别的校验规则，包括数值范围、字符串模式匹配以及跨字段比较等。每个条目同时提供人类可读的 `expression` 字符串和便于程序处理的结构化 `conditionTree`。当连接器未定义值约束时，该数组为空。 -- 在 `conditionTree` 中，`compareOperator` 字段（如 `>=`、`<`、`>`）和 `compareOption` 字段用于数值比较和跨字段比较场景；对于等值判断及其他非比较类条件，这两个字段为 `null`。 -- `conditionOperator` 字段提供稳定的、机器可读的操作符标识（如 `GREATER_OR_EQUAL`、`NOT_BLANK`、`FIELD_LESS_THAN`），`conditionOperatorCategory` 字段标明操作符所属分类（如 `NUMERIC`、`STRING`、`COLLECTION`、`EQUALITY`）。这两个字段专为前端应用和自动化工具的程序化消费而设计。 +- 在 `conditionTree` 中，`compareOperator` 字段在 `EQUAL` 场景下为 `null`，其他情况下会返回运行时规则暴露的操作符符号，例如 `>=`、`is not blank` 或 `extension`。`compareOption` 字段仅在跨字段比较场景下有值。 +- `conditionOperator` 是稳定的操作符标识，可选值包括 `EQUAL`、`GREATER_OR_EQUAL`、`NOT_BLANK`、`FIELD_LESS_THAN`、`EXTENSION` 等；`conditionOperatorCategory` 是操作符的分类，可选值包括 `NUMERIC`、`STRING`、`COLLECTION`、`EQUALITY`、`EXTENSION` 等。 +- 对于 `EXTENSION` 条件，`expectValue` 承载的是 `ConditionExtension.description()` 返回的规则说明文本。 diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/Condition.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/Condition.java index 5511b7ce7dd2..e94c21b9045b 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/Condition.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/Condition.java @@ -32,15 +32,25 @@ public class Condition { private final T expectValue; private final ConditionOperator operator; private final Option compareOption; + private final ConditionExtension extension; private Boolean and = null; private Condition next = null; Condition(Option option, T expectValue) { - this(option, ConditionOperator.EQUAL, expectValue, null); + this(option, ConditionOperator.EQUAL, expectValue, null, null); } Condition( Option option, ConditionOperator operator, T expectValue, Option compareOption) { + this(option, operator, expectValue, compareOption, null); + } + + Condition( + Option option, + ConditionOperator operator, + T expectValue, + Option compareOption, + ConditionExtension extension) { if (option == null) { throw new IllegalArgumentException("Condition option must not be null"); } @@ -61,10 +71,15 @@ public class Condition { "Operator %s requires an expectValue, but expectValue is null", operator.name())); } + if (operator == ConditionOperator.EXTENSION && extension == null) { + throw new IllegalArgumentException( + "Operator EXTENSION requires a non-null ConditionExtension"); + } this.option = option; this.operator = operator; this.expectValue = expectValue; this.compareOption = compareOption; + this.extension = extension; } public static Condition of(Option option, T expectValue) { @@ -219,6 +234,9 @@ private static String conditionToString(Condition cond) { ConditionOperator op = cond.operator; String key = "'" + cond.option.key() + "'"; + if (op == ConditionOperator.EXTENSION) { + return key + " " + cond.extension.description(); + } if (op.getSource() == ConditionOperator.Source.FIELD) { return key + " " + op.getSymbol() + " '" + cond.compareOption.key() + "'"; } diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionEvaluators.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionEvaluators.java index 82a72d08e91c..eacb9566e64a 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionEvaluators.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionEvaluators.java @@ -169,6 +169,14 @@ private static Map createRegistry() { return compareNumbers(v, other) >= 0; }); + // Extension (custom logic delegated to ConditionExtension) + m.put( + ConditionOperator.EXTENSION, + (v, c, cfg) -> { + ConditionExtension ext = (ConditionExtension) c.getExtension(); + return ext.evaluate(cfg, v); + }); + for (ConditionOperator op : ConditionOperator.values()) { if (!m.containsKey(op)) { throw new IllegalStateException( diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionExtension.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionExtension.java new file mode 100644 index 000000000000..204805967852 --- /dev/null +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionExtension.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.api.configuration.util; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; + +/** + * Pluggable validation extension for cases where built-in {@link ConditionOperator} operators are + * not expressive enough — for example, validating the internal structure of a {@code List} or + * enforcing cross-key constraints inside nested configs. + * + *

Wire an implementation via {@link + * Conditions#extension(org.apache.seatunnel.api.configuration.Option, ConditionExtension)}. The + * extension plugs into the same {@code valueConstraints} pipeline as all built-in operators and + * supports {@code .and()} / {@code .or()} chaining, {@code required}, {@code optional}, and {@code + * conditional} rules. + * + *

Implementations should avoid I/O (database connections, HTTP calls, file access) and only + * validate structure and values. {@link #evaluate} runs only during job submission validation; REST + * metadata queries only serialize {@link #description()}. + * + * @param the option value type + */ +public interface ConditionExtension { + + /** + * Rule description used in error messages ({@link Condition#toString()}) and metadata + * serialization (REST {@code /option-rules} and CLI metadata export). + * + * @return non-null description, e.g. {@code "must be between 1 and 65535"} + */ + String description(); + + /** + * Evaluates whether {@code value} passes this validation rule. + * + *

Return {@code false} for simple failure — the framework composes the error from {@link + * #description()} automatically. Throw {@link OptionValidationException} when a richer, + * context-specific message is needed. Avoid other unchecked exceptions — they propagate + * unwrapped. + * + * @param config full configuration context (read-only), available for cross-field checks + * @param value the resolved option value; may be {@code null} + * @return {@code true} if valid + * @throws OptionValidationException for detailed error reporting + */ + boolean evaluate(ReadonlyConfig config, T value) throws OptionValidationException; +} diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionOperator.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionOperator.java index 8dbd07288f23..632c444a0ba1 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionOperator.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionOperator.java @@ -61,24 +61,31 @@ public enum ConditionOperator { FIELD_LESS_THAN("<", Category.NUMERIC, Arity.BINARY, Source.FIELD), FIELD_LESS_OR_EQUAL("<=", Category.NUMERIC, Arity.BINARY, Source.FIELD), FIELD_GREATER_THAN(">", Category.NUMERIC, Arity.BINARY, Source.FIELD), - FIELD_GREATER_OR_EQUAL(">=", Category.NUMERIC, Arity.BINARY, Source.FIELD); + FIELD_GREATER_OR_EQUAL(">=", Category.NUMERIC, Arity.BINARY, Source.FIELD), + + // ==================== Extension ==================== + + EXTENSION("extension", Category.EXTENSION, Arity.EXTENSION, Source.EXTENSION); public enum Category { EQUALITY, NUMERIC, STRING, COLLECTION, - MAP + MAP, + EXTENSION } public enum Arity { UNARY, - BINARY + BINARY, + EXTENSION } public enum Source { LITERAL, - FIELD + FIELD, + EXTENSION } private final String symbol; diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/Conditions.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/Conditions.java index a510abbf1725..90ad9beda1d6 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/Conditions.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/Conditions.java @@ -36,8 +36,6 @@ * .build(); * } * - *

Currently supported operators (19 total, 5 categories): - * *

    *
  • Numeric: {@code greaterThan}, {@code greaterOrEqual}, {@code lessThan}, {@code * lessOrEqual} @@ -47,6 +45,7 @@ *
  • Map: {@code mapNotEmpty}, {@code mapContainsKey}, {@code mapContainsKeys} *
  • Cross-field: {@code lessThanField}, {@code lessOrEqualField}, {@code * greaterThanField}, {@code greaterOrEqualField} + *
  • Extension: {@code extension} (custom logic via {@link ConditionExtension}) *
* *

Additionally, equality checks are available via {@link Condition#of(Option, Object)} (EQUAL) @@ -139,4 +138,10 @@ public static Condition greaterThanField(Option option, Option othe public static Condition greaterOrEqualField(Option option, Option other) { return new Condition<>(option, ConditionOperator.FIELD_GREATER_OR_EQUAL, null, other); } + + // ==================== Extension (pluggable validation) ==================== + + public static Condition extension(Option option, ConditionExtension ext) { + return new Condition<>(option, ConditionOperator.EXTENSION, null, null, ext); + } } diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java index fec4b5e05b99..2b859bb05981 100644 --- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java +++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java @@ -17,6 +17,8 @@ package org.apache.seatunnel.api.configuration.util; +import org.apache.seatunnel.shade.com.fasterxml.jackson.core.type.TypeReference; + import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.OptionTest; import org.apache.seatunnel.api.configuration.Options; @@ -407,7 +409,6 @@ public void testDuplicatedNestedOption() { @Test public void testMultipleValueNestedRule() { OptionRule subOption1 = OptionRule.builder().required(KEY_USERNAME, KEY_PASSWORD).build(); - OptionRule subOption2 = OptionRule.builder().required(KEY_BEARER_TOKEN).build(); OptionRule optionRule = OptionRule.builder() .optional(SINGLE_CHOICE_VALUE_TEST) @@ -453,9 +454,6 @@ public void testMultipleValueNestedRule() { public static final Option DB_NAME = Options.key("db_name").stringType().noDefaultValue().withDescription("database name"); - public static final Option DELIMITER = - Options.key("delimiter").stringType().noDefaultValue().withDescription("delimiter"); - public static final Option START_TS = Options.key("start_ts").longType().noDefaultValue().withDescription("start timestamp"); @@ -2753,7 +2751,7 @@ public void testExclusiveWithOptionalValueConstraint() { // list option present with valid list -> pass Map config6 = new HashMap<>(); - config6.put(TEST_TOPIC.key(), Arrays.asList("topic1")); + config6.put(TEST_TOPIC.key(), Collections.singletonList("topic1")); Assertions.assertDoesNotThrow(() -> validate(config6, rule)); } @@ -2787,4 +2785,702 @@ public void testBundledWithOptionalValueConstraint() { config4.put(TEST_TOPIC.key(), Collections.emptyList()); assertThrows(OptionValidationException.class, () -> validate(config4, rule)); } + + static final Option EXT_PORT = + Options.key("ext.port").intType().noDefaultValue().withDescription("port"); + + static final Option OPT_PORT = + Options.key("opt.port").intType().defaultValue(8080).withDescription("optional port"); + + static class PortRangeExtension implements ConditionExtension { + @Override + public String description() { + return "must be between 1 and 65535"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) + throws OptionValidationException { + return value != null && value >= 1 && value <= 65535; + } + } + + @Test + public void testExtensionRequiredPass() { + OptionRule rule = + OptionRule.builder() + .required( + EXT_PORT, Conditions.extension(EXT_PORT, new PortRangeExtension())) + .build(); + Map config = new HashMap<>(); + config.put("ext.port", 8080); + Assertions.assertDoesNotThrow(() -> validate(config, rule)); + } + + @Test + public void testExtensionRequiredFail() { + OptionRule rule = + OptionRule.builder() + .required( + EXT_PORT, Conditions.extension(EXT_PORT, new PortRangeExtension())) + .build(); + Map config = new HashMap<>(); + config.put("ext.port", 99999); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionOptionalAbsentSkip() { + OptionRule rule = + OptionRule.builder() + .optional( + OPT_PORT, Conditions.extension(OPT_PORT, new PortRangeExtension())) + .build(); + Map config = new HashMap<>(); + Assertions.assertDoesNotThrow(() -> validate(config, rule)); + } + + @Test + public void testExtensionOptionalPresentPass() { + OptionRule rule = + OptionRule.builder() + .optional( + OPT_PORT, Conditions.extension(OPT_PORT, new PortRangeExtension())) + .build(); + Map config = new HashMap<>(); + config.put("opt.port", 443); + Assertions.assertDoesNotThrow(() -> validate(config, rule)); + } + + @Test + public void testExtensionOptionalPresentFail() { + OptionRule rule = + OptionRule.builder() + .optional( + OPT_PORT, Conditions.extension(OPT_PORT, new PortRangeExtension())) + .build(); + Map config = new HashMap<>(); + config.put("opt.port", 0); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionAndBuiltinCombined() { + Condition combined = + greaterOrEqual(EXT_PORT, 1) + .and(Conditions.extension(EXT_PORT, new PortRangeExtension())); + OptionRule rule = OptionRule.builder().required(EXT_PORT, combined).build(); + + Map pass = new HashMap<>(); + pass.put("ext.port", 80); + Assertions.assertDoesNotThrow(() -> validate(pass, rule)); + + Map fail = new HashMap<>(); + fail.put("ext.port", 70000); + assertThrows(OptionValidationException.class, () -> validate(fail, rule)); + } + + @Test + public void testExtensionOrBuiltinCombined() { + ConditionExtension isWellKnown = + new ConditionExtension() { + @Override + public String description() { + return "is a well-known port (1-1023)"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) + throws OptionValidationException { + return value != null && value >= 1 && value <= 1023; + } + }; + Condition combined = + Conditions.extension(EXT_PORT, isWellKnown) + .or(Condition.of(EXT_PORT, ConditionOperator.EQUAL, 8080)); + OptionRule rule = OptionRule.builder().required(EXT_PORT, combined).build(); + + Map passWellKnown = new HashMap<>(); + passWellKnown.put("ext.port", 443); + Assertions.assertDoesNotThrow(() -> validate(passWellKnown, rule)); + + Map passExact = new HashMap<>(); + passExact.put("ext.port", 8080); + Assertions.assertDoesNotThrow(() -> validate(passExact, rule)); + + Map fail = new HashMap<>(); + fail.put("ext.port", 5000); + assertThrows(OptionValidationException.class, () -> validate(fail, rule)); + } + + @Test + public void testExtensionConditionToString() { + Condition cond = Conditions.extension(EXT_PORT, new PortRangeExtension()); + assertEquals("'ext.port' must be between 1 and 65535", cond.toString()); + } + + @Test + public void testExtensionConditionEquals() { + Condition a = Conditions.extension(EXT_PORT, new PortRangeExtension()); + Condition b = Conditions.extension(EXT_PORT, new PortRangeExtension()); + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + + Condition c = + Conditions.extension( + EXT_PORT, + new ConditionExtension() { + @Override + public String description() { + return "different impl"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) { + return value != null; + } + }); + assertEquals(a, c); + assertEquals(a.hashCode(), c.hashCode()); + } + + static final Option>> LIST_MAP_OPTION = + Options.key("rules") + .type(new TypeReference>>() {}) + .noDefaultValue() + .withDescription("list of rule maps"); + + static final Option>>> NESTED_MAP_OPTION = + Options.key("nested.config") + .type(new TypeReference>>>() {}) + .noDefaultValue() + .withDescription("nested map of list of maps"); + + static class ListMapStructureExtension + implements ConditionExtension>> { + @Override + public String description() { + return "each rule must contain 'field' and 'type' keys"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List> value) + throws OptionValidationException { + if (value == null || value.isEmpty()) return false; + for (Map rule : value) { + if (!rule.containsKey("field") || !rule.containsKey("type")) { + return false; + } + } + return true; + } + } + + static class NestedMapExtension + implements ConditionExtension>>> { + @Override + public String description() { + return "each group must have non-empty rules with 'name' key"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Map>> value) + throws OptionValidationException { + if (value == null || value.isEmpty()) return false; + for (Map.Entry>> entry : value.entrySet()) { + List> rules = entry.getValue(); + if (rules == null || rules.isEmpty()) return false; + for (Map rule : rules) { + if (!rule.containsKey("name")) return false; + } + } + return true; + } + } + + @Test + public void testExtensionListMapValidStructure() { + OptionRule rule = + OptionRule.builder() + .required( + LIST_MAP_OPTION, + Conditions.extension( + LIST_MAP_OPTION, new ListMapStructureExtension())) + .build(); + + Map ruleItem1 = new HashMap<>(); + ruleItem1.put("field", "name"); + ruleItem1.put("type", "string"); + Map ruleItem2 = new HashMap<>(); + ruleItem2.put("field", "age"); + ruleItem2.put("type", "int"); + + Map config = new HashMap<>(); + config.put("rules", Arrays.asList(ruleItem1, ruleItem2)); + Assertions.assertDoesNotThrow(() -> validate(config, rule)); + } + + @Test + public void testExtensionListMapMissingKey() { + OptionRule rule = + OptionRule.builder() + .required( + LIST_MAP_OPTION, + Conditions.extension( + LIST_MAP_OPTION, new ListMapStructureExtension())) + .build(); + + Map ruleItem = new HashMap<>(); + ruleItem.put("field", "name"); + + Map config = new HashMap<>(); + config.put("rules", Collections.singletonList(ruleItem)); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionListMapEmptyList() { + OptionRule rule = + OptionRule.builder() + .required( + LIST_MAP_OPTION, + Conditions.extension( + LIST_MAP_OPTION, new ListMapStructureExtension())) + .build(); + + Map config = new HashMap<>(); + config.put("rules", Collections.emptyList()); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionListMapNullValue() { + OptionRule rule = + OptionRule.builder() + .required( + LIST_MAP_OPTION, + Conditions.extension( + LIST_MAP_OPTION, new ListMapStructureExtension())) + .build(); + + Map config = new HashMap<>(); + config.put("rules", null); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionListMapSingleItemValid() { + OptionRule rule = + OptionRule.builder() + .required( + LIST_MAP_OPTION, + Conditions.extension( + LIST_MAP_OPTION, new ListMapStructureExtension())) + .build(); + + Map item = new HashMap<>(); + item.put("field", "id"); + item.put("type", "long"); + + Map config = new HashMap<>(); + config.put("rules", Collections.singletonList(item)); + Assertions.assertDoesNotThrow(() -> validate(config, rule)); + } + + @Test + public void testExtensionListMapPartialInvalid() { + OptionRule rule = + OptionRule.builder() + .required( + LIST_MAP_OPTION, + Conditions.extension( + LIST_MAP_OPTION, new ListMapStructureExtension())) + .build(); + + Map good = new HashMap<>(); + good.put("field", "id"); + good.put("type", "long"); + Map bad = new HashMap<>(); + bad.put("field", "name"); + + Map config = new HashMap<>(); + config.put("rules", Arrays.asList(good, bad)); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionNestedMapValid() { + OptionRule rule = + OptionRule.builder() + .required( + NESTED_MAP_OPTION, + Conditions.extension(NESTED_MAP_OPTION, new NestedMapExtension())) + .build(); + + Map item1 = new HashMap<>(); + item1.put("name", "rule1"); + Map item2 = new HashMap<>(); + item2.put("name", "rule2"); + + Map nested = new HashMap<>(); + nested.put("group_a", Arrays.asList(item1, item2)); + + Map config = new HashMap<>(); + config.put("nested.config", nested); + Assertions.assertDoesNotThrow(() -> validate(config, rule)); + } + + @Test + public void testExtensionNestedMapEmptyGroup() { + OptionRule rule = + OptionRule.builder() + .required( + NESTED_MAP_OPTION, + Conditions.extension(NESTED_MAP_OPTION, new NestedMapExtension())) + .build(); + + Map nested = new HashMap<>(); + nested.put("group_a", Collections.emptyList()); + + Map config = new HashMap<>(); + config.put("nested.config", nested); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionNestedMapMissingNameKey() { + OptionRule rule = + OptionRule.builder() + .required( + NESTED_MAP_OPTION, + Conditions.extension(NESTED_MAP_OPTION, new NestedMapExtension())) + .build(); + + Map item = new HashMap<>(); + item.put("value", "something"); + Map nested = new HashMap<>(); + nested.put("group_a", Collections.singletonList(item)); + + Map config = new HashMap<>(); + config.put("nested.config", nested); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionNestedMapEmptyOuter() { + OptionRule rule = + OptionRule.builder() + .required( + NESTED_MAP_OPTION, + Conditions.extension(NESTED_MAP_OPTION, new NestedMapExtension())) + .build(); + + Map config = new HashMap<>(); + config.put("nested.config", Collections.emptyMap()); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionNestedMapMultiGroupPartialInvalid() { + OptionRule rule = + OptionRule.builder() + .required( + NESTED_MAP_OPTION, + Conditions.extension(NESTED_MAP_OPTION, new NestedMapExtension())) + .build(); + + Map good = new HashMap<>(); + good.put("name", "ok"); + Map bad = new HashMap<>(); + bad.put("other", "missing name"); + Map nested = new HashMap<>(); + nested.put("group_a", Collections.singletonList(good)); + nested.put("group_b", Collections.singletonList(bad)); + + Map config = new HashMap<>(); + config.put("nested.config", nested); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + } + + @Test + public void testExtensionThrowsOptionValidationException() { + OptionRule rule = + OptionRule.builder() + .required( + EXT_PORT, + Conditions.extension( + EXT_PORT, + new ConditionExtension() { + @Override + public String description() { + return "must be even"; + } + + @Override + public boolean evaluate( + ReadonlyConfig config, Integer value) + throws OptionValidationException { + if (value != null && value % 2 != 0) { + throw new OptionValidationException( + "Value %d is odd, must be even", value); + } + return value != null; + } + })) + .build(); + + Map pass = new HashMap<>(); + pass.put("ext.port", 80); + Assertions.assertDoesNotThrow(() -> validate(pass, rule)); + + Map fail = new HashMap<>(); + fail.put("ext.port", 81); + OptionValidationException ex = + assertThrows(OptionValidationException.class, () -> validate(fail, rule)); + Assertions.assertTrue(ex.getMessage().contains("odd")); + } + + @Test + public void testExtensionListMapOptionalAbsent() { + OptionRule rule = + OptionRule.builder() + .optional( + LIST_MAP_OPTION, + Conditions.extension( + LIST_MAP_OPTION, new ListMapStructureExtension())) + .build(); + + Map config = new HashMap<>(); + Assertions.assertDoesNotThrow(() -> validate(config, rule)); + } + + @Test + public void testExtensionListMapToString() { + Condition>> cond = + Conditions.extension(LIST_MAP_OPTION, new ListMapStructureExtension()); + assertEquals("'rules' each rule must contain 'field' and 'type' keys", cond.toString()); + } + + @Test + public void testExtensionNestedMapToString() { + Condition>>> cond = + Conditions.extension(NESTED_MAP_OPTION, new NestedMapExtension()); + assertEquals( + "'nested.config' each group must have non-empty rules with 'name' key", + cond.toString()); + } + + @Test + public void testExtensionListMapAndBuiltinChain() { + Condition>> combined = + notEmpty(LIST_MAP_OPTION) + .and( + Conditions.extension( + LIST_MAP_OPTION, new ListMapStructureExtension())); + OptionRule rule = OptionRule.builder().required(LIST_MAP_OPTION, combined).build(); + + Map config = new HashMap<>(); + config.put("rules", Collections.emptyList()); + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + + Map good = new HashMap<>(); + good.put("field", "x"); + good.put("type", "y"); + Map config2 = new HashMap<>(); + config2.put("rules", Collections.singletonList(good)); + Assertions.assertDoesNotThrow(() -> validate(config2, rule)); + } + + @Test + public void testExtensionAndExtensionChain() { + ConditionExtension positiveExt = + new ConditionExtension() { + @Override + public String description() { + return "must be positive"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) { + return value != null && value > 0; + } + }; + ConditionExtension evenExt = + new ConditionExtension() { + @Override + public String description() { + return "must be even"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) { + return value != null && value % 2 == 0; + } + }; + Condition combined = + Conditions.extension(EXT_PORT, positiveExt) + .and(Conditions.extension(EXT_PORT, evenExt)); + OptionRule rule = OptionRule.builder().required(EXT_PORT, combined).build(); + + Map pass = new HashMap<>(); + pass.put("ext.port", 80); + Assertions.assertDoesNotThrow(() -> validate(pass, rule)); + + // positive but odd -> fail + Map failOdd = new HashMap<>(); + failOdd.put("ext.port", 81); + assertThrows(OptionValidationException.class, () -> validate(failOdd, rule)); + + // even but negative -> fail + Map failNeg = new HashMap<>(); + failNeg.put("ext.port", -2); + assertThrows(OptionValidationException.class, () -> validate(failNeg, rule)); + } + + @Test + public void testExtensionOrExtensionChain() { + ConditionExtension wellKnownExt = + new ConditionExtension() { + @Override + public String description() { + return "well-known port (1-1023)"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) { + return value != null && value >= 1 && value <= 1023; + } + }; + ConditionExtension highPortExt = + new ConditionExtension() { + @Override + public String description() { + return "high port (49152-65535)"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) { + return value != null && value >= 49152 && value <= 65535; + } + }; + Condition combined = + Conditions.extension(EXT_PORT, wellKnownExt) + .or(Conditions.extension(EXT_PORT, highPortExt)); + OptionRule rule = OptionRule.builder().required(EXT_PORT, combined).build(); + + Map passWellKnown = new HashMap<>(); + passWellKnown.put("ext.port", 443); + Assertions.assertDoesNotThrow(() -> validate(passWellKnown, rule)); + + Map passHigh = new HashMap<>(); + passHigh.put("ext.port", 50000); + Assertions.assertDoesNotThrow(() -> validate(passHigh, rule)); + + // middle range -> fail both + Map fail = new HashMap<>(); + fail.put("ext.port", 8080); + assertThrows(OptionValidationException.class, () -> validate(fail, rule)); + } + + @Test + public void testExtensionWithExclusiveOptional() { + ConditionExtension patternExt = + new ConditionExtension() { + @Override + public String description() { + return "must start with a letter"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String value) { + return value != null + && !value.isEmpty() + && Character.isLetter(value.charAt(0)); + } + }; + OptionRule rule = + OptionRule.builder() + .exclusive(TEST_TOPIC_PATTERN, TEST_TOPIC) + .optional( + TEST_TOPIC_PATTERN, + Conditions.extension(TEST_TOPIC_PATTERN, patternExt)) + .optional(TEST_TOPIC, notEmpty(TEST_TOPIC)) + .build(); + + // neither present -> fails exclusive + Map config1 = new HashMap<>(); + assertThrows(OptionValidationException.class, () -> validate(config1, rule)); + + // pattern present with valid value -> pass + Map config2 = new HashMap<>(); + config2.put(TEST_TOPIC_PATTERN.key(), "topic.*"); + Assertions.assertDoesNotThrow(() -> validate(config2, rule)); + + // pattern present but starts with digit -> fails extension + Map config3 = new HashMap<>(); + config3.put(TEST_TOPIC_PATTERN.key(), "123topic"); + assertThrows(OptionValidationException.class, () -> validate(config3, rule)); + + // both present -> fails exclusive + Map config4 = new HashMap<>(); + config4.put(TEST_TOPIC_PATTERN.key(), "topic.*"); + config4.put(TEST_TOPIC.key(), Collections.singletonList("t1")); + assertThrows(OptionValidationException.class, () -> validate(config4, rule)); + + // topic present with valid list -> pass + Map config5 = new HashMap<>(); + config5.put(TEST_TOPIC.key(), Arrays.asList("t1", "t2")); + Assertions.assertDoesNotThrow(() -> validate(config5, rule)); + } + + @Test + public void testExtensionWithBundledOptional() { + ConditionExtension patternExt = + new ConditionExtension() { + @Override + public String description() { + return "must contain a wildcard"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String value) { + return value != null && value.contains("*"); + } + }; + OptionRule rule = + OptionRule.builder() + .bundled(TEST_TOPIC_PATTERN, TEST_TOPIC) + .optional( + TEST_TOPIC_PATTERN, + Conditions.extension(TEST_TOPIC_PATTERN, patternExt)) + .optional(TEST_TOPIC, notEmpty(TEST_TOPIC)) + .build(); + + // neither present -> pass (bundled group absent) + Map config1 = new HashMap<>(); + Assertions.assertDoesNotThrow(() -> validate(config1, rule)); + + // both present with valid values -> pass + Map config2 = new HashMap<>(); + config2.put(TEST_TOPIC_PATTERN.key(), "topic.*"); + config2.put(TEST_TOPIC.key(), Collections.singletonList("t1")); + Assertions.assertDoesNotThrow(() -> validate(config2, rule)); + + // only one present -> fails bundled + Map config3 = new HashMap<>(); + config3.put(TEST_TOPIC_PATTERN.key(), "topic.*"); + assertThrows(OptionValidationException.class, () -> validate(config3, rule)); + + // both present but pattern has no wildcard -> fails extension + Map config4 = new HashMap<>(); + config4.put(TEST_TOPIC_PATTERN.key(), "topic-fixed"); + config4.put(TEST_TOPIC.key(), Collections.singletonList("t1")); + assertThrows(OptionValidationException.class, () -> validate(config4, rule)); + + // both present but topic is empty -> fails notEmpty + Map config5 = new HashMap<>(); + config5.put(TEST_TOPIC_PATTERN.key(), "topic.*"); + config5.put(TEST_TOPIC.key(), Collections.emptyList()); + assertThrows(OptionValidationException.class, () -> validate(config5, rule)); + } } diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/OptionRuleTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/OptionRuleTest.java index 1c3db5bdfcea..38b9a47adc8e 100644 --- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/OptionRuleTest.java +++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/OptionRuleTest.java @@ -22,6 +22,7 @@ import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.OptionTest; import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -350,6 +351,35 @@ public void testVerify() { TEST_TOPIC_PATTERN, Conditions.notBlank(TEST_TOPIC_PATTERN)) .build(); assertThrows(OptionValidationException.class, executable); + + // test extension condition builds correctly + ConditionExtension positiveExt = + new ConditionExtension() { + @Override + public String description() { + return "must be positive"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) { + return value != null && value > 0; + } + }; + OptionRule extRule = + OptionRule.builder() + .required(TEST_PORTS) + .optional(TEST_NUM, Conditions.extension(TEST_NUM, positiveExt)) + .build(); + Assertions.assertNotNull(extRule); + assertEquals(1, extRule.getValueConstraints().size()); + assertEquals( + ConditionOperator.EXTENSION, extRule.getValueConstraints().get(0).getOperator()); + + // test extension with null extension throws + assertThrows(IllegalArgumentException.class, () -> Conditions.extension(TEST_NUM, null)); + + // test extension with null option throws + assertThrows(IllegalArgumentException.class, () -> Conditions.extension(null, positiveExt)); } @Test diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/MetadataExportCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/MetadataExportCommand.java index 69d6f78b6b73..ff47c66aeb2c 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/MetadataExportCommand.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/MetadataExportCommand.java @@ -420,10 +420,14 @@ private ObjectNode exportCondition(Condition condition) { } ObjectNode node = mapper.createObjectNode(); node.put("key", condition.getOption().key()); - if (condition.getExpectValue() != null) { - node.put("expectValue", String.valueOf(condition.getExpectValue())); - } ConditionOperator op = condition.getOperator(); + Object expectValue = condition.getExpectValue(); + if (op == ConditionOperator.EXTENSION && condition.getExtension() != null) { + expectValue = condition.getExtension().description(); + } + if (expectValue != null) { + node.put("expectValue", String.valueOf(expectValue)); + } if (op != null) { node.put("conditionOperator", op.name()); node.put("conditionOperatorCategory", op.getCategory().name()); diff --git a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/MetadataExportCommandTest.java b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/MetadataExportCommandTest.java new file mode 100644 index 000000000000..19eb432fb749 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/MetadataExportCommandTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.command; + +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; + +import org.apache.seatunnel.api.common.PluginIdentifier; +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.common.constants.PluginType; +import org.apache.seatunnel.core.starter.seatunnel.args.MetadataExportCommandArgs; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class MetadataExportCommandTest { + + @Test + void shouldExportExtensionConditionDescription() throws Exception { + Option port = + Options.key("port").intType().noDefaultValue().withDescription("Port number"); + ConditionExtension portRangeExtension = + new ConditionExtension() { + @Override + public String description() { + return "must be between 1 and 65535"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) { + return value != null && value >= 1 && value <= 65535; + } + }; + OptionRule optionRule = + OptionRule.builder() + .required(port, Conditions.extension(port, portRangeExtension)) + .build(); + + MetadataExportCommand command = new MetadataExportCommand(new MetadataExportCommandArgs()); + Method exportConnector = + MetadataExportCommand.class.getDeclaredMethod( + "exportConnector", + PluginIdentifier.class, + OptionRule.class, + PluginType.class); + exportConnector.setAccessible(true); + + ObjectNode connectorNode = + (ObjectNode) + exportConnector.invoke( + command, + PluginIdentifier.of("seatunnel", "source", "ExtensionSource"), + optionRule, + PluginType.SOURCE); + + JsonNode valueConstraint = connectorNode.get("valueConstraints").get(0); + assertTrue( + valueConstraint.get("expression").asText().contains("must be between 1 and 65535")); + + JsonNode conditionTree = valueConstraint.get("conditionTree"); + assertEquals("port", conditionTree.get("key").asText()); + assertEquals("must be between 1 and 65535", conditionTree.get("expectValue").asText()); + assertEquals("extension", conditionTree.get("compareOperator").asText()); + assertEquals("EXTENSION", conditionTree.get("conditionOperator").asText()); + assertEquals("EXTENSION", conditionTree.get("conditionOperatorCategory").asText()); + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/service/OptionRulesService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/service/OptionRulesService.java index 002a9530033d..a2f8c7ce5721 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/service/OptionRulesService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/service/OptionRulesService.java @@ -282,9 +282,13 @@ private OptionRuleResponse.ConditionNode toConditionNode(Condition condition) : null; String conditionOperator = (op != null) ? op.name() : null; String conditionOperatorCategory = (op != null) ? op.getCategory().name() : null; + Object expectValue = condition.getExpectValue(); + if (op == ConditionOperator.EXTENSION && condition.getExtension() != null) { + expectValue = condition.getExtension().description(); + } return new OptionRuleResponse.ConditionNode( toOptionMetadata(condition.getOption()), - condition.getExpectValue(), + expectValue, compareOperatorSymbol, compareOptionMeta, conditionOperator, diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/service/OptionRulesServiceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/service/OptionRulesServiceTest.java index 151c3ec3e9d5..ac5c7bd4ab6e 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/service/OptionRulesServiceTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/service/OptionRulesServiceTest.java @@ -20,7 +20,9 @@ import org.apache.seatunnel.api.common.PluginIdentifier; import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.SingleChoiceOption; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.engine.server.rest.response.OptionRuleResponse; @@ -360,6 +362,47 @@ void shouldPreserveCrossFieldConstraintMetadata() { assertEquals("end_ts", tree.getCompareOption().getKey()); } + @Test + void shouldPreserveExtensionConstraintMetadata() { + Option port = + Options.key("port").intType().noDefaultValue().withDescription("Port number"); + ConditionExtension portRangeExtension = + new ConditionExtension() { + @Override + public String description() { + return "must be between 1 and 65535"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) { + return value != null && value >= 1 && value <= 65535; + } + }; + + OptionRule optionRule = + OptionRule.builder() + .required(port, Conditions.extension(port, portRangeExtension)) + .build(); + + OptionRuleResponse response = + service.buildResponse( + PluginIdentifier.of("seatunnel", "source", "ExtensionSource"), optionRule); + + List constraints = + response.getOptionRule().getValueConstraints(); + assertEquals(1, constraints.size()); + + OptionRuleResponse.ValueConstraintMetadata constraint = constraints.get(0); + assertTrue(constraint.getExpression().contains("must be between 1 and 65535")); + + OptionRuleResponse.ConditionNode tree = constraint.getConditionTree(); + assertNotNull(tree); + assertEquals("must be between 1 and 65535", tree.getExpectValue()); + assertEquals("extension", tree.getCompareOperator()); + assertEquals("EXTENSION", tree.getConditionOperator()); + assertEquals("EXTENSION", tree.getConditionOperatorCategory()); + } + private enum AuthMode { PASSWORD, TOKEN From 1775c0e87dda5ab3674a0b91e47ee2093b09fcce Mon Sep 17 00:00:00 2001 From: niumy <65696095+niumy0701@users.noreply.github.com> Date: Fri, 12 Jun 2026 19:05:36 +0800 Subject: [PATCH 007/375] [Chore]Restore SeaTunnelEngineLocalExample (#11073) Co-authored-by: niumy --- .../example/engine/SeaTunnelEngineLocalExample.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineLocalExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineLocalExample.java index 6ebe266c3482..3c23400214bb 100644 --- a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineLocalExample.java +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineLocalExample.java @@ -27,7 +27,6 @@ import java.net.URL; import java.nio.file.Paths; -/** Local example that shows how to enable stain trace reporting for an engine job submission. */ public class SeaTunnelEngineLocalExample { static { @@ -37,17 +36,8 @@ public class SeaTunnelEngineLocalExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { - String configurePath = - args.length > 0 ? args[0] : "/examples/stain_trace_fake_sql_union_to_console.conf"; + String configurePath = args.length > 0 ? args[0] : "/examples/fake_to_console.conf"; String configFile = getTestConfigFile(configurePath); - - // Load the stain-trace engine config (enables stain-trace-enabled + file output path). - // Only applied when the user has not explicitly overridden it via -Dseatunnel.config. - if (System.getProperty("seatunnel.config") == null) { - String engineConfigFile = getTestConfigFile("/examples/stain_trace_seatunnel.yaml"); - System.setProperty("seatunnel.config", engineConfigFile); - } - ClientCommandArgs clientCommandArgs = new ClientCommandArgs(); clientCommandArgs.setConfigFile(configFile); clientCommandArgs.setCheckConfig(false); From f3461c9ba96998f5ddfa880a3e43a72df2cf327a Mon Sep 17 00:00:00 2001 From: jieguiQu <15037143579@163.com> Date: Fri, 12 Jun 2026 19:25:31 +0800 Subject: [PATCH 008/375] [Docs]Repair document links (#11033) Co-authored-by: qgj <1660791719@qq.com> --- config/v2.batch.config.template | 4 ++-- config/v2.streaming.conf.template | 4 ++-- docs/en/connectors/sink/DB2.md | 6 +++--- docs/en/connectors/sink/HdfsFile.md | 6 +++--- docs/en/connectors/sink/Kingbase.md | 6 +++--- docs/en/connectors/sink/Mysql.md | 6 +++--- docs/en/connectors/sink/OceanBase.md | 6 +++--- docs/en/connectors/sink/Oracle.md | 6 +++--- docs/en/connectors/sink/PostgreSql.md | 6 +++--- docs/en/connectors/sink/S3File.md | 6 +++--- docs/en/connectors/sink/Snowflake.md | 6 +++--- docs/en/connectors/sink/SqlServer.md | 8 ++++---- docs/en/connectors/sink/Vertica.md | 6 +++--- docs/en/connectors/source/AmazonSqs.md | 2 +- docs/en/connectors/source/DB2.md | 2 +- docs/en/connectors/source/Doris.md | 6 +++--- docs/en/connectors/source/DuckDB.md | 2 +- docs/en/connectors/source/Kingbase.md | 2 +- docs/en/connectors/source/OceanBase.md | 2 +- docs/en/connectors/source/Oracle.md | 2 +- docs/en/connectors/source/SqlServer.md | 10 +++++----- docs/en/connectors/source/Vertica.md | 2 +- docs/zh/connectors/source/Doris.md | 6 +++--- docs/zh/connectors/source/Kingbase.md | 2 +- docs/zh/connectors/source/OceanBase.md | 2 +- .../src/main/resources/maxcompute_to_maxcompute.conf | 4 ++-- .../src/test/resources/config.variables.conf | 4 ++-- .../test/java/resources/test_flink_run_parameter.conf | 2 +- .../src/test/resources/fake_source_to_sink.conf | 4 ++-- .../src/test/resources/localfile_source_to_sink.conf | 4 ++-- .../resources/amazondynamodbIT_source_to_sink.conf | 4 ++-- .../src/test/resources/amazonsqsIT_source_to_sink.conf | 2 +- .../test/resources/assertion/fakesource_to_assert.conf | 2 +- .../src/test/resources/cassandra_to_cassandra.conf | 2 +- .../src/test/resources/clickhouse_to_clickhouse.conf | 4 ++-- .../src/test/resources/clickhouse_to_console.conf | 4 ++-- .../clickhouse_with_create_schema_when_comment.conf | 4 ++-- .../resources/clickhouse_with_join_complex_sql.conf | 4 ++-- .../resources/clickhouse_with_multi_table_source.conf | 2 +- .../clickhouse_with_parallelism_add_filter_query.conf | 4 ++-- ...clickhouse_with_parallelism_add_partition_list.conf | 4 ++-- .../resources/clickhouse_with_parallelism_read.conf | 4 ++-- .../clickhouse_with_sql_and_filter_query.conf | 4 ++-- .../resources/firestore/fake_to_google_firestore.conf | 2 +- .../src/test/resources/jdbc_db2_source_and_sink.conf | 4 ++-- .../resources/jdbc_db2_source_and_sink_upsert.conf | 4 ++-- .../src/test/resources/jdbc_oracle_source_to_sink.conf | 4 ++-- .../jdbc_oracle_source_to_sink_use_select1.conf | 4 ++-- .../jdbc_oracle_source_to_sink_use_select2.conf | 4 ++-- .../jdbc_oracle_source_to_sink_use_select3.conf | 4 ++-- ...jdbc_oracle_source_to_sink_with_blob_as_string.conf | 4 ++-- ...dbc_oracle_source_with_multiple_tables_to_sink.conf | 2 +- ...jdbc_oracle_source_with_pattern_tables_to_sink.conf | 2 +- .../jdbc_oceanbase_mysql_source_and_sink.conf | 4 ++-- .../test/resources/jdbc_phoenix_source_and_sink.conf | 6 +++--- .../test/resources/jdbc_teradata_source_and_sink.conf | 4 ++-- .../test/resources/jdbc_sqlserver_source_to_sink.conf | 6 +++--- .../resources/jdbc_cloudberry_source_and_sink.conf | 6 +++--- .../test/resources/jdbc_gbase8a_source_to_assert.conf | 4 ++-- .../test/resources/jdbc_greenplum_source_and_sink.conf | 6 +++--- .../kafka/kafkasource_earliest_to_console.conf | 2 +- .../kafka/kafkasource_endTimestamp_to_console.conf | 2 +- ...source_format_error_handle_way_fail_to_console.conf | 2 +- ...source_format_error_handle_way_skip_to_console.conf | 2 +- .../kafka/kafkasource_group_offset_to_console.conf | 2 +- ...rce_group_offset_to_console_with_commit_offset.conf | 2 +- .../resources/kafka/kafkasource_latest_to_console.conf | 2 +- .../kafka/kafkasource_specific_offsets_to_console.conf | 2 +- .../kafka/kafkasource_timestamp_to_console.conf | 2 +- ...afkasource_timestamp_to_console_skip_partition.conf | 2 +- .../resources/tdengine/tdengine_source_to_sink.conf | 4 ++-- .../src/test/resources/fake_to_console.variables.conf | 4 ++-- .../resources/stream_fake_to_console_biginterval.conf | 4 ++-- .../stream_fake_to_console_with_checkpoint.conf | 4 ++-- .../main/resources/examples/fake_to_console_batch.conf | 4 ++-- .../resources/examples/fake_to_console_streaming.conf | 4 ++-- .../main/resources/examples/fake_to_console_batch.conf | 4 ++-- .../resources/examples/fake_to_console_streaming.conf | 4 ++-- .../main/resources/examples/fake_to_console_batch.conf | 4 ++-- .../resources/examples/fake_to_console_streaming.conf | 4 ++-- .../src/main/resources/examples/spark.batch.conf | 4 ++-- 81 files changed, 156 insertions(+), 156 deletions(-) diff --git a/config/v2.batch.config.template b/config/v2.batch.config.template index 5affe8af4238..2e85cfb5c967 100644 --- a/config/v2.batch.config.template +++ b/config/v2.batch.config.template @@ -40,7 +40,7 @@ source { } # If you would like to get more information about how to configure SeaTunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } sink { @@ -48,5 +48,5 @@ sink { } # If you would like to get more information about how to configure SeaTunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/config/v2.streaming.conf.template b/config/v2.streaming.conf.template index 9211659b6f8f..625258b5706d 100644 --- a/config/v2.streaming.conf.template +++ b/config/v2.streaming.conf.template @@ -40,7 +40,7 @@ source { } # If you would like to get more information about how to configure SeaTunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } sink { @@ -48,5 +48,5 @@ sink { } # If you would like to get more information about how to configure SeaTunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/docs/en/connectors/sink/DB2.md b/docs/en/connectors/sink/DB2.md index f31da697ba05..7941dc136c15 100644 --- a/docs/en/connectors/sink/DB2.md +++ b/docs/en/connectors/sink/DB2.md @@ -112,12 +112,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -129,7 +129,7 @@ sink { query = "insert into test_table(name,age) values(?,?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/HdfsFile.md b/docs/en/connectors/sink/HdfsFile.md index 2d80255d1f82..2888722e05dc 100644 --- a/docs/en/connectors/sink/HdfsFile.md +++ b/docs/en/connectors/sink/HdfsFile.md @@ -156,12 +156,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -171,7 +171,7 @@ sink { file_format_type = "orc" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/Kingbase.md b/docs/en/connectors/sink/Kingbase.md index 1f4142d1dd5c..db131d024ab1 100644 --- a/docs/en/connectors/sink/Kingbase.md +++ b/docs/en/connectors/sink/Kingbase.md @@ -126,12 +126,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -143,7 +143,7 @@ sink { query = "insert into test_table(c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double,c_decimal,c_date,c_time,c_timestamp) values(?,?,?,?,?,?,?,?,?,?,?,?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/Mysql.md b/docs/en/connectors/sink/Mysql.md index 4b399a5cbcbd..393996efa74a 100644 --- a/docs/en/connectors/sink/Mysql.md +++ b/docs/en/connectors/sink/Mysql.md @@ -124,12 +124,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -141,7 +141,7 @@ sink { query = "insert into test_table(name,age) values(?,?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/OceanBase.md b/docs/en/connectors/sink/OceanBase.md index fa69bf978223..178480167ffd 100644 --- a/docs/en/connectors/sink/OceanBase.md +++ b/docs/en/connectors/sink/OceanBase.md @@ -122,12 +122,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -140,7 +140,7 @@ sink { query = "insert into test_table(name,age) values(?,?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/Oracle.md b/docs/en/connectors/sink/Oracle.md index 9730f2983fcb..9617b0d44b7f 100644 --- a/docs/en/connectors/sink/Oracle.md +++ b/docs/en/connectors/sink/Oracle.md @@ -121,12 +121,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -138,7 +138,7 @@ sink { query = "INSERT INTO TEST.TEST_TABLE(NAME,AGE) VALUES(?,?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/PostgreSql.md b/docs/en/connectors/sink/PostgreSql.md index 2cbaf97e0f55..e1ae7339abb4 100644 --- a/docs/en/connectors/sink/PostgreSql.md +++ b/docs/en/connectors/sink/PostgreSql.md @@ -165,12 +165,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -183,7 +183,7 @@ sink { query = "insert into test_table(name,age) values(?,?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/S3File.md b/docs/en/connectors/sink/S3File.md index 694539e2bf3b..4e85a346edab 100644 --- a/docs/en/connectors/sink/S3File.md +++ b/docs/en/connectors/sink/S3File.md @@ -373,12 +373,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -406,7 +406,7 @@ sink { } } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/Snowflake.md b/docs/en/connectors/sink/Snowflake.md index 060c5654ae15..ca0d55bdfe11 100644 --- a/docs/en/connectors/sink/Snowflake.md +++ b/docs/en/connectors/sink/Snowflake.md @@ -100,11 +100,11 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { jdbc { @@ -115,7 +115,7 @@ sink { query = "insert into test_table(name,age) values(?,?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/sink/SqlServer.md b/docs/en/connectors/sink/SqlServer.md index 46fef5be834e..2431f010c3ff 100644 --- a/docs/en/connectors/sink/SqlServer.md +++ b/docs/en/connectors/sink/SqlServer.md @@ -120,13 +120,13 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/source/Jdbc } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -138,7 +138,7 @@ sink { query = "insert into full_types_jdbc_sink( id, val_char, val_varchar, val_text, val_nchar, val_nvarchar, val_ntext, val_decimal, val_numeric, val_float, val_real, val_smallmoney, val_money, val_bit, val_tinyint, val_smallint, val_int, val_bigint, val_date, val_time, val_datetime2, val_datetime, val_smalldatetime ) values( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } ``` @@ -177,7 +177,7 @@ Jdbc { xa_data_source_class_name = "com.microsoft.sqlserver.jdbc.SQLServerXADataSource" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc ``` diff --git a/docs/en/connectors/sink/Vertica.md b/docs/en/connectors/sink/Vertica.md index 9f2365cf5fcc..cc50f370980d 100644 --- a/docs/en/connectors/sink/Vertica.md +++ b/docs/en/connectors/sink/Vertica.md @@ -120,12 +120,12 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2 + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -137,7 +137,7 @@ sink { query = "insert into test_table(name,age) values(?,?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } ``` diff --git a/docs/en/connectors/source/AmazonSqs.md b/docs/en/connectors/source/AmazonSqs.md index a9f8a5ac4f5e..67b179d5615b 100644 --- a/docs/en/connectors/source/AmazonSqs.md +++ b/docs/en/connectors/source/AmazonSqs.md @@ -69,7 +69,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/en/connectors/source/DB2.md b/docs/en/connectors/source/DB2.md index 9ee87f9ad57c..ac5a3366b0bf 100644 --- a/docs/en/connectors/source/DB2.md +++ b/docs/en/connectors/source/DB2.md @@ -111,7 +111,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/en/connectors/source/Doris.md b/docs/en/connectors/source/Doris.md index 96fc277973cd..af1843badabb 100644 --- a/docs/en/connectors/source/Doris.md +++ b/docs/en/connectors/source/Doris.md @@ -113,7 +113,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -141,7 +141,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -169,7 +169,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/en/connectors/source/DuckDB.md b/docs/en/connectors/source/DuckDB.md index e61892147179..3ac8e73c2dfb 100644 --- a/docs/en/connectors/source/DuckDB.md +++ b/docs/en/connectors/source/DuckDB.md @@ -157,7 +157,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/en/connectors/source/Kingbase.md b/docs/en/connectors/source/Kingbase.md index 4c0d74388e55..8da7252ab839 100644 --- a/docs/en/connectors/source/Kingbase.md +++ b/docs/en/connectors/source/Kingbase.md @@ -107,7 +107,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/en/connectors/source/OceanBase.md b/docs/en/connectors/source/OceanBase.md index 0e9d6bdcff64..8bbe66ad40aa 100644 --- a/docs/en/connectors/source/OceanBase.md +++ b/docs/en/connectors/source/OceanBase.md @@ -121,7 +121,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/en/connectors/source/Oracle.md b/docs/en/connectors/source/Oracle.md index 0a1dce5a640a..b26accc384aa 100644 --- a/docs/en/connectors/source/Oracle.md +++ b/docs/en/connectors/source/Oracle.md @@ -198,7 +198,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/en/connectors/source/SqlServer.md b/docs/en/connectors/source/SqlServer.md index b78192501589..92944e0d1863 100644 --- a/docs/en/connectors/source/SqlServer.md +++ b/docs/en/connectors/source/SqlServer.md @@ -182,7 +182,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -217,7 +217,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -251,19 +251,19 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/source/Jdbc } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { Console {} # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } ``` diff --git a/docs/en/connectors/source/Vertica.md b/docs/en/connectors/source/Vertica.md index 0ae510e06421..4f0ecd086201 100644 --- a/docs/en/connectors/source/Vertica.md +++ b/docs/en/connectors/source/Vertica.md @@ -108,7 +108,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/zh/connectors/source/Doris.md b/docs/zh/connectors/source/Doris.md index e73f6410f1b5..f15bdf584ac3 100644 --- a/docs/zh/connectors/source/Doris.md +++ b/docs/zh/connectors/source/Doris.md @@ -113,7 +113,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -141,7 +141,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -169,7 +169,7 @@ source{ transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/zh/connectors/source/Kingbase.md b/docs/zh/connectors/source/Kingbase.md index 4980e86e3815..1a56f5fe3e1e 100644 --- a/docs/zh/connectors/source/Kingbase.md +++ b/docs/zh/connectors/source/Kingbase.md @@ -107,7 +107,7 @@ source { transform { # 如果您想了解有关如何配置 seatunnel 的更多信息并查看完整的转换插件列表， - # 请访问 https://seatunnel.apache.org/docs/transform/sql + # 请访问 https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/docs/zh/connectors/source/OceanBase.md b/docs/zh/connectors/source/OceanBase.md index 73b3bda5650a..751b94cdacff 100644 --- a/docs/zh/connectors/source/OceanBase.md +++ b/docs/zh/connectors/source/OceanBase.md @@ -121,7 +121,7 @@ source { transform { # 如果您想了解有关如何配置 seatunnel 的更多信息并查看完整的转换插件列表， - # 请访问 https://seatunnel.apache.org/docs/transform/sql + # 请访问 https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/seatunnel-connectors-v2/connector-maxcompute/src/main/resources/maxcompute_to_maxcompute.conf b/seatunnel-connectors-v2/connector-maxcompute/src/main/resources/maxcompute_to_maxcompute.conf index fb7901300b15..f3f394bc52e0 100644 --- a/seatunnel-connectors-v2/connector-maxcompute/src/main/resources/maxcompute_to_maxcompute.conf +++ b/seatunnel-connectors-v2/connector-maxcompute/src/main/resources/maxcompute_to_maxcompute.conf @@ -66,7 +66,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -91,5 +91,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-core/seatunnel-core-starter/src/test/resources/config.variables.conf b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.variables.conf index e0758e804168..491c0792cc4b 100644 --- a/seatunnel-core/seatunnel-core-starter/src/test/resources/config.variables.conf +++ b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.variables.conf @@ -40,7 +40,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -64,5 +64,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/test/java/resources/test_flink_run_parameter.conf b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/test/java/resources/test_flink_run_parameter.conf index e11699bd6648..a1fe7eb12cb6 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/test/java/resources/test_flink_run_parameter.conf +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/test/java/resources/test_flink_run_parameter.conf @@ -62,7 +62,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink{ diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/src/test/resources/fake_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/src/test/resources/fake_source_to_sink.conf index b06970806f1e..f7f05337e74c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/src/test/resources/fake_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/src/test/resources/fake_source_to_sink.conf @@ -82,7 +82,7 @@ source { # } # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source transform { @@ -113,5 +113,5 @@ sink { # } # If you would like to get more information about how to configure seatunnel and see full list of output plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/src/test/resources/localfile_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/src/test/resources/localfile_source_to_sink.conf index 321d38a1ef70..d82d41e1a706 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/src/test/resources/localfile_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/src/test/resources/localfile_source_to_sink.conf @@ -80,7 +80,7 @@ source { # } # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source transform { @@ -113,5 +113,5 @@ sink { # } # If you would like to get more information about how to configure seatunnel and see full list of output plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf index d9fbd8e9ea3b..ec36b2a5db01 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf @@ -57,7 +57,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -71,5 +71,5 @@ sink { parallel_scan_threads=4 } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/src/test/resources/amazonsqsIT_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/src/test/resources/amazonsqsIT_source_to_sink.conf index b49e820e8f1d..7558c33a4604 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/src/test/resources/amazonsqsIT_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/src/test/resources/amazonsqsIT_source_to_sink.conf @@ -41,7 +41,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-assert-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-assert-e2e/src/test/resources/assertion/fakesource_to_assert.conf index 9f6ddd1c91e3..195ca92dd775 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-assert-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-assert-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -109,5 +109,5 @@ sink { } } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Assert + # please go to https://seatunnel.apache.org/docs/connectors/sink/Assert } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cassandra-e2e/src/test/resources/cassandra_to_cassandra.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cassandra-e2e/src/test/resources/cassandra_to_cassandra.conf index 104b12d43f09..3dbb1599e2e6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cassandra-e2e/src/test/resources/cassandra_to_cassandra.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cassandra-e2e/src/test/resources/cassandra_to_cassandra.conf @@ -35,7 +35,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_clickhouse.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_clickhouse.conf index a45a2f1bcc8d..874d7b1a217b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_clickhouse.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_clickhouse.conf @@ -35,7 +35,7 @@ source { plugin_output = "source_table" } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Clickhouse } sink { @@ -48,5 +48,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_console.conf index 48b83a2ba971..383917ca7310 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_console.conf @@ -34,12 +34,12 @@ source { plugin_output = "source_table" } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Clickhouse } sink { console { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_create_schema_when_comment.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_create_schema_when_comment.conf index 419515e43289..b68b3e95928b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_create_schema_when_comment.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_create_schema_when_comment.conf @@ -35,7 +35,7 @@ source { plugin_output = "source_table" } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Clickhouse } sink { @@ -58,5 +58,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_join_complex_sql.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_join_complex_sql.conf index 75770f241660..3425773cf3df 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_join_complex_sql.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_join_complex_sql.conf @@ -32,7 +32,7 @@ source { sql = "select d1.* from default.source_table d1 join default.source_merge_tree_table d2 on d1.id = d2.id" } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Clickhouse } sink { @@ -45,5 +45,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_multi_table_source.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_multi_table_source.conf index 3b37ff6f7074..932815aeffb0 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_multi_table_source.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_multi_table_source.conf @@ -52,5 +52,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_add_filter_query.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_add_filter_query.conf index 4b8e150e5ab7..1941342aedbd 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_add_filter_query.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_add_filter_query.conf @@ -34,7 +34,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Clickhouse } sink { @@ -47,5 +47,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_add_partition_list.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_add_partition_list.conf index 762b147bd858..f3dedc0c32eb 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_add_partition_list.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_add_partition_list.conf @@ -34,7 +34,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Clickhouse } sink { @@ -47,5 +47,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_read.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_read.conf index 9baa26b31a2e..7e8d56c884ba 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_read.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_parallelism_read.conf @@ -33,7 +33,7 @@ source { batch_size = 10 } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Clickhouse } sink { @@ -46,5 +46,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_sql_and_filter_query.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_sql_and_filter_query.conf index ad323c935f90..6428e896d13d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_sql_and_filter_query.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_sql_and_filter_query.conf @@ -35,7 +35,7 @@ source { batch_size = 10 } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Clickhouse } sink { @@ -48,6 +48,6 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-google-firestore-e2e/src/test/resources/firestore/fake_to_google_firestore.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-google-firestore-e2e/src/test/resources/firestore/fake_to_google_firestore.conf index 5149ee9f33e4..493087bf0df8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-google-firestore-e2e/src/test/resources/firestore/fake_to_google_firestore.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-google-firestore-e2e/src/test/resources/firestore/fake_to_google_firestore.conf @@ -61,5 +61,5 @@ sink { credentials = "dummy-credentials" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_db2_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_db2_source_and_sink.conf index 15bf0dbcd081..1cdd909aaf50 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_db2_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_db2_source_and_sink.conf @@ -33,7 +33,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/source/Jdbc } sink { @@ -49,5 +49,5 @@ insert into "E2E".SINK (C_BOOLEAN, C_SMALLINT, C_INT, C_INTEGER, C_BIGINT, C_DEC } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_db2_source_and_sink_upsert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_db2_source_and_sink_upsert.conf index 94d401512392..a55e6ee1df62 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_db2_source_and_sink_upsert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_db2_source_and_sink_upsert.conf @@ -33,7 +33,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/source/Jdbc } sink { @@ -53,5 +53,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink.conf index 4cfb234e5862..7f5c7319a0e7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink.conf @@ -37,7 +37,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { @@ -53,5 +53,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select1.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select1.conf index 468d4b495372..d3edd6370879 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select1.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select1.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { @@ -54,5 +54,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select2.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select2.conf index 1654660e6693..2ad11c534eb5 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select2.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select2.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { @@ -54,5 +54,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select3.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select3.conf index ea6c48cb2ad8..d751394109c6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select3.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_use_select3.conf @@ -39,7 +39,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { @@ -55,5 +55,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_with_blob_as_string.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_with_blob_as_string.conf index 9a79d9f2ef1d..6a5f0bdabf6b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_with_blob_as_string.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_to_sink_with_blob_as_string.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { @@ -100,5 +100,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_with_multiple_tables_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_with_multiple_tables_to_sink.conf index 678b13829a08..a12fe29e65e2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_with_multiple_tables_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_with_multiple_tables_to_sink.conf @@ -45,7 +45,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_with_pattern_tables_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_with_pattern_tables_to_sink.conf index 08d42b093e6d..49c068ad9871 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_with_pattern_tables_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_oracle_source_with_pattern_tables_to_sink.conf @@ -43,7 +43,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf index c923090c2a2e..85706052b62a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_oceanbase_mysql_source_and_sink.conf @@ -35,7 +35,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { @@ -48,5 +48,5 @@ sink { compatible_mode = "mysql" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_phoenix_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_phoenix_source_and_sink.conf index 61e64852375e..5f79dd87df70 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_phoenix_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_phoenix_source_and_sink.conf @@ -32,14 +32,14 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -50,5 +50,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_teradata_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_teradata_source_and_sink.conf index cd6e6232df88..f35245470c08 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_teradata_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_teradata_source_and_sink.conf @@ -45,7 +45,7 @@ source { """ } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/source/Jdbc } sink { @@ -73,5 +73,5 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_sqlserver_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_sqlserver_source_to_sink.conf index a448875371f3..351928901230 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_sqlserver_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_sqlserver_source_to_sink.conf @@ -34,13 +34,13 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/source/Jdbc } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -55,5 +55,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_cloudberry_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_cloudberry_source_and_sink.conf index ceb45b21f905..bfe59cd3754d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_cloudberry_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_cloudberry_source_and_sink.conf @@ -34,13 +34,13 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/source/Jdbc } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -53,5 +53,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_gbase8a_source_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_gbase8a_source_to_assert.conf index 64fd2ebfa6a5..afddf122279e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_gbase8a_source_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_gbase8a_source_to_assert.conf @@ -34,7 +34,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + # please go to https://seatunnel.apache.org/docs/connectors/source/FakeSource } sink { @@ -54,5 +54,5 @@ sink { } } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Assert + # please go to https://seatunnel.apache.org/docs/connectors/sink/Assert } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_greenplum_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_greenplum_source_and_sink.conf index 87346d7ec5e6..bef640acf592 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_greenplum_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/src/test/resources/jdbc_greenplum_source_and_sink.conf @@ -36,13 +36,13 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/source/Jdbc } transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform-v2/sql + # please go to https://seatunnel.apache.org/docs/transforms/sql } sink { @@ -55,5 +55,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # please go to https://seatunnel.apache.org/docs/connectors/sink/Jdbc } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_earliest_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_earliest_to_console.conf index bd9fb2b3e424..634b3737d0e6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_earliest_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_earliest_to_console.conf @@ -41,7 +41,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_endTimestamp_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_endTimestamp_to_console.conf index 91646ea35351..6a4129e04db9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_endTimestamp_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_endTimestamp_to_console.conf @@ -41,7 +41,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_format_error_handle_way_fail_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_format_error_handle_way_fail_to_console.conf index f5c3192eaf7e..47172e12abd7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_format_error_handle_way_fail_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_format_error_handle_way_fail_to_console.conf @@ -60,7 +60,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_format_error_handle_way_skip_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_format_error_handle_way_skip_to_console.conf index fc55de5733ec..d9cdcabd346d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_format_error_handle_way_skip_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_format_error_handle_way_skip_to_console.conf @@ -60,7 +60,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_group_offset_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_group_offset_to_console.conf index 653838c0f8eb..c4a494c34f40 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_group_offset_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_group_offset_to_console.conf @@ -42,7 +42,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_group_offset_to_console_with_commit_offset.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_group_offset_to_console_with_commit_offset.conf index c6c0491f2029..6647fe4d3b81 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_group_offset_to_console_with_commit_offset.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_group_offset_to_console_with_commit_offset.conf @@ -44,7 +44,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_latest_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_latest_to_console.conf index c4f61d506f78..434c9ffb9b25 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_latest_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_latest_to_console.conf @@ -39,7 +39,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_specific_offsets_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_specific_offsets_to_console.conf index afbcd71436bb..27079f9ca26c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_specific_offsets_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_specific_offsets_to_console.conf @@ -43,7 +43,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_timestamp_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_timestamp_to_console.conf index 126a7e2a89f6..8b48828563ee 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_timestamp_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_timestamp_to_console.conf @@ -40,7 +40,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_timestamp_to_console_skip_partition.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_timestamp_to_console_skip_partition.conf index 1ed3533df5b5..246e26a20d33 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_timestamp_to_console_skip_partition.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafkasource_timestamp_to_console_skip_partition.conf @@ -41,7 +41,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/KafkaSource + # please go to https://seatunnel.apache.org/docs/connectors/source/Kafka } transform { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/resources/tdengine/tdengine_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/resources/tdengine/tdengine_source_to_sink.conf index 16d802d65327..e7f017fd928b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/resources/tdengine/tdengine_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/resources/tdengine/tdengine_source_to_sink.conf @@ -36,7 +36,7 @@ source { plugin_output = "tdengine_result" } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -52,5 +52,5 @@ sink { timezone: "UTC" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/fake_to_console.variables.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/fake_to_console.variables.conf index 0ee933326fc7..22be6a424a81 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/fake_to_console.variables.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/fake_to_console.variables.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -58,5 +58,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/resources/stream_fake_to_console_biginterval.conf b/seatunnel-engine/seatunnel-engine-server/src/test/resources/stream_fake_to_console_biginterval.conf index f58e78e9fdad..d105f1f7a0bd 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/resources/stream_fake_to_console_biginterval.conf +++ b/seatunnel-engine/seatunnel-engine-server/src/test/resources/stream_fake_to_console_biginterval.conf @@ -40,7 +40,7 @@ source { } # If you would like to get more information about how to configure SeaTunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } sink { @@ -48,5 +48,5 @@ sink { } # If you would like to get more information about how to configure SeaTunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/resources/stream_fake_to_console_with_checkpoint.conf b/seatunnel-engine/seatunnel-engine-server/src/test/resources/stream_fake_to_console_with_checkpoint.conf index 6833a43eeedc..dea5f23eea3c 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/resources/stream_fake_to_console_with_checkpoint.conf +++ b/seatunnel-engine/seatunnel-engine-server/src/test/resources/stream_fake_to_console_with_checkpoint.conf @@ -40,7 +40,7 @@ source { } # If you would like to get more information about how to configure SeaTunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } sink { @@ -48,5 +48,5 @@ sink { } # If you would like to get more information about how to configure SeaTunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } diff --git a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-13-example/src/main/resources/examples/fake_to_console_batch.conf b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-13-example/src/main/resources/examples/fake_to_console_batch.conf index 9656e54b65b2..18336e1a2d7f 100644 --- a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-13-example/src/main/resources/examples/fake_to_console_batch.conf +++ b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-13-example/src/main/resources/examples/fake_to_console_batch.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -58,5 +58,5 @@ sink { plugin_input = "fake1" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-13-example/src/main/resources/examples/fake_to_console_streaming.conf b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-13-example/src/main/resources/examples/fake_to_console_streaming.conf index 09ddccdaee2d..94ce5a6f2469 100644 --- a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-13-example/src/main/resources/examples/fake_to_console_streaming.conf +++ b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-13-example/src/main/resources/examples/fake_to_console_streaming.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -58,5 +58,5 @@ sink { plugin_input = "fake1" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-15-example/src/main/resources/examples/fake_to_console_batch.conf b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-15-example/src/main/resources/examples/fake_to_console_batch.conf index 9656e54b65b2..18336e1a2d7f 100644 --- a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-15-example/src/main/resources/examples/fake_to_console_batch.conf +++ b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-15-example/src/main/resources/examples/fake_to_console_batch.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -58,5 +58,5 @@ sink { plugin_input = "fake1" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-15-example/src/main/resources/examples/fake_to_console_streaming.conf b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-15-example/src/main/resources/examples/fake_to_console_streaming.conf index 09ddccdaee2d..94ce5a6f2469 100644 --- a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-15-example/src/main/resources/examples/fake_to_console_streaming.conf +++ b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-15-example/src/main/resources/examples/fake_to_console_streaming.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -58,5 +58,5 @@ sink { plugin_input = "fake1" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-20-example/src/main/resources/examples/fake_to_console_batch.conf b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-20-example/src/main/resources/examples/fake_to_console_batch.conf index 9656e54b65b2..18336e1a2d7f 100644 --- a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-20-example/src/main/resources/examples/fake_to_console_batch.conf +++ b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-20-example/src/main/resources/examples/fake_to_console_batch.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -58,5 +58,5 @@ sink { plugin_input = "fake1" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-20-example/src/main/resources/examples/fake_to_console_streaming.conf b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-20-example/src/main/resources/examples/fake_to_console_streaming.conf index 09ddccdaee2d..94ce5a6f2469 100644 --- a/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-20-example/src/main/resources/examples/fake_to_console_streaming.conf +++ b/seatunnel-examples/seatunnel-flink-examples/seatunnel-flink-20-example/src/main/resources/examples/fake_to_console_streaming.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -58,5 +58,5 @@ sink { plugin_input = "fake1" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf index e8b18a5ca397..bddee963268d 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf @@ -65,7 +65,7 @@ source { # } # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source + # please go to https://seatunnel.apache.org/docs/connectors/source } transform { @@ -95,5 +95,5 @@ sink { # } # If you would like to get more information about how to configure seatunnel and see full list of output plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink + # please go to https://seatunnel.apache.org/docs/connectors/sink } From ba53e82babde02fc2d8c411ae0a94d71a3e9648f Mon Sep 17 00:00:00 2001 From: shown Date: Sat, 13 Jun 2026 13:44:16 +0800 Subject: [PATCH 009/375] [Fix][Connector-V2] Ignore BOM in file source readers (#11056) Signed-off-by: yuluo-yx --- .../source/reader/AbstractReadStrategy.java | 21 +++++ .../file/source/reader/CsvReadStrategy.java | 15 +--- .../file/source/reader/JsonReadStrategy.java | 6 +- .../file/source/reader/TextReadStrategy.java | 6 +- .../file/source/reader/XmlReadStrategy.java | 6 +- .../reader/AbstractReadStrategyTest.java | 80 +++++++++++++++++++ 6 files changed, 111 insertions(+), 23 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java index a685a69feccf..918e4641b54a 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java @@ -47,6 +47,8 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipParameters; +import org.apache.commons.io.ByteOrderMark; +import org.apache.commons.io.input.BOMInputStream; import org.apache.commons.io.input.BoundedInputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; @@ -55,12 +57,15 @@ import lombok.extern.slf4j.Slf4j; +import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.math.BigDecimal; +import java.nio.charset.Charset; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.LocalDate; @@ -602,6 +607,22 @@ protected static InputStream safeSlice(InputStream in, long start, long length) return new BoundedInputStream(in, length); } + protected static BufferedReader createBomAwareBufferedReader( + InputStream inputStream, String encoding) throws IOException { + BOMInputStream bomInputStream = + new BOMInputStream( + inputStream, + ByteOrderMark.UTF_8, + ByteOrderMark.UTF_16BE, + ByteOrderMark.UTF_16LE, + ByteOrderMark.UTF_32BE, + ByteOrderMark.UTF_32LE); + ByteOrderMark bom = bomInputStream.getBOM(); + Charset charset = + bom == null ? Charset.forName(encoding) : Charset.forName(bom.getCharsetName()); + return new BufferedReader(new InputStreamReader(bomInputStream, charset)); + } + @Override public void close() throws IOException { try { diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java index a049d9fa5544..7ac420a7cce9 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java @@ -45,7 +45,6 @@ import org.apache.commons.csv.CSVFormat.Builder; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; -import org.apache.commons.io.input.BOMInputStream; import io.airlift.compress.lzo.LzopCodec; import lombok.extern.slf4j.Slf4j; @@ -53,8 +52,6 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; @@ -107,9 +104,9 @@ public void readProcess( split.getStart(), split.getLength()); final boolean useSplitRead = isSplitReadEnabled(split); - try (BOMInputStream bomIn = new BOMInputStream(wrapInputStream(inputStream, split)); - BufferedReader reader = - new BufferedReader(new InputStreamReader(bomIn, getCharset(bomIn))); + try (BufferedReader reader = + createBomAwareBufferedReader( + wrapInputStream(inputStream, split), encoding); CSVParser csvParser = new CSVParser(reader, getCSVFormat(split))) { // skip lines // if split range is used, no need to skip @@ -204,12 +201,6 @@ private InputStream wrapInputStream(InputStream inputStream, FileSourceSplit spl return resultStream; } - private Charset getCharset(BOMInputStream bomIn) throws IOException { - return bomIn.getBOM() == null - ? Charset.forName(encoding) - : Charset.forName(bomIn.getBOM().getCharsetName()); - } - private boolean isSplitReadEnabled(FileSourceSplit split) { return enableSplitFile && split.getLength() > -1; } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/JsonReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/JsonReadStrategy.java index 98b8de3ca2a8..a3bca9c5b13a 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/JsonReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/JsonReadStrategy.java @@ -39,7 +39,6 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -116,10 +115,9 @@ public void readProcess( } // rebuild inputStream if (enableSplitFile && split.getLength() > -1) { - actualInputStream = safeSlice(inputStream, split.getStart(), split.getLength()); + actualInputStream = safeSlice(actualInputStream, split.getStart(), split.getLength()); } - try (BufferedReader reader = - new BufferedReader(new InputStreamReader(actualInputStream, encoding))) { + try (BufferedReader reader = createBomAwareBufferedReader(actualInputStream, encoding)) { reader.lines() .forEach( line -> { diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java index fa45521fbbe3..4c6f07836245 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java @@ -46,7 +46,6 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Optional; @@ -207,10 +206,9 @@ public void readProcess( // rebuild inputStream final boolean useSplitRead = enableSplitFile && split.getLength() > -1; if (useSplitRead) { - actualInputStream = safeSlice(inputStream, split.getStart(), split.getLength()); + actualInputStream = safeSlice(actualInputStream, split.getStart(), split.getLength()); } - try (BufferedReader reader = - new BufferedReader(new InputStreamReader(actualInputStream, encoding))) { + try (BufferedReader reader = createBomAwareBufferedReader(actualInputStream, encoding)) { LineProcessor lineProcessor = line -> { diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/XmlReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/XmlReadStrategy.java index 58002f0f5498..fcf7daf2ddae 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/XmlReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/XmlReadStrategy.java @@ -52,9 +52,9 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -106,8 +106,8 @@ public void readProcess( throws IOException { SAXReader saxReader = new SAXReader(); Document document; - try { - document = saxReader.read(new InputStreamReader(inputStream, encoding)); + try (BufferedReader reader = createBomAwareBufferedReader(inputStream, encoding)) { + document = saxReader.read(reader); } catch (DocumentException e) { throw new FileConnectorException( FileConnectorErrorCode.FILE_READ_FAILED, diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java index 626a33bbd782..69481f30d58d 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java @@ -26,6 +26,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.file.source.split.FileSourceSplit; import org.apache.seatunnel.connectors.seatunnel.file.util.LocalFileSystemConf; import org.apache.avro.Schema; @@ -46,6 +47,7 @@ import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -381,6 +383,84 @@ public void testGetSeaTunnelRowTypeInfoShouldNotThrowWhenFileListIsEmpty() throw } } + @Test + void testTextReadStrategyShouldSkipUtf8Bom() throws Exception { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"name"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + CatalogTable catalogTable = CatalogTableUtil.getCatalogTable("test", rowType); + TempCollector collector = new TempCollector(); + + try (TextReadStrategy textReadStrategy = new TextReadStrategy()) { + textReadStrategy.setPluginConfig(ConfigFactory.empty()); + textReadStrategy.setCatalogTable(catalogTable); + textReadStrategy.readProcess( + new FileSourceSplit("test", "/tmp/bom.txt"), + collector, + new ByteArrayInputStream( + ("\uFEFF" + "alice\n").getBytes(StandardCharsets.UTF_8)), + new HashMap<>(), + "bom.txt"); + } + + Assertions.assertEquals(1, collector.getRows().size()); + Assertions.assertEquals("alice", collector.getRows().get(0).getField(0)); + } + + @Test + void testJsonReadStrategyShouldSkipUtf8Bom() throws Exception { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"name"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + CatalogTable catalogTable = CatalogTableUtil.getCatalogTable("test", rowType); + TempCollector collector = new TempCollector(); + + try (JsonReadStrategy jsonReadStrategy = new JsonReadStrategy()) { + jsonReadStrategy.setPluginConfig(ConfigFactory.empty()); + jsonReadStrategy.init(new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT)); + jsonReadStrategy.setCatalogTable(catalogTable); + jsonReadStrategy.readProcess( + new FileSourceSplit("test", "/tmp/bom.json"), + collector, + new ByteArrayInputStream( + ("\uFEFF" + "{\"name\":\"alice\"}\n").getBytes(StandardCharsets.UTF_8)), + new HashMap<>(), + "bom.json"); + } + + Assertions.assertEquals(1, collector.getRows().size()); + Assertions.assertEquals("alice", collector.getRows().get(0).getField(0)); + } + + @Test + void testXmlReadStrategyShouldSkipUtf8Bom() throws Exception { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"name"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + CatalogTable catalogTable = CatalogTableUtil.getCatalogTable("test", rowType); + Map pluginConfig = new HashMap<>(); + pluginConfig.put(FileBaseSourceOptions.XML_ROW_TAG.key(), "row"); + pluginConfig.put(FileBaseSourceOptions.XML_USE_ATTR_FORMAT.key(), false); + TempCollector collector = new TempCollector(); + + try (XmlReadStrategy xmlReadStrategy = new XmlReadStrategy()) { + xmlReadStrategy.setPluginConfig(ConfigFactory.parseMap(pluginConfig)); + xmlReadStrategy.init(new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT)); + xmlReadStrategy.setCatalogTable(catalogTable); + xmlReadStrategy.readProcess( + new FileSourceSplit("test", "/tmp/bom.xml"), + collector, + new ByteArrayInputStream( + ("\uFEFF" + "alice") + .getBytes(StandardCharsets.UTF_8)), + new HashMap<>(), + "bom.xml"); + } + + Assertions.assertEquals(1, collector.getRows().size()); + Assertions.assertEquals("alice", collector.getRows().get(0).getField(0)); + } + @Test void testResolveRelativePathWithSftpUri() { String basePath = "sftp://server:22/path"; From f35a56d0ca02a04a0f75ca66c3be730a3ae9de91 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Sat, 13 Jun 2026 17:27:04 +0800 Subject: [PATCH 010/375] [Improve][Engine] fix engine UT flaky test (#11000) --- .../seatunnel/api/tracing/MDCTracerTest.java | 247 +++++++++--------- .../command/ServerExecuteCommandTest.java | 61 +++-- .../SeaTunnelEngineClusterRoleTest.java | 45 ++-- .../engine/server/CoordinatorServiceTest.java | 141 +++++----- ...inatorServiceWithCancelPendingJobTest.java | 16 +- .../checkpoint/CheckpointStorageTest.java | 21 +- .../checkpoint/CheckpointTimeOutTest.java | 2 +- .../master/FollowerRunningJobsFilterTest.java | 4 +- .../server/master/JobHistoryServiceTest.java | 3 +- .../engine/server/rest/BaseServletTest.java | 3 +- .../server/rest/RestApiHttpBasicTest.java | 4 +- .../rest/RestApiHttpsForTruststoreTest.java | 3 +- .../engine/server/rest/RestApiHttpsTest.java | 13 +- .../EngineStateStoreMetricExportsTest.java | 4 +- 14 files changed, 322 insertions(+), 245 deletions(-) diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/tracing/MDCTracerTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/tracing/MDCTracerTest.java index 694bac8293d0..63c81548653b 100644 --- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/tracing/MDCTracerTest.java +++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/tracing/MDCTracerTest.java @@ -23,7 +23,9 @@ import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; @@ -127,129 +129,138 @@ public Object get() { public void testMDCTracedExecutorService() throws Exception { MDCContext mdcContext = MDCContext.of(1, 2, 3); - MDCExecutorService tracedExecutorService = - MDCTracer.tracing(mdcContext, Executors.newSingleThreadExecutor()); - - Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); - Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); - tracedExecutorService - .submit( - new Runnable() { - @Override - public void run() { - Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); - Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); - } - }) - .get(); - Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); - Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); - - tracedExecutorService - .submit( - new Callable() { - @Override - public Void call() throws Exception { - Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); - Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); - return null; - } - }) - .get(); - Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); - Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); - - MDCScheduledExecutorService tracedScheduledExecutorService = - MDCTracer.tracing(mdcContext, Executors.newSingleThreadScheduledExecutor()); - Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); - Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + ExecutorService rawExecutor = Executors.newSingleThreadExecutor(); + ScheduledExecutorService rawScheduledExecutor = + Executors.newSingleThreadScheduledExecutor(); + try { + MDCExecutorService tracedExecutorService = MDCTracer.tracing(mdcContext, rawExecutor); + + Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); + Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + tracedExecutorService + .submit( + new Runnable() { + @Override + public void run() { + Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); + Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); + } + }) + .get(); + Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); + Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + + tracedExecutorService + .submit( + new Callable() { + @Override + public Void call() throws Exception { + Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); + Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); + return null; + } + }) + .get(); + Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); + Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + + MDCScheduledExecutorService tracedScheduledExecutorService = + MDCTracer.tracing(mdcContext, rawScheduledExecutor); + Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); + Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + + tracedScheduledExecutorService + .schedule( + new Runnable() { + @Override + public void run() { + Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); + Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); + } + }, + 1, + TimeUnit.SECONDS) + .get(); + Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); + Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + + tracedScheduledExecutorService + .schedule( + new Callable() { + @Override + public Object call() { + Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); + Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); + return null; + } + }, + 1, + TimeUnit.SECONDS) + .get(); + Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); + Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); - tracedScheduledExecutorService - .schedule( - new Runnable() { - @Override - public void run() { - Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); - Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); - } - }, - 1, - TimeUnit.SECONDS) - .get(); - Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); - Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + CompletableFuture futureWithScheduleAtFixedRate = new CompletableFuture<>(); + tracedScheduledExecutorService.scheduleAtFixedRate( + new Runnable() { + AtomicInteger executeCount = new AtomicInteger(0); - tracedScheduledExecutorService - .schedule( - new Callable() { - @Override - public Object call() { - Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); - Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); - return null; + @Override + public void run() { + Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); + Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); + executeCount.incrementAndGet(); + if (executeCount.get() > 10 + && !futureWithScheduleAtFixedRate.isDone()) { + futureWithScheduleAtFixedRate.complete(true); } - }, - 1, - TimeUnit.SECONDS) - .get(); - Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); - Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); - - CompletableFuture futureWithScheduleAtFixedRate = new CompletableFuture<>(); - tracedScheduledExecutorService.scheduleAtFixedRate( - new Runnable() { - AtomicInteger executeCount = new AtomicInteger(0); - - @Override - public void run() { - Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); - Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); - executeCount.incrementAndGet(); - if (executeCount.get() > 10 && !futureWithScheduleAtFixedRate.isDone()) { - futureWithScheduleAtFixedRate.complete(true); } - } - }, - 0, - 10, - TimeUnit.MILLISECONDS); - futureWithScheduleAtFixedRate.join(); - - CompletableFuture futureWithScheduleAtFixedDelay = new CompletableFuture<>(); - tracedScheduledExecutorService.scheduleWithFixedDelay( - new Runnable() { - AtomicInteger executeCount = new AtomicInteger(0); - - @Override - public void run() { - Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); - Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); - executeCount.incrementAndGet(); - if (executeCount.get() > 10 && !futureWithScheduleAtFixedDelay.isDone()) { - futureWithScheduleAtFixedDelay.complete(true); + }, + 0, + 10, + TimeUnit.MILLISECONDS); + futureWithScheduleAtFixedRate.get(30, TimeUnit.SECONDS); + + CompletableFuture futureWithScheduleAtFixedDelay = new CompletableFuture<>(); + tracedScheduledExecutorService.scheduleWithFixedDelay( + new Runnable() { + AtomicInteger executeCount = new AtomicInteger(0); + + @Override + public void run() { + Assertions.assertEquals("1", MDC.get(MDCContext.JOB_ID)); + Assertions.assertEquals("2", MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertEquals("3", MDC.get(MDCContext.TASK_ID)); + executeCount.incrementAndGet(); + if (executeCount.get() > 10 + && !futureWithScheduleAtFixedDelay.isDone()) { + futureWithScheduleAtFixedDelay.complete(true); + } } - } - }, - 0, - 10, - TimeUnit.MILLISECONDS); - futureWithScheduleAtFixedDelay.join(); - - Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); - Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); - Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + }, + 0, + 10, + TimeUnit.MILLISECONDS); + futureWithScheduleAtFixedDelay.get(30, TimeUnit.SECONDS); + + Assertions.assertNull(MDC.get(MDCContext.JOB_ID)); + Assertions.assertNull(MDC.get(MDCContext.PIPELINE_ID)); + Assertions.assertNull(MDC.get(MDCContext.TASK_ID)); + } finally { + rawExecutor.shutdownNow(); + rawScheduledExecutor.shutdownNow(); + } } @Test diff --git a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommandTest.java b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommandTest.java index 16f76aef7496..78b7d06b7f9c 100644 --- a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommandTest.java +++ b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommandTest.java @@ -28,7 +28,10 @@ import org.junit.jupiter.api.condition.JRE; import com.hazelcast.cluster.Member; +import com.hazelcast.instance.impl.HazelcastInstanceImpl; +import java.util.ArrayList; +import java.util.List; import java.util.Set; public class ServerExecuteCommandTest { @@ -37,33 +40,57 @@ public class ServerExecuteCommandTest { @DisabledOnJre(value = JRE.JAVA_11, disabledReason = "the test case only works on Java 8") public void testJavaVersionCheck() { String realVersion = System.getProperty("java.version"); - System.setProperty("java.version", "1.8.0_191"); - Assertions.assertFalse(ServerExecuteCommand.isAllocatingThreadGetName()); - System.setProperty("java.version", "1.8.0_60"); - Assertions.assertTrue(ServerExecuteCommand.isAllocatingThreadGetName()); - System.setProperty("java.version", realVersion); + try { + System.setProperty("java.version", "1.8.0_191"); + Assertions.assertFalse(ServerExecuteCommand.isAllocatingThreadGetName()); + System.setProperty("java.version", "1.8.0_60"); + Assertions.assertTrue(ServerExecuteCommand.isAllocatingThreadGetName()); + } finally { + System.setProperty("java.version", realVersion); + } } @Test - public void testMemberList() { + public void testMemberList() throws InterruptedException { String clusterName = getClusterName("ServerExecuteCommandTest"); SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); seaTunnelConfig.getHazelcastConfig().setClusterName(clusterName); seaTunnelConfig.getEngineConfig().getHttpConfig().setEnableDynamicPort(true); - SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig); - SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig); - SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig); - SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig); - SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig); + List instances = new ArrayList<>(); + try { + instances.add(SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig)); + instances.add(SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig)); + instances.add(SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig)); + instances.add(SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig)); + instances.add(SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig)); - ServerCommandArgs serverCommandArgs = new ServerCommandArgs(); - serverCommandArgs.setClusterName(clusterName); - serverCommandArgs.setShowClusterMembers(true); + HazelcastInstanceImpl firstInstance = instances.get(0); + long deadline = System.currentTimeMillis() + 30_000; + while (firstInstance.getCluster().getMembers().size() < 5) { + if (System.currentTimeMillis() > deadline) { + Assertions.fail( + "Cluster did not form within 30s, members: " + + firstInstance.getCluster().getMembers().size()); + } + Thread.sleep(500); + } - ServerExecuteCommand serverExecuteCommand = new ServerExecuteCommand(serverCommandArgs); - Set members = serverExecuteCommand.showClusterMembers(); - Assertions.assertEquals(5, members.size()); + ServerCommandArgs serverCommandArgs = new ServerCommandArgs(); + serverCommandArgs.setClusterName(clusterName); + serverCommandArgs.setShowClusterMembers(true); + + ServerExecuteCommand serverExecuteCommand = new ServerExecuteCommand(serverCommandArgs); + Set members = serverExecuteCommand.showClusterMembers(); + Assertions.assertEquals(5, members.size()); + } finally { + for (HazelcastInstanceImpl inst : instances) { + try { + inst.shutdown(); + } catch (Exception ignored) { + } + } + } } public static String getClusterName(String testClassName) { diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java b/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java index 0dee6aac4c18..4da834720602 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java +++ b/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java @@ -80,7 +80,7 @@ public void testClusterWillDownWhenNoMasterNode() { masterNode = SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig); HazelcastInstanceImpl finalMasterNode = masterNode; Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -91,7 +91,7 @@ public void testClusterWillDownWhenNoMasterNode() { HazelcastInstanceImpl finalWorkerNode = workerNode1; Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -149,7 +149,7 @@ public void canNotSubmitJobWhenHaveNoWorkerNode() { HazelcastInstanceImpl finalMasterNode = masterNode; Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -189,6 +189,8 @@ public void canNotSubmitJobWhenHaveNoWorkerNode() { @Test public void enterPendingWhenResourcesNotEnough() { HazelcastInstanceImpl masterNode = null; + HazelcastInstanceImpl workerNode1 = null; + HazelcastInstanceImpl workerNode2 = null; String testClusterName = "Test_enterPendingWhenResourcesNotEnough"; SeaTunnelClient seaTunnelClient = null; @@ -214,7 +216,7 @@ public void enterPendingWhenResourcesNotEnough() { HazelcastInstanceImpl finalMasterNode = masterNode; Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -226,7 +228,7 @@ public void enterPendingWhenResourcesNotEnough() { seaTunnelClient.createExecutionContext(filePath, jobConfig, seaTunnelConfig); final ClientJobProxy clientJobProxy = jobExecutionEnv.execute(); Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -235,8 +237,8 @@ public void enterPendingWhenResourcesNotEnough() { status.contains("PENDING"); // start two worker nodes - SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig); - SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig); + workerNode1 = SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig); + workerNode2 = SeaTunnelServerStarter.createWorkerHazelcastInstance(seaTunnelConfig); // There are already resources available, wait for job enter running or complete Awaitility.await() @@ -251,6 +253,12 @@ public void enterPendingWhenResourcesNotEnough() { if (seaTunnelClient != null) { seaTunnelClient.close(); } + if (workerNode1 != null) { + workerNode1.shutdown(); + } + if (workerNode2 != null) { + workerNode2.shutdown(); + } if (masterNode != null) { masterNode.shutdown(); } @@ -291,7 +299,7 @@ public void pendingJobCancel() { seaTunnelClient.createExecutionContext(filePath, jobConfig, seaTunnelConfig); final ClientJobProxy clientJobProxy = jobExecutionEnv.execute(); Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -382,7 +390,7 @@ public void testWorkerIsFirstMemberThenGetJobDetailStatus() { masterNode1 = SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig); HazelcastInstanceImpl finalMasterNode1 = masterNode1; Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -398,14 +406,14 @@ public void testWorkerIsFirstMemberThenGetJobDetailStatus() { masterNode2 = SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig2); HazelcastInstanceImpl finalWorkerNode = workerNode1; Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( 4, finalWorkerNode.getCluster().getMembers().size())); masterNode1.shutdown(); Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -424,7 +432,7 @@ public void testWorkerIsFirstMemberThenGetJobDetailStatus() { .client; HazelcastClientInstanceImpl finalHazelcastClient = hazelcastClient; Awaitility.await() - .atMost(10000, TimeUnit.MILLISECONDS) + .atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { UUID masterUuid = @@ -454,12 +462,15 @@ public void testWorkerIsFirstMemberThenGetJobDetailStatus() { .listJobStatus(true) .contains("RUNNING"))); jobClient.cancelJob(jobId); - await().pollDelay(10000, TimeUnit.MILLISECONDS) - .atMost(60000, TimeUnit.MILLISECONDS) + await().atMost(120000, TimeUnit.MILLISECONDS) .untilAsserted( - () -> - Assertions.assertEquals( - "CANCELED", jobClient.getJobStatus(jobId))); + () -> { + String status = jobClient.getJobStatus(jobId); + Assertions.assertEquals( + "CANCELED", + status, + "Expected terminal state but was: " + status); + }); } finally { if (hazelcastClient != null) { hazelcastClient.shutdown(); diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceTest.java index 18a83020e959..a2ab94924875 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceTest.java @@ -109,7 +109,7 @@ void testTerminalZombieJobShouldNotRestartAfterMasterSwitch() { SeaTunnelServer server1 = instance1.node.getNodeEngine().getService(SeaTunnelServer.SERVICE_NAME); - await().atMost(10, TimeUnit.SECONDS) + await().atMost(60, TimeUnit.SECONDS) .untilAsserted( () -> { Assertions.assertTrue(server1.isMasterNode()); @@ -123,7 +123,7 @@ void testTerminalZombieJobShouldNotRestartAfterMasterSwitch() { SeaTunnelServer server2 = instance2.node.getNodeEngine().getService(SeaTunnelServer.SERVICE_NAME); - await().atMost(10, TimeUnit.SECONDS) + await().atMost(60, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -345,10 +345,10 @@ void testCheckNewActiveMasterCanSchedulePendingQueue() throws Exception { masterFlag.set(true); invokeCheckNewActiveMaster(coordinatorService); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertTrue(coordinatorService.isCoordinatorActive())); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> Assertions.assertEquals(0L, runLatch.getCount())); Mockito.verify(jobMaster, Mockito.times(1)).run(); } finally { @@ -396,10 +396,10 @@ void testFailoverStopsOldPendingQueueAndNewCoordinatorCanSchedule() throws Excep invokeCheckNewActiveMaster(newCoordinator); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertTrue(newCoordinator.isCoordinatorActive())); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> Assertions.assertEquals(0L, newRunLatch.getCount())); Mockito.verify(newPendingJob, Mockito.times(1)).run(); Assertions.assertFalse(newCoordinator.getPendingJobQueue().contains(30001L)); @@ -424,10 +424,10 @@ void testCheckNewActiveMasterIsIdempotentWhenAlreadyActive() throws Exception { JobMaster jobMaster = enqueueMockPendingJob(coordinatorService, 40001L, runLatch); invokeCheckNewActiveMaster(coordinatorService); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertTrue(coordinatorService.isCoordinatorActive())); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> Assertions.assertEquals(0L, runLatch.getCount())); invokeCheckNewActiveMaster(coordinatorService); @@ -454,7 +454,7 @@ void testPendingJobWithInsufficientResourceRespectsWaitStrategy() throws Excepti enqueueMockPendingJob(coordinatorService, 50001L, runLatch, false); invokeCheckNewActiveMaster(coordinatorService); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertTrue(coordinatorService.isCoordinatorActive())); await().during(1, TimeUnit.SECONDS) @@ -483,10 +483,10 @@ void testPendingJobWithInsufficientResourceRespectsRejectStrategy() throws Excep enqueueMockPendingJob(coordinatorService, 60001L, runLatch, false); invokeCheckNewActiveMaster(coordinatorService); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertTrue(coordinatorService.isCoordinatorActive())); - await().atMost(5, TimeUnit.SECONDS) + await().atMost(30, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertFalse( @@ -669,32 +669,33 @@ public void testInvocationFutureUseCompletableFutureExecutor() { SeaTunnelServerStarter.createHazelcastInstance( TestUtils.getClusterName( "CoordinatorServiceTest_testInvocationFutureUseCompletableFutureExecutor")); - - NodeEngineUtil.sendOperationToMemberNode( - instance.node.getNodeEngine(), - new PrintMessageOperation("hello"), - instance.getCluster().getLocalMember().getAddress()) - .whenComplete( - (aVoid, error) -> { - Assertions.assertTrue( - Thread.currentThread() - .getName() - .startsWith("SeaTunnel-CompletableFuture-Thread")); - }) - .join(); - - NodeEngineUtil.sendOperationToMasterNode( - instance.node.getNodeEngine(), new PrintMessageOperation("hello")) - .whenCompleteAsync( - (aVoid, error) -> { - Assertions.assertTrue( - Thread.currentThread() - .getName() - .startsWith("SeaTunnel-CompletableFuture-Thread")); - }) - .join(); - - instance.shutdown(); + try { + NodeEngineUtil.sendOperationToMemberNode( + instance.node.getNodeEngine(), + new PrintMessageOperation("hello"), + instance.getCluster().getLocalMember().getAddress()) + .whenComplete( + (aVoid, error) -> { + Assertions.assertTrue( + Thread.currentThread() + .getName() + .startsWith("SeaTunnel-CompletableFuture-Thread")); + }) + .join(); + + NodeEngineUtil.sendOperationToMasterNode( + instance.node.getNodeEngine(), new PrintMessageOperation("hello")) + .whenCompleteAsync( + (aVoid, error) -> { + Assertions.assertTrue( + Thread.currentThread() + .getName() + .startsWith("SeaTunnel-CompletableFuture-Thread")); + }) + .join(); + } finally { + instance.shutdown(); + } } private static final class BlockingEventProcessor implements EventProcessor { @@ -770,7 +771,7 @@ void testForceStopRunningJob() { "test_force_stop_running_job"); CoordinatorService coordinatorService = jobInformation.coordinatorService; - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { Assertions.assertEquals( @@ -806,7 +807,7 @@ void testForceStopAbnormalSavepointJob() { "test_force_stop_abnormal_savepoint_job"); CoordinatorService coordinatorService = jobInformation.coordinatorService; - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { Assertions.assertEquals( @@ -853,7 +854,7 @@ void testCleanupPendingJobMasterMapAfterJobFailed() { .getPendingJobQueue() .contains(jobInformation.jobId)); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertFalse( @@ -879,7 +880,7 @@ void testCleanupRunningJobStateIMap() { IMap runningJobStateIMap = coordinatorService.getJobMaster(jobInformation.jobId).getRunningJobStateIMap(); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { Assertions.assertEquals( @@ -894,7 +895,7 @@ void testCleanupRunningJobStateIMap() { .containsKey(jobInformation.jobId)); }); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { Assertions.assertEquals( @@ -921,9 +922,9 @@ void testCleanupMetricsImap() { CoordinatorService coordinatorService = jobInformation.coordinatorService; IMap> metricsImap = coordinatorService.getMetricsImap(); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted(() -> Assertions.assertFalse(metricsImap.isEmpty())); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted(() -> Assertions.assertTrue(metricsImap.isEmpty())); jobInformation.coordinatorService.clearCoordinatorService(); @@ -942,9 +943,9 @@ void testCleanupMetricsImapWithPartitionConfig() { CoordinatorService coordinatorService = jobInformation.coordinatorService; IMap> metricsImap = coordinatorService.getMetricsImap(); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted(() -> Assertions.assertFalse(metricsImap.isEmpty())); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted(() -> Assertions.assertTrue(metricsImap.isEmpty())); jobInformation.coordinatorService.clearCoordinatorService(); @@ -987,7 +988,7 @@ void testMetricsImapSizeWithPartitionConfig() { throw new CompletionException(e); } }); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted(() -> Assertions.assertEquals(10, metricsImap.size())); } finally { instance1.shutdown(); @@ -1002,14 +1003,19 @@ void testCleanupPendingJobMasterMapWhenJobSubmitFutureIsExceptionally() { "CoordinatorServiceTest_testCleanPendingJobMasterMap", "batch_fake_to_inmemory.conf", "test_clean_pending_jobmastermap"); - CoordinatorService coordinatorService = jobInformation.coordinatorService; - await().atMost(20000, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> - Assertions.assertFalse( - coordinatorService - .getPendingJobQueue() - .contains(jobInformation.jobId))); + try { + CoordinatorService coordinatorService = jobInformation.coordinatorService; + await().atMost(20000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> + Assertions.assertFalse( + coordinatorService + .getPendingJobQueue() + .contains(jobInformation.jobId))); + } finally { + jobInformation.coordinatorService.clearCoordinatorService(); + jobInformation.coordinatorServiceTest.shutdown(); + } } @Test @@ -1051,7 +1057,7 @@ void testSubmitJobOperationCanCompleteOnHazelcastOperationThread() { jobImmutableInformation.isStartWithSavePoint())) .join()); - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertNotEquals( @@ -1200,18 +1206,22 @@ public void testClearCoordinatorService() { Long jobId = jobInformation.jobId; HazelcastInstanceImpl coordinatorServiceTest = jobInformation.coordinatorServiceTest; - // waiting for job status turn to running - await().atMost(10000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertEquals( JobStatus.RUNNING, coordinatorService.getJobStatus(jobId))); - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + await().atMost(30, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertTrue( + Thread.getAllStackTraces().keySet().stream() + .anyMatch( + thread -> + thread.getName() + .startsWith( + "pending-job-schedule-runner")))); int scheduleRunnerThreadCount = (int) @@ -1250,7 +1260,7 @@ void testRestoreUsesProvidedJobInfoInitializationTimestamp() throws Exception { SeaTunnelServer server = instance.node.getNodeEngine().getService(SeaTunnelServer.SERVICE_NAME); CoordinatorService coordinatorService = server.getCoordinatorService(); - await().atMost(10, TimeUnit.SECONDS) + await().atMost(60, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertTrue(coordinatorService.isCoordinatorActive())); @@ -1296,7 +1306,7 @@ void testRestoreUsesProvidedJobInfoInitializationTimestamp() throws Exception { coordinatorService, "runningJobInfoIMap", runningJobInfoIMap); } - await().atMost(10, TimeUnit.SECONDS) + await().atMost(60, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertTrue( @@ -1478,6 +1488,7 @@ void testDistributedMetricsPerformance() throws Exception { executor.awaitTermination(30, TimeUnit.SECONDS); instance1.shutdown(); instance2.shutdown(); + instance3.shutdown(); } } diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceWithCancelPendingJobTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceWithCancelPendingJobTest.java index 2639ef228007..2544b71ef0cd 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceWithCancelPendingJobTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceWithCancelPendingJobTest.java @@ -142,8 +142,7 @@ public void testCancelPendingJob() throws InterruptedException { nodeEngine.getHazelcastInstance().getMap(Constant.IMAP_STATE_TIMESTAMPS); // Verify if the final status of the task is cancelled - await().pollDelay(3, TimeUnit.SECONDS) - .atMost(120, TimeUnit.SECONDS) + await().atMost(120, TimeUnit.SECONDS) .untilAsserted( () -> { Assertions.assertEquals( @@ -185,14 +184,15 @@ private JobMaster newJobInstanceWithRunningState(long jobId, boolean restore) JobMaster jobMaster = server.getCoordinatorService().getJobMaster(jobId); - // waiting for job status turn to running await().atMost(120, TimeUnit.SECONDS) .untilAsserted( - () -> Assertions.assertEquals(JobStatus.PENDING, jobMaster.getJobStatus())); - - // Because handleCheckpointTimeout is an async method, so we need sleep 5s to waiting job - // status become running again - Thread.sleep(5000); + () -> { + JobStatus status = jobMaster.getJobStatus(); + Assertions.assertTrue( + JobStatus.PENDING.equals(status) + || JobStatus.RUNNING.equals(status), + "Expected PENDING or RUNNING but was: " + status); + }); return jobMaster; } } diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointStorageTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointStorageTest.java index b95435523f02..d6b29834db67 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointStorageTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointStorageTest.java @@ -33,6 +33,8 @@ import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; +import java.io.FileNotFoundException; +import java.nio.file.NoSuchFileException; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -124,9 +126,22 @@ public void testBatchJobWithCheckpoint() throws CheckpointStorageException { Assertions.assertEquals( JobStatus.FINISHED, server.getCoordinatorService().getJobStatus(jobId))); - List allCheckpoints = - checkpointStorage.getAllCheckpoints(String.valueOf(jobId)); - Assertions.assertEquals(0, allCheckpoints.size()); + await().atMost(30000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + try { + List allCheckpoints = + checkpointStorage.getAllCheckpoints(String.valueOf(jobId)); + Assertions.assertEquals(0, allCheckpoints.size()); + } catch (CheckpointStorageException e) { + Throwable cause = e.getCause(); + if (cause instanceof FileNotFoundException + || cause instanceof NoSuchFileException) { + return; + } + throw e; + } + }); } @Test diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointTimeOutTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointTimeOutTest.java index af118329aeed..79c3115c54c7 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointTimeOutTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointTimeOutTest.java @@ -43,7 +43,7 @@ public class CheckpointTimeOutTest extends AbstractSeaTunnelServerTest { @Test public void testJobLevelCheckpointTimeOut() { long jobId = System.currentTimeMillis(); - startJob(System.currentTimeMillis(), CONF_PATH); + startJob(jobId, CONF_PATH); await().atMost(120000, TimeUnit.MILLISECONDS) .untilAsserted( diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/FollowerRunningJobsFilterTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/FollowerRunningJobsFilterTest.java index 846e9fd24e1c..266eaed4daa1 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/FollowerRunningJobsFilterTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/FollowerRunningJobsFilterTest.java @@ -60,7 +60,7 @@ void testFollowerRunningJobsApiHidesRetainedTerminalJob() { follower = SeaTunnelServerStarter.createHazelcastInstance(seaTunnelConfig); Awaitility.await() - .atMost(10, TimeUnit.SECONDS) + .atMost(60, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertEquals( @@ -80,7 +80,7 @@ void testFollowerRunningJobsApiHidesRetainedTerminalJob() { JobInfoService followerJobInfoService = new JobInfoService((NodeEngineImpl) follower.node.nodeEngine); Awaitility.await() - .atMost(10, TimeUnit.SECONDS) + .atMost(60, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertTrue( diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceTest.java index 50e5fcb3173c..a9169b02c9bb 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceTest.java @@ -71,8 +71,7 @@ public void testlistJobState() throws Exception { }); // waiting for JOB_1 status turn to FINISHED - await().pollDelay(5, TimeUnit.SECONDS) - .atMost(60000, TimeUnit.MILLISECONDS) + await().atMost(60000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { List jobStatusData = listJob(); diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/BaseServletTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/BaseServletTest.java index 8cbf7bc7f3c5..3d3d734cfcd4 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/BaseServletTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/BaseServletTest.java @@ -46,8 +46,9 @@ class BaseServletTest extends AbstractSeaTunnelServerTest { private static final Long JOB_1 = System.currentTimeMillis() + 1L; + @Override @BeforeAll - void setUp() { + public void before() { String name = this.getClass().getName(); Config hazelcastConfig = Config.loadFromString(getHazelcastConfig()); hazelcastConfig.setClusterName(TestUtils.getClusterName("RestApiServletTest_" + name)); diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpBasicTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpBasicTest.java index 0117325913b2..235ae0278eb3 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpBasicTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpBasicTest.java @@ -64,8 +64,9 @@ class RestApiHttpBasicTest extends AbstractSeaTunnelServerTest { private static final String AUTHORIZATION_HEADER = "Authorization"; private static final String BASIC_PREFIX = "Basic "; + @Override @BeforeAll - void setUp() { + public void before() { String name = this.getClass().getName(); Config hazelcastConfig = Config.loadFromString(getHazelcastConfig()); hazelcastConfig.setClusterName( @@ -99,6 +100,7 @@ public void after() { httpConfig.setEnableBasicAuth(Boolean.FALSE); httpConfig.setBasicAuthUsername(""); httpConfig.setBasicAuthPassword(""); + super.after(); } @Test diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpsForTruststoreTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpsForTruststoreTest.java index f7e848f0c68a..54e0f5246cee 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpsForTruststoreTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpsForTruststoreTest.java @@ -54,8 +54,9 @@ public class RestApiHttpsForTruststoreTest extends AbstractSeaTunnelServerTest { private static final String CLIENT_KEYSTORE_PASSWORD = "client_keystore_password"; private static final String CLIENT_TRUSTSTORE_PASSWORD = "client_truststore_password"; + @Override @BeforeAll - public void setUp() { + public void before() { String name = this.getClass().getName(); Config hazelcastConfig = Config.loadFromString(getHazelcastConfig()); hazelcastConfig.setClusterName( diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpsTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpsTest.java index 29c3b90b5126..f9d5ebae7e15 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpsTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/rest/RestApiHttpsTest.java @@ -67,8 +67,9 @@ public class RestApiHttpsTest extends AbstractSeaTunnelServerTest { private static final String SERVER_KEYSTORE_PASSWORD = "server_keystore_password"; private static final String CLIENT_KEYSTORE_PASSWORD = "client_keystore_password"; + @Override @BeforeAll - public void setUp() { + public void before() { String name = this.getClass().getName(); Config hazelcastConfig = Config.loadFromString(getHazelcastConfig()); hazelcastConfig.setClusterName(TestUtils.getClusterName("RestApiHttpsTest_" + name)); @@ -149,9 +150,8 @@ public void testFinishedJobsApi() throws Exception { } // wait until all jobs are finished - await().pollDelay(5, TimeUnit.SECONDS) - .atMost(30, TimeUnit.SECONDS) - .pollInterval(100, TimeUnit.MILLISECONDS) + await().atMost(60, TimeUnit.SECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) .untilAsserted( () -> assertEquals( @@ -255,9 +255,8 @@ public void testPageNumberOutOfRange() throws Exception { } // wait until all jobs are finished - await().pollDelay(5, TimeUnit.SECONDS) - .atMost(30, TimeUnit.SECONDS) - .pollInterval(100, TimeUnit.MILLISECONDS) + await().atMost(60, TimeUnit.SECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) .untilAsserted( () -> assertEquals( diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExportsTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExportsTest.java index 71c070031a9c..f9390dd99371 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExportsTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExportsTest.java @@ -57,7 +57,7 @@ void collectShouldExportLocalStateStoreMetrics() { instance = SeaTunnelServerStarter.createHazelcastInstance( TestUtils.getClusterName("EngineStateStoreMetricExportsTest_localMetrics")); - await().atMost(10, TimeUnit.SECONDS) + await().atMost(60, TimeUnit.SECONDS) .untilAsserted(() -> Assertions.assertTrue(instance.node.isMaster())); instance.getMap(Constant.IMAP_RUNNING_JOB_INFO).put(1L, "job-info"); @@ -92,7 +92,7 @@ void collectShouldCoverAllEngineStateStores() { SeaTunnelServerStarter.createHazelcastInstance( TestUtils.getClusterName( "EngineStateStoreMetricExportsTest_allStateStores")); - await().atMost(10, TimeUnit.SECONDS) + await().atMost(60, TimeUnit.SECONDS) .untilAsserted(() -> Assertions.assertTrue(instance.node.isMaster())); List metrics = From 122d090877b4a33407afc9f31c28d4d8fb9608cc Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 13 Jun 2026 17:41:00 +0800 Subject: [PATCH 011/375] [Docs] Clarify REST API defaults and add connector troubleshooting cookbooks (#11035) --- docs/en/connectors/sink/Jdbc.md | 28 ++++++- docs/en/connectors/sink/PostgreSql.md | 42 ++++++++++ docs/en/connectors/source/Http.md | 106 +++++++++++++++++++++++++- docs/en/engines/zeta/rest-api-v2.md | 38 +++++++-- docs/en/transforms/metadata.md | 54 +++++++++++++ docs/zh/connectors/sink/Jdbc.md | 28 ++++++- docs/zh/connectors/sink/PostgreSql.md | 41 +++++++++- docs/zh/connectors/source/Http.md | 106 +++++++++++++++++++++++++- docs/zh/engines/zeta/rest-api-v2.md | 36 ++++++++- docs/zh/transforms/metadata.md | 53 +++++++++++++ 10 files changed, 516 insertions(+), 16 deletions(-) diff --git a/docs/en/connectors/sink/Jdbc.md b/docs/en/connectors/sink/Jdbc.md index 40f47c98ada9..c5ee8ccd798f 100644 --- a/docs/en/connectors/sink/Jdbc.md +++ b/docs/en/connectors/sink/Jdbc.md @@ -569,7 +569,9 @@ Not all databases support XA transactions. Verify that your database and JDBC dr ### How do I configure upsert (INSERT or UPDATE) behavior? -Specify `primary_keys` to enable upsert behavior. SeaTunnel generates an INSERT ... ON DUPLICATE KEY UPDATE (or equivalent) statement based on the target database dialect: +SeaTunnel only enters the upsert / update path after it has a final key set. That key can come from explicit `primary_keys`, or, when `primary_keys` is omitted, from upstream catalog metadata. If no primary key is available, SeaTunnel also tries to inherit the first unique key. + +When a final key set exists and `enable_upsert = true`, SeaTunnel prefers the database-native upsert statement provided by the target dialect. For example, PostgreSQL generates `INSERT ... ON CONFLICT (...) DO UPDATE` (or `DO NOTHING` when every column is part of the key and there is nothing left to update): ```hocon sink { @@ -582,7 +584,29 @@ sink { } ``` -Without `primary_keys`, JDBC Sink performs plain INSERTs and does not handle duplicate key conflicts. +When a final key set exists but `enable_upsert = false`, SeaTunnel stops using native database upsert SQL and falls back to the row-kind-driven insert/update path: + +- `INSERT` rows are written as plain INSERTs +- CDC `UPDATE_AFTER` rows are written as UPDATEs +- CDC `DELETE` rows are written as DELETEs + +As a result, `enable_upsert = false` is not appropriate for ordinary batch imports that rely on duplicate-key overwrite behavior. + +### What happens if I do not configure `primary_keys`? + +If `primary_keys` is not configured, SeaTunnel first tries to inherit the primary key from upstream catalog metadata. If there is no primary key, it then tries the first unique key. + +JDBC Sink falls back to plain INSERT only when there is no explicit key and nothing usable can be inherited from upstream metadata. In that keyless mode, no database-native upsert SQL is generated, and the sink no longer uses row-kind-aware UPDATE / DELETE executors. For CDC inputs, the write path therefore effectively degrades to plain INSERT batching, and duplicate-key behavior depends entirely on the target table constraints. + +### When should I enable `use_copy_statement`? + +`use_copy_statement = true` makes JDBC Sink prefer the `COPY (...) FROM STDIN WITH CSV` path instead of regular INSERT / UPSERT SQL. This happens before the normal primary-key-based write path, so COPY is still chosen even if `primary_keys` is configured. + +This option is mainly for high-volume PostgreSQL imports, and it has three important constraints: + +- the JDBC driver connection must expose `getCopyAPI()`, otherwise the job fails and tells you to switch `use_copy_statement` back to `false` +- it is not a replacement for `ON CONFLICT`, so it does not provide duplicate-key overwrite semantics +- `MAP`, `ARRAY`, and `ROW` types are not supported ### How do I write to multiple tables in a single job? diff --git a/docs/en/connectors/sink/PostgreSql.md b/docs/en/connectors/sink/PostgreSql.md index e1ae7339abb4..6ec9fc16dbdf 100644 --- a/docs/en/connectors/sink/PostgreSql.md +++ b/docs/en/connectors/sink/PostgreSql.md @@ -99,6 +99,7 @@ semantics (using XA transaction guarantee). | data_save_mode | Enum | no | APPEND_DATA | Before the synchronous task is turned on, different processing schemes are selected for data existing data on the target side. | | custom_sql | String | no | - | When data_save_mode selects CUSTOM_PROCESSING, you should fill in the CUSTOM_SQL parameter. This parameter usually fills in a SQL that can be executed. SQL will be executed before synchronization tasks. | | enable_upsert | Boolean | No | true | Enable upsert by primary_keys exist, If the task has no key duplicate data, setting this parameter to `false` can speed up data import | +| use_copy_statement | Boolean | No | false | Use PostgreSQL `COPY
(...) FROM STDIN WITH CSV` for bulk import. This option takes precedence over the regular INSERT / UPSERT path, requires a JDBC driver connection that exposes `getCopyAPI()`, and does not support `MAP`, `ARRAY`, or `ROW` types. | ### table [string] @@ -273,6 +274,47 @@ sink { } ``` +## FAQ + +### When does PostgreSQL Sink generate `ON CONFLICT`? + +PostgreSQL Sink generates `INSERT ... ON CONFLICT (...) DO UPDATE` only after it has a final +primary-key / unique-key set and `enable_upsert = true`. + +That key can come from two places: + +- explicit `primary_keys` +- inherited upstream catalog metadata when `primary_keys` is omitted; if no primary key exists, SeaTunnel also tries the first unique key + +If every column is part of the key and there is nothing left to update, PostgreSQL degrades this to +`ON CONFLICT (...) DO NOTHING`. + +### What happens when the target table has no primary key? + +If there is no explicit `primary_keys` setting and no inheritable primary key or unique key in +upstream metadata, PostgreSQL Sink falls back to plain INSERT and does not generate `ON CONFLICT`. +In that keyless mode, the sink also stops using row-kind-aware UPDATE / DELETE executors, so CDC +inputs effectively degrade to plain INSERT batching. Duplicate-key behavior then depends entirely on +the target table constraints. + +### How should I choose between `use_copy_statement` and `enable_upsert`? + +`use_copy_statement = true` makes PostgreSQL Sink enter the COPY bulk-load path before the normal +INSERT / UPSERT path. In other words, COPY still wins even if `primary_keys` is configured and +`enable_upsert = true`. + +COPY is a good fit when: + +- the target is PostgreSQL +- the goal is high-throughput bulk import +- you do not rely on PostgreSQL native upsert semantics for duplicate-key overwrite + +COPY is not a good fit when: + +- you want PostgreSQL native upsert conflict handling +- your data contains `MAP`, `ARRAY`, or `ROW` types +- the current JDBC driver connection does not expose `getCopyAPI()` + ## Changelog diff --git a/docs/en/connectors/source/Http.md b/docs/en/connectors/source/Http.md index b0752c72a800..2d07388a869c 100644 --- a/docs/en/connectors/source/Http.md +++ b/docs/en/connectors/source/Http.md @@ -192,7 +192,7 @@ connector will generate data as the following: |----------------------------------------------------------| | {"code": 200, "data": "get success", "success": true} | -when you assign format is `binary`, the HTTP response body is treated as raw bytes for downloading files (PDF, images, ZIP, etc.). The output schema is fixed as `(data: bytes, relativePath: string, partIndex: long)`. Large files are automatically split into multiple rows based on `binary_chunk_size`. Only supports BATCH mode. +when you assign format is `binary`, the HTTP response body is treated as raw bytes for downloading files (PDF, images, ZIP, etc.). The output schema is fixed as `(data: bytes, relativePath: string, partIndex: long)`. Large files are automatically split into multiple rows based on `binary_chunk_size`. Only supports BATCH mode and does not support pagination (`pageing`). Example: Download a file via HTTP and write to LocalFileSink: @@ -266,6 +266,110 @@ headers { } ``` +### Pagination and final request shape + +The easiest way to troubleshoot the Http source is to reason from the final outbound request: + +1. For `GET`, `params` are always appended to the URL query string. +2. For `POST` with `keep_params_as_form = false`: + - `params` still go to the URL query string + - on the default non-form path, `body` is serialized as the JSON request body + - if `body` is not configured and the request stays on that default non-form path, the runtime sends an empty JSON object `{}` as the request body + - if you explicitly force `Content-Type: application/x-www-form-urlencoded`, the runtime follows the form-body branch instead of the default JSON branch +3. For `POST` with `keep_params_as_form = true`: + - `params` are merged into the form body + - if `Content-Type` is not set explicitly, SeaTunnel adds `application/x-www-form-urlencoded` + - if `body` and `params` contain the same key, the value from `params` overrides the value in `body` +4. `keep_page_param_as_http_param = true` writes the paging field directly into `params` +5. `keep_page_param_as_http_param = false` only updates existing keys or placeholders in headers, params, and body; it does not invent new paging fields automatically +6. `pageing.use_placeholder_replacement = true` supports `${page}` and `${cursor}` placeholders, and also prefixed/suffixed replacements such as `"10${page}" -> "105"`; when `false`, only key-based replacement is applied + +Example 1: GET pagination with the page number in query parameters + +```hocon +source { + Http { + url = "https://api.example.com/orders" + method = "GET" + params = { + page = "${page}" + size = "100" + } + pageing = { + page_field = "page" + page_type = "PageNumber" + start_page_number = 3 + use_placeholder_replacement = true + } + } +} +``` + +When the page advances to `3`, the final request is: + +```text +GET https://api.example.com/orders?page=3&size=100 +``` + +Example 2: POST JSON on the default non-form path, with URL query parameters and the paging field inside the body + +```hocon +source { + Http { + url = "https://api.example.com/orders/search" + method = "POST" + keep_params_as_form = false + params = { + tenant = "acme" + } + body = """{"page":"${page}","pageSize":100}""" + pageing = { + page_field = "page" + page_type = "PageNumber" + start_page_number = 3 + use_placeholder_replacement = true + } + } +} +``` + +When the page advances to `3`, the final request is: + +```text +POST https://api.example.com/orders/search?tenant=acme +Content-Type: application/json +Body: {"page":"3","pageSize":100} +``` + +Example 3: POST form submission with paging fields merged into the form body + +```hocon +source { + Http { + url = "https://api.example.com/orders/search" + method = "POST" + keep_params_as_form = true + keep_page_param_as_http_param = true + params = { + size = "100" + } + pageing = { + page_field = "page" + page_type = "PageNumber" + start_page_number = 3 + } + } +} +``` + +When the page advances to `3`, the final request is: + +```text +POST https://api.example.com/orders/search +Content-Type: application/x-www-form-urlencoded +Body: size=100&page=3 +``` + ### content_field This parameter can get some json data.If you only need the data in the 'book' section, configure `content_field = "$.store.book.*"`. diff --git a/docs/en/engines/zeta/rest-api-v2.md b/docs/en/engines/zeta/rest-api-v2.md index 672e3f962dbb..7afe0fc85391 100644 --- a/docs/en/engines/zeta/rest-api-v2.md +++ b/docs/en/engines/zeta/rest-api-v2.md @@ -5,10 +5,31 @@ completed jobs. The monitoring API is a RESTful API that accepts HTTP requests a ## Overview -The v2 version of the api uses jetty support. It is the same as the interface specification of v1 version -, you can specify the port and context-path by modifying the configuration items in `seatunnel.yaml`, -you can configure `enable-dynamic-port` to enable dynamic ports (the default port is accumulated starting from `port`), and the default is enabled, -If enable-dynamic-port is true, We will use the unused port in the range within the range of `port` and `port` + `port-range`, default range is 100 +The v2 API and the Web UI are both served by the embedded Jetty server. Jetty starts only when +`seatunnel.engine.http.enable-http = true` or `enable-https = true`. + +There are two different "default" sources that are easy to mix up: + +- Code defaults: `enable-http = false`, `enable-https = false`, `port = 8080`, `context-path = ""`, `enable-dynamic-port = false`, `port-range = 100` +- The packaged `seatunnel.yaml` example: it already sets `enable-http: true` and `port: 8080` + +As a result, if you start SeaTunnel with the packaged configuration, the Web UI and REST API usually +listen on `http://:8080/`. If you build a minimal config yourself, rely on code defaults, or +remove `enable-http`, Jetty will not start by default. + +Use the following configuration for a fixed port: + +```yaml + +seatunnel: + engine: + http: + enable-http: true + port: 8080 +``` + +If you want Jetty to choose the first free port between `port` and `port + port-range`, enable +dynamic ports explicitly: ```yaml @@ -21,7 +42,7 @@ seatunnel: port-range: 100 ``` -Context-path can also be configured as follows: +`context-path` can also be configured as follows: ```yaml @@ -33,6 +54,13 @@ seatunnel: context-path: /seatunnel ``` +## Web UI and Port 8080 Troubleshooting + +- If `http://:8080/` is unreachable, first check whether `seatunnel.engine.http.enable-http` or `enable-https` is actually enabled. The `network.rest-api.enabled` setting in `hazelcast.yaml` does not replace the Jetty switch. +- If `enable-dynamic-port = true`, the actual listening port may not be 8080. Jetty will choose the first available port between `port` and `port + port-range`. Use the startup log `SeaTunnel REST service will start on port xxx` as the source of truth. +- If `context-path = /seatunnel`, both the Web UI and REST endpoints move under that prefix. For example, the overview endpoint becomes `/seatunnel/overview`. +- The Web UI static resources and REST endpoints share the same Jetty service. If Jetty does not start, both are unavailable together. + ## Enable HTTPS Please refer [security](security.md) diff --git a/docs/en/transforms/metadata.md b/docs/en/transforms/metadata.md index 1ccba8781bbd..5a311465b04f 100644 --- a/docs/en/transforms/metadata.md +++ b/docs/en/transforms/metadata.md @@ -246,3 +246,57 @@ sink { ``` Here `pt` is derived from the Kafka event time and can be used as a Hive partition column. + +### Example 4: Combine Metadata and Sql to extract table suffixes and add a load date + +When the upstream CDC source uses sharded tables such as monthly or daily tables, a common pattern +is to expose the `Table` metadata as a regular field first, then use `Sql` to derive the shard +suffix and a formatted load date. + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" +} + +source { + MySQL-CDC { + plugin_output = "orders_cdc" + server-id = 5652 + username = "root" + password = "your_password" + table-names = ["app.orders_202401", "app.orders_202402"] + url = "jdbc:mysql://localhost:3306/app" + } +} + +transform { + Metadata { + plugin_input = "orders_cdc" + plugin_output = "orders_with_meta" + metadata_fields { + Table = source_table + EventTime = event_ts + } + } + + Sql { + plugin_input = "orders_with_meta" + plugin_output = "orders_normalized" + query = "select id, amount, source_table, REGEXP_SUBSTR(source_table, '[0-9]+$') as table_suffix, FROM_UNIXTIME(event_ts / 1000, 'yyyy-MM-dd HH:mm:ss', 'Asia/Shanghai') as event_time_str, FORMATDATETIME(CURRENT_TIMESTAMP, 'yyyyMMdd') as load_date from orders_with_meta" + } +} + +sink { + Console { + plugin_input = "orders_normalized" + } +} +``` + +If the current record comes from `orders_202402`, then: + +- `source_table = "orders_202402"` +- `table_suffix = "202402"` +- `event_time_str` comes from the CDC event time +- `load_date` is the formatted runtime date string diff --git a/docs/zh/connectors/sink/Jdbc.md b/docs/zh/connectors/sink/Jdbc.md index fb6f7075dfe3..732eaa8df6c8 100644 --- a/docs/zh/connectors/sink/Jdbc.md +++ b/docs/zh/connectors/sink/Jdbc.md @@ -474,7 +474,9 @@ sink { ### 如何配置 Upsert（INSERT OR UPDATE）行为？ -指定 `primary_keys` 即可启用 upsert。SeaTunnel 会根据目标数据库方言自动生成 `INSERT ... ON DUPLICATE KEY UPDATE`（或等效）语句： +SeaTunnel 只有在最终拿到了主键/唯一键信息时，才会进入 upsert / update 路径。这个 key 既可以来自显式配置的 `primary_keys`，也可以在未显式配置时从上游 catalog 元数据里继承主键；如果没有主键，还会尝试继承第一组 unique key。 + +当最终存在 key 且 `enable_upsert = true` 时，SeaTunnel 会优先使用目标数据库方言支持的原生 upsert 语句。例如 PostgreSQL 会生成 `INSERT ... ON CONFLICT (...) DO UPDATE`（如果所有字段都是主键，没有可更新列，则退化为 `DO NOTHING`）： ```hocon sink { @@ -487,7 +489,29 @@ sink { } ``` -不设置 `primary_keys` 时，JDBC Sink 执行普通 INSERT，不处理主键冲突。 +当最终存在 key 但 `enable_upsert = false` 时，SeaTunnel 不再生成数据库原生 upsert 语句，而是回到按行类型执行的 insert/update 路径： + +- `INSERT` 行执行普通 INSERT +- CDC `UPDATE_AFTER` 行执行 UPDATE +- CDC `DELETE` 行执行 DELETE + +因此，`enable_upsert = false` 不适合依赖重复键自动覆盖的普通批量导入场景。 + +### 未显式配置 `primary_keys` 时会发生什么？ + +如果你没有显式配置 `primary_keys`，SeaTunnel 会先尝试从上游 catalog 元数据继承主键；如果没有主键，则再尝试继承第一组 unique key。 + +只有在“显式配置也没有、上游元数据里也没有可继承 key”时，JDBC Sink 才会退回普通 INSERT。进入这个无 key 模式后，不仅不会生成数据库原生 upsert 语句，Sink 也不会再使用按 `RowKind` 执行 UPDATE / DELETE 的写入器。对于 CDC 输入，这条写链路会实质上退化为普通 INSERT batching，重复键是否报错完全取决于目标表自身约束。 + +### 什么时候应该开启 `use_copy_statement`？ + +`use_copy_statement = true` 会让 JDBC Sink 直接优先走 `COPY
(...) FROM STDIN WITH CSV` 路径，而不是常规的 INSERT / UPSERT 语句。即使同时配置了 `primary_keys`，也会优先进入 COPY 路径。 + +这个选项更适合 PostgreSQL 大批量导入场景，但要同时满足下面几个前提： + +- JDBC 驱动连接对象必须提供 `getCopyAPI()` 能力；否则任务会直接报错，并提示把 `use_copy_statement` 改回 `false` +- 它不是 `ON CONFLICT` 的替代品，不负责处理重复键覆盖逻辑 +- 当前不支持 `MAP`、`ARRAY`、`ROW` 类型 ### 如何在一个任务中写入多张表？ diff --git a/docs/zh/connectors/sink/PostgreSql.md b/docs/zh/connectors/sink/PostgreSql.md index 885721eef3b4..947437b664bb 100644 --- a/docs/zh/connectors/sink/PostgreSql.md +++ b/docs/zh/connectors/sink/PostgreSql.md @@ -95,6 +95,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; | data_save_mode | Enum | 否 | APPEND_DATA | 在同步任务开启之前，根据目标端现有数据选择不同处理方案。 | | custom_sql | String | 否 | - | 当 `data_save_mode` 选择 `CUSTOM_PROCESSING` 时，您应该填写 `CUSTOM_SQL` 参数。此参数通常填入可执行的 SQL。SQL 将在同步任务之前执行。 | | enable_upsert | Boolean | 否 | true | 通过主键存在启用 upsert，如果任务没有重复数据，设置此参数为 `false` 可以加快数据导入。 | +| use_copy_statement | Boolean | 否 | false | 直接使用 PostgreSQL `COPY
(...) FROM STDIN WITH CSV` 进行批量导入。该选项优先级高于常规 INSERT / UPSERT 路径；要求 JDBC 驱动连接对象提供 `getCopyAPI()`，且当前不支持 `MAP`、`ARRAY`、`ROW` 类型。 | ### table [字符串] @@ -269,6 +270,44 @@ sink { } ``` +## 常见问题 + +### PostgreSQL Sink 什么时候会生成 `ON CONFLICT`？ + +只有在最终拿到了主键/唯一键信息，并且 `enable_upsert = true` 时，PostgreSQL Sink 才会生成 +`INSERT ... ON CONFLICT (...) DO UPDATE`。 + +这个 key 可以来自两种地方： + +- 你显式配置的 `primary_keys` +- 你没有显式配置时，SeaTunnel 从上游 catalog 元数据继承到的主键；如果没有主键，还会尝试第一组 unique key + +如果所有字段本身都是主键，没有可更新列，PostgreSQL 会退化为 +`ON CONFLICT (...) DO NOTHING`。 + +### 目标表没有主键时会发生什么？ + +如果既没有显式配置 `primary_keys`，上游元数据里也没有可继承的主键或 unique key， +PostgreSQL Sink 会退回普通 INSERT，不会生成 `ON CONFLICT`。进入这个无 key 模式后， +Sink 也不会再使用按 `RowKind` 执行 UPDATE / DELETE 的写入器，因此 CDC 输入会实质上退化为普通 +INSERT batching。这时重复键是否报错，完全取决于目标表自身约束。 + +### `use_copy_statement` 和 `enable_upsert` 应该怎么选？ + +`use_copy_statement = true` 会优先进入 PostgreSQL COPY 批量导入路径，而不是常规 INSERT / UPSERT 语句。也就是说，即使同时配置了 `primary_keys` 和 `enable_upsert = true`，COPY 仍然会优先执行。 + +适合开启 COPY 的场景： + +- 目标是 PostgreSQL +- 重点是高吞吐批量导入 +- 不依赖 `ON CONFLICT` 处理重复键覆盖逻辑 + +不适合开启 COPY 的场景： + +- 你希望通过 PostgreSQL 原生 upsert 语义处理冲突 +- 你的数据包含 `MAP`、`ARRAY`、`ROW` 类型 +- 当前 JDBC 驱动连接对象不提供 `getCopyAPI()` + ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/source/Http.md b/docs/zh/connectors/source/Http.md index a3e5dcf8ad45..defc7c664f74 100644 --- a/docs/zh/connectors/source/Http.md +++ b/docs/zh/connectors/source/Http.md @@ -181,7 +181,7 @@ schema { |----------------------------------------------------------| | {"code": 200, "data": "get success", "success": true} | -当您指定 format 为 `binary` 时，HTTP 响应体作为原始字节处理，用于下载文件（PDF、图片、ZIP 等）。输出 schema 固定为 `(data: bytes, relativePath: string, partIndex: long)`。大文件会根据 `binary_chunk_size` 自动拆分为多行。仅支持 BATCH 模式。 +当您指定 format 为 `binary` 时，HTTP 响应体作为原始字节处理，用于下载文件（PDF、图片、ZIP 等）。输出 schema 固定为 `(data: bytes, relativePath: string, partIndex: long)`。大文件会根据 `binary_chunk_size` 自动拆分为多行。仅支持 BATCH 模式，且不支持分页（`pageing`）。 示例：通过 HTTP 下载文件并写入 LocalFileSink： @@ -255,6 +255,110 @@ headers { } ``` +### 分页与最终请求形态排查 + +下面几条规则最容易混淆，建议先按“最终发出的 HTTP 请求长什么样”来理解： + +1. `GET` 请求：`params` 一定会被拼到 URL 查询串里。 +2. `POST` 且 `keep_params_as_form = false`： + - `params` 仍然会拼到 URL 查询串里 + - 在默认的非 form 分支上，`body` 会作为 JSON body 发送 + - 如果没有配置 `body`，并且请求仍然走这个默认非 form 分支，运行时会发送一个空 JSON 对象 `{}` 作为请求体 + - 如果你显式把 `Content-Type` 设为 `application/x-www-form-urlencoded`，运行时会改走 form-body 分支，而不是默认 JSON 分支 +3. `POST` 且 `keep_params_as_form = true`： + - `params` 会并入表单 body + - 如果未显式设置 `Content-Type`，SeaTunnel 会自动补 `application/x-www-form-urlencoded` + - 如果 `body` 与 `params` 出现同名键，`params` 的值会覆盖 `body` 中同名键 +4. `keep_page_param_as_http_param = true`：分页字段会直接写入 `params` +5. `keep_page_param_as_http_param = false`：SeaTunnel 只会更新 headers、params、body 里已经存在的同名键或占位符，不会凭空新增分页字段 +6. `pageing.use_placeholder_replacement = true`：支持 `${page}`、`${cursor}` 占位符，也支持 `"10${page}" -> "105"` 这种带前后缀的替换；为 `false` 时只做按 key 的整值替换 + +示例 1：GET 分页，页码写入查询参数 + +```hocon +source { + Http { + url = "https://api.example.com/orders" + method = "GET" + params = { + page = "${page}" + size = "100" + } + pageing = { + page_field = "page" + page_type = "PageNumber" + start_page_number = 3 + use_placeholder_replacement = true + } + } +} +``` + +当页码推进到 `3` 时，最终请求为： + +```text +GET https://api.example.com/orders?page=3&size=100 +``` + +示例 2：POST JSON（默认非 form 分支），请求参数进 URL，分页字段留在 body + +```hocon +source { + Http { + url = "https://api.example.com/orders/search" + method = "POST" + keep_params_as_form = false + params = { + tenant = "acme" + } + body = """{"page":"${page}","pageSize":100}""" + pageing = { + page_field = "page" + page_type = "PageNumber" + start_page_number = 3 + use_placeholder_replacement = true + } + } +} +``` + +当页码推进到 `3` 时，最终请求为： + +```text +POST https://api.example.com/orders/search?tenant=acme +Content-Type: application/json +Body: {"page":"3","pageSize":100} +``` + +示例 3：POST 表单，请求参数和分页字段都写入表单 body + +```hocon +source { + Http { + url = "https://api.example.com/orders/search" + method = "POST" + keep_params_as_form = true + keep_page_param_as_http_param = true + params = { + size = "100" + } + pageing = { + page_field = "page" + page_type = "PageNumber" + start_page_number = 3 + } + } +} +``` + +当页码推进到 `3` 时，最终请求为： + +```text +POST https://api.example.com/orders/search +Content-Type: application/x-www-form-urlencoded +Body: size=100&page=3 +``` + ### content_field 此参数可以获取一些 json 数据。如果您只需要 'book' 部分的数据，配置 `content_field = "$.store.book.*"`。 diff --git a/docs/zh/engines/zeta/rest-api-v2.md b/docs/zh/engines/zeta/rest-api-v2.md index b5f3d1365829..9b9676de35ce 100644 --- a/docs/zh/engines/zeta/rest-api-v2.md +++ b/docs/zh/engines/zeta/rest-api-v2.md @@ -4,9 +4,30 @@ SeaTunnel有一个用于监控的API，可用于查询运行作业的状态和 ## 概述 -v2版本的api使用jetty支持，与v1版本的接口规范相同 ,可以通过修改`seatunnel.yaml`中的配置项来指定端口和context-path， -同时可以配置 `enable-dynamic-port` 开启动态端口(默认从 `port` 开始累加)，默认为开启， -如果`enable-dynamic-port`为`true`，我们将使用`port`和`port`+`port-range`范围内未使用的端口，默认范围是100。 +v2 版本的 API 和 Web UI 都由内嵌 Jetty 提供，与 v1 版本保持相同的接口规范。只有当 +`seatunnel.engine.http.enable-http = true` 或 `enable-https = true` 时，Jetty 才会启动。 + +这里需要区分两个容易混淆的“默认值”来源： + +- 代码默认值：`enable-http = false`、`enable-https = false`、`port = 8080`、`context-path = ""`、`enable-dynamic-port = false`、`port-range = 100` +- 发行包自带的 `seatunnel.yaml` 示例：默认写入了 `enable-http: true` 和 `port: 8080` + +因此，直接使用发行包自带配置启动时，Web UI 和 REST API 通常会监听 +`http://:8080/`。如果你是自己精简配置文件、按代码默认值装配配置，或者把 +`enable-http` 删掉了，那么 Jetty 默认不会启动。 + +固定端口示例如下： + +```yaml + +seatunnel: + engine: + http: + enable-http: true + port: 8080 +``` + +如需在 `port` 到 `port + port-range` 范围内自动挑选空闲端口，再显式开启动态端口： ```yaml @@ -19,7 +40,7 @@ seatunnel: port-range: 100 ``` -同时也可以配置context-path,配置如下： +同时也可以配置 `context-path`，配置如下： ```yaml @@ -31,6 +52,13 @@ seatunnel: context-path: /seatunnel ``` +## Web UI 与 8080 排查 + +- 如果 `http://:8080/` 打不开，先检查 `seatunnel.engine.http.enable-http` 或 `enable-https` 是否真的开启；仅配置 `hazelcast.yaml` 中的 `network.rest-api.enabled` 不能替代 Jetty 开关。 +- 如果开启了 `enable-dynamic-port = true`，实际监听端口可能不是 8080，而是 `port` 到 `port + port-range` 之间的第一个空闲端口。以启动日志 `SeaTunnel REST service will start on port xxx` 为准。 +- 如果配置了 `context-path = /seatunnel`，Web UI 首页和 REST 路径都会整体前移，例如概览接口会变成 `/seatunnel/overview`。 +- Web UI 静态资源和 REST API 共用同一个 Jetty 服务。只要 Jetty 没启动，两者都会一起不可用。 + ## 开启 HTTPS 请参考 [security](security.md) diff --git a/docs/zh/transforms/metadata.md b/docs/zh/transforms/metadata.md index 552ea73ad921..d51c3264a7a9 100644 --- a/docs/zh/transforms/metadata.md +++ b/docs/zh/transforms/metadata.md @@ -246,3 +246,56 @@ sink { ``` 上面的 `pt` 字段由 Kafka 事件时间转换而来，可在 Hive 中作为分区列使用，便于补数和校准分区。 + +### 示例 4：结合 Metadata 和 Sql 提取分表后缀并生成装载日期 + +当上游是按月或按天分表的 CDC 源时，常见需求是先把 `Table` 元数据暴露成普通字段，再用 +`Sql` 提取分表后缀、补充任务装载日期。 + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" +} + +source { + MySQL-CDC { + plugin_output = "orders_cdc" + server-id = 5652 + username = "root" + password = "your_password" + table-names = ["app.orders_202401", "app.orders_202402"] + url = "jdbc:mysql://localhost:3306/app" + } +} + +transform { + Metadata { + plugin_input = "orders_cdc" + plugin_output = "orders_with_meta" + metadata_fields { + Table = source_table + EventTime = event_ts + } + } + + Sql { + plugin_input = "orders_with_meta" + plugin_output = "orders_normalized" + query = "select id, amount, source_table, REGEXP_SUBSTR(source_table, '[0-9]+$') as table_suffix, FROM_UNIXTIME(event_ts / 1000, 'yyyy-MM-dd HH:mm:ss', 'Asia/Shanghai') as event_time_str, FORMATDATETIME(CURRENT_TIMESTAMP, 'yyyyMMdd') as load_date from orders_with_meta" + } +} + +sink { + Console { + plugin_input = "orders_normalized" + } +} +``` + +如果当前记录来自 `orders_202402`，那么： + +- `source_table = "orders_202402"` +- `table_suffix = "202402"` +- `event_time_str` 来自 CDC 事件时间 +- `load_date` 是任务运行时格式化后的日期字符串 From 089fe6a7ff7fc20f82b24c43fe413ecd088b94a7 Mon Sep 17 00:00:00 2001 From: cloverdew Date: Sat, 13 Jun 2026 17:48:53 +0800 Subject: [PATCH 012/375] [Fix][Connector-v2][CDC] Fix job hang caused by checkpoint failure during schema evolution under multi-parallelism on Flink1.13 (#10951) --- .../execution/SourceExecuteProcessor.java | 71 ++++++ .../AbstractSourceExecuteProcessor.java | 238 ++++++++++++++++++ .../execution/SourceExecuteProcessor.java | 156 +----------- ...cdc_to_mysql_with_flink_schema_change.conf | 2 +- ...with_flink_schema_change_exactly_once.conf | 2 +- .../flink/schema/SchemaOperator13.java | 106 ++++++++ .../flink/schema/SchemaOperator.java | 157 +++++++++--- .../coordinator/LocalSchemaCoordinator.java | 78 ++++-- .../flink/source/FlinkSourceReader.java | 23 +- .../source/FlinkSourceReaderContext.java | 4 + .../flink/schema/SchemaOperatorTest.java | 76 ++++++ .../flink/sink/FlinkSinkWriterTest.java | 66 ++++- 12 files changed, 773 insertions(+), 206 deletions(-) create mode 100644 seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java create mode 100644 seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/AbstractSourceExecuteProcessor.java create mode 100644 seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-13/src/main/java/org/apache/seatunnel/translation/flink/schema/SchemaOperator13.java diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java new file mode 100644 index 000000000000..981227491773 --- /dev/null +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.flink.execution; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import org.apache.seatunnel.api.common.JobContext; +import org.apache.seatunnel.api.source.SupportSchemaEvolution; +import org.apache.seatunnel.translation.flink.schema.SchemaOperator; +import org.apache.seatunnel.translation.flink.schema.SchemaOperator13; + +import java.net.URL; +import java.util.List; + +/** + * Flink 1.13-specific source execution processor. Shadows the common {@code SourceExecuteProcessor} + * at runtime (same package, same class name) to provide two Flink 1.13-specific behaviours without + * using reflection: + * + *
    + *
  1. {@link #createSchemaOperator} returns {@link SchemaOperator13}, which registers the + * checkpoint-stall fallback timer via the strongly-typed {@code + * ProcessingTimeService.registerTimer} API instead of a background {@code + * ScheduledExecutorService} + reflection. + *
  2. {@link #supportsSinkFunctionFinish} hard-codes {@code false}: Flink 1.13's {@code + * SinkFunction} does not expose a {@code finish()} method, so this fact is known at compile + * time and no reflection is needed. + *
+ */ +@SuppressWarnings("unchecked,rawtypes") +public class SourceExecuteProcessor extends AbstractSourceExecuteProcessor { + + public SourceExecuteProcessor( + List jarPaths, + Config envConfig, + List pluginConfigs, + JobContext jobContext) { + super(jarPaths, envConfig, pluginConfigs, jobContext); + } + + @Override + protected SchemaOperator createSchemaOperator( + String jobId, SupportSchemaEvolution source, Config pluginConfig) { + return new SchemaOperator13(jobId, source, pluginConfig); + } + + /** + * Flink 1.13's {@code SinkFunction} does not have a {@code finish()} method, so source + * keep-alive must always be enabled when schema evolution is active. Returns {@code false} + * directly rather than using reflection. + */ + @Override + protected boolean supportsSinkFunctionFinish() { + return false; + } +} diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/AbstractSourceExecuteProcessor.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/AbstractSourceExecuteProcessor.java new file mode 100644 index 000000000000..158570238a40 --- /dev/null +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/AbstractSourceExecuteProcessor.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.flink.execution; + +import org.apache.seatunnel.shade.com.google.common.collect.Lists; +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; + +import org.apache.seatunnel.api.common.JobContext; +import org.apache.seatunnel.api.common.PluginIdentifier; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.options.EnvCommonOptions; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.api.source.SupportSchemaEvolution; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.factory.FactoryUtil; +import org.apache.seatunnel.api.table.factory.TableSourceFactory; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.constants.EngineType; +import org.apache.seatunnel.common.constants.PluginType; +import org.apache.seatunnel.core.starter.execution.SourceTableInfo; +import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelFactoryDiscovery; +import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSourcePluginDiscovery; +import org.apache.seatunnel.translation.flink.schema.SchemaOperator; +import org.apache.seatunnel.translation.flink.source.FlinkSource; + +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +import scala.Tuple2; + +import java.io.Serializable; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +import static org.apache.seatunnel.api.options.ConnectorCommonOptions.PLUGIN_NAME; +import static org.apache.seatunnel.api.options.ConnectorCommonOptions.PLUGIN_OUTPUT; +import static org.apache.seatunnel.api.table.factory.FactoryUtil.ensureJobModeMatch; +import static org.apache.seatunnel.common.constants.JobMode.STREAMING; + +@SuppressWarnings("unchecked,rawtypes") +public abstract class AbstractSourceExecuteProcessor + extends FlinkAbstractPluginExecuteProcessor { + + private static final String SOURCE_KEEP_ALIVE_CONFIG = "schema-changes.source-keep-alive"; + + protected AbstractSourceExecuteProcessor( + List jarPaths, + Config envConfig, + List pluginConfigs, + JobContext jobContext) { + super(jarPaths, envConfig, pluginConfigs, jobContext); + } + + @Override + public List execute(List upstreamDataStreams) { + StreamExecutionEnvironment executionEnvironment = + flinkRuntimeEnvironment.getStreamExecutionEnvironment(); + List sources = new ArrayList<>(); + for (int i = 0; i < plugins.size(); i++) { + SourceTableInfo sourceTableInfo = plugins.get(i); + SeaTunnelSource internalSource = sourceTableInfo.getSource(); + Config pluginConfig = pluginConfigs.get(i); + + DataStreamSource sourceStream = + executionEnvironment.fromSource( + new FlinkSource<>( + internalSource, + enableSourceKeepAliveIfNeeded( + internalSource, pluginConfig, envConfig)), + WatermarkStrategy.noWatermarks(), + String.format("%s-Source", internalSource.getPluginName())); + + if (pluginConfig.hasPath(EnvCommonOptions.PARALLELISM.key())) { + int parallelism = pluginConfig.getInt(EnvCommonOptions.PARALLELISM.key()); + sourceStream.setParallelism(parallelism); + } + + boolean isStreaming = + envConfig.hasPath("job.mode") + && STREAMING + .toString() + .equalsIgnoreCase(envConfig.getString("job.mode")); + + boolean enableSchemaChange = false; + for (Config cfg : pluginConfigs) { + if (cfg.hasPath("schema-changes.enabled") + && cfg.getBoolean("schema-changes.enabled")) { + enableSchemaChange = true; + break; + } + } + // add schema evolution functionality to cdc source + DataStream evolvedStream = null; + if (isStreaming + && enableSchemaChange + && sourceTableInfo.getSource() instanceof SupportSchemaEvolution) { + evolvedStream = + sourceStream.transform( + "schema-evolution", + TypeInformation.of(SeaTunnelRow.class), + createSchemaOperator( + jobContext.getJobId(), + (SupportSchemaEvolution) sourceTableInfo.getSource(), + pluginConfig)); + } + + if (evolvedStream != null) { + sources.add( + new DataStreamTableInfo( + evolvedStream, + sourceTableInfo.getCatalogTables(), + ReadonlyConfig.fromConfig(pluginConfig).get(PLUGIN_OUTPUT))); + } else { + sources.add( + new DataStreamTableInfo( + sourceStream, + sourceTableInfo.getCatalogTables(), + ReadonlyConfig.fromConfig(pluginConfig).get(PLUGIN_OUTPUT))); + } + } + return sources; + } + + private Config enableSourceKeepAliveIfNeeded( + SeaTunnelSource source, Config pluginConfig, Config currentEnvConfig) { + boolean isStreaming = + currentEnvConfig.hasPath("job.mode") + && STREAMING + .toString() + .equalsIgnoreCase(currentEnvConfig.getString("job.mode")); + boolean enableSchemaChange = + pluginConfig.hasPath("schema-changes.enabled") + && pluginConfig.getBoolean("schema-changes.enabled"); + boolean shouldEnableKeepAlive = + isStreaming + && enableSchemaChange + && source instanceof SupportSchemaEvolution + && !supportsSinkFunctionFinish(); + if (!shouldEnableKeepAlive) { + return currentEnvConfig; + } + return currentEnvConfig.withValue( + SOURCE_KEEP_ALIVE_CONFIG, ConfigValueFactory.fromAnyRef(true)); + } + + /** + * Returns the {@link SchemaOperator} instance to attach after schema-evolution-capable sources. + * Subclasses may override to return a version-specific operator (e.g. {@code SchemaOperator13} + * for Flink 1.13) that uses the public {@code ProcessingTimeService} API instead of reflection. + */ + protected SchemaOperator createSchemaOperator( + String jobId, SupportSchemaEvolution source, Config pluginConfig) { + return new SchemaOperator(jobId, source, pluginConfig); + } + + /** + * Returns {@code true} if the current Flink runtime's {@code SinkFunction} exposes a {@code + * finish()} method (introduced in Flink 1.14). When {@code false}, source keep-alive is enabled + * so pending schema changes can still be applied after all source subtasks finish. Subclasses + * may override with a hard-coded value to avoid reflection. + */ + protected boolean supportsSinkFunctionFinish() { + for (java.lang.reflect.Method method : + org.apache.flink.streaming.api.functions.sink.SinkFunction.class.getMethods()) { + if ("finish".equals(method.getName()) && method.getParameterCount() == 0) { + return true; + } + } + return false; + } + + @Override + protected List initializePlugins( + List jarPaths, List pluginConfigs) { + SeaTunnelFactoryDiscovery factoryDiscovery = + new SeaTunnelFactoryDiscovery(TableSourceFactory.class, ADD_URL_TO_CLASSLOADER); + SeaTunnelSourcePluginDiscovery sourcePluginDiscovery = + new SeaTunnelSourcePluginDiscovery(ADD_URL_TO_CLASSLOADER); + Function fallbackCreateSource = + sourcePluginDiscovery::createPluginInstance; + + List sources = new ArrayList<>(); + Set jars = new HashSet<>(); + for (Config sourceConfig : pluginConfigs) { + PluginIdentifier pluginIdentifier = + PluginIdentifier.of( + EngineType.SEATUNNEL.getEngine(), + PluginType.SOURCE.getType(), + sourceConfig.getString(PLUGIN_NAME.key())); + jars.addAll( + sourcePluginDiscovery.getPluginJarAndDependencyPaths( + Lists.newArrayList(pluginIdentifier))); + + Tuple2, List> source = + FactoryUtil.createAndPrepareSource( + ReadonlyConfig.fromConfig(sourceConfig), + classLoader, + pluginIdentifier.getPluginName(), + fallbackCreateSource, + (TableSourceFactory) + factoryDiscovery + .createOptionalPluginInstance(pluginIdentifier) + .orElse(null), + null); + + source._1().setJobContext(jobContext); + ensureJobModeMatch(jobContext, source._1()); + + sources.add(new SourceTableInfo(source._1(), source._2())); + } + jarPaths.addAll(jars); + return sources; + } +} diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java index 94393b0ec45c..6cd1efcf2050 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java @@ -17,51 +17,22 @@ package org.apache.seatunnel.core.starter.flink.execution; -import org.apache.seatunnel.shade.com.google.common.collect.Lists; import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.api.common.JobContext; -import org.apache.seatunnel.api.common.PluginIdentifier; -import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.options.EnvCommonOptions; -import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.source.SourceSplit; -import org.apache.seatunnel.api.source.SupportSchemaEvolution; -import org.apache.seatunnel.api.table.catalog.CatalogTable; -import org.apache.seatunnel.api.table.factory.FactoryUtil; -import org.apache.seatunnel.api.table.factory.TableSourceFactory; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.common.constants.EngineType; -import org.apache.seatunnel.common.constants.PluginType; -import org.apache.seatunnel.core.starter.execution.SourceTableInfo; -import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelFactoryDiscovery; -import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSourcePluginDiscovery; -import org.apache.seatunnel.translation.flink.schema.SchemaOperator; -import org.apache.seatunnel.translation.flink.source.FlinkSource; -import org.apache.flink.api.common.eventtime.WatermarkStrategy; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import scala.Tuple2; - -import java.io.Serializable; import java.net.URL; -import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.function.Function; - -import static org.apache.seatunnel.api.options.ConnectorCommonOptions.PLUGIN_NAME; -import static org.apache.seatunnel.api.options.ConnectorCommonOptions.PLUGIN_OUTPUT; -import static org.apache.seatunnel.api.table.factory.FactoryUtil.ensureJobModeMatch; -import static org.apache.seatunnel.common.constants.JobMode.STREAMING; +/** + * Default (Flink 1.15+) source execution processor. Delegates entirely to {@link + * AbstractSourceExecuteProcessor}. For Flink 1.13, this class is shadowed at runtime by the version + * in {@code seatunnel-flink-13-starter}, which overrides {@link #createSchemaOperator} and {@link + * #supportsSinkFunctionFinish} with strongly-typed Flink 1.13 implementations that avoid + * reflection. + */ @SuppressWarnings("unchecked,rawtypes") -public class SourceExecuteProcessor extends FlinkAbstractPluginExecuteProcessor { +public class SourceExecuteProcessor extends AbstractSourceExecuteProcessor { public SourceExecuteProcessor( List jarPaths, @@ -70,115 +41,4 @@ public SourceExecuteProcessor( JobContext jobContext) { super(jarPaths, envConfig, pluginConfigs, jobContext); } - - @Override - public List execute(List upstreamDataStreams) { - StreamExecutionEnvironment executionEnvironment = - flinkRuntimeEnvironment.getStreamExecutionEnvironment(); - List sources = new ArrayList<>(); - for (int i = 0; i < plugins.size(); i++) { - SourceTableInfo sourceTableInfo = plugins.get(i); - SeaTunnelSource internalSource = sourceTableInfo.getSource(); - Config pluginConfig = pluginConfigs.get(i); - FlinkSource flinkSource = new FlinkSource<>(internalSource, envConfig); - - DataStreamSource sourceStream = - executionEnvironment.fromSource( - flinkSource, - WatermarkStrategy.noWatermarks(), - String.format("%s-Source", internalSource.getPluginName())); - - if (pluginConfig.hasPath(EnvCommonOptions.PARALLELISM.key())) { - int parallelism = pluginConfig.getInt(EnvCommonOptions.PARALLELISM.key()); - sourceStream.setParallelism(parallelism); - } - - boolean isStreaming = - envConfig.hasPath("job.mode") - && STREAMING - .toString() - .equalsIgnoreCase(envConfig.getString("job.mode")); - - boolean enableSchemaChange = false; - for (Config cfg : pluginConfigs) { - if (cfg.hasPath("schema-changes.enabled") - && cfg.getBoolean("schema-changes.enabled")) { - enableSchemaChange = true; - break; - } - } - // add schema evolution functionality to cdc source - DataStream evolvedStream = null; - if (isStreaming - && enableSchemaChange - && sourceTableInfo.getSource() instanceof SupportSchemaEvolution) { - evolvedStream = - sourceStream.transform( - "schema-evolution", - TypeInformation.of(SeaTunnelRow.class), - new SchemaOperator( - jobContext.getJobId(), - (SupportSchemaEvolution) sourceTableInfo.getSource(), - pluginConfig)); - } - - if (evolvedStream != null) { - sources.add( - new DataStreamTableInfo( - evolvedStream, - sourceTableInfo.getCatalogTables(), - ReadonlyConfig.fromConfig(pluginConfig).get(PLUGIN_OUTPUT))); - } else { - sources.add( - new DataStreamTableInfo( - sourceStream, - sourceTableInfo.getCatalogTables(), - ReadonlyConfig.fromConfig(pluginConfig).get(PLUGIN_OUTPUT))); - } - } - return sources; - } - - @Override - protected List initializePlugins( - List jarPaths, List pluginConfigs) { - SeaTunnelFactoryDiscovery factoryDiscovery = - new SeaTunnelFactoryDiscovery(TableSourceFactory.class, ADD_URL_TO_CLASSLOADER); - SeaTunnelSourcePluginDiscovery sourcePluginDiscovery = - new SeaTunnelSourcePluginDiscovery(ADD_URL_TO_CLASSLOADER); - Function fallbackCreateSource = - sourcePluginDiscovery::createPluginInstance; - - List sources = new ArrayList<>(); - Set jars = new HashSet<>(); - for (Config sourceConfig : pluginConfigs) { - PluginIdentifier pluginIdentifier = - PluginIdentifier.of( - EngineType.SEATUNNEL.getEngine(), - PluginType.SOURCE.getType(), - sourceConfig.getString(PLUGIN_NAME.key())); - jars.addAll( - sourcePluginDiscovery.getPluginJarAndDependencyPaths( - Lists.newArrayList(pluginIdentifier))); - - Tuple2, List> source = - FactoryUtil.createAndPrepareSource( - ReadonlyConfig.fromConfig(sourceConfig), - classLoader, - pluginIdentifier.getPluginName(), - fallbackCreateSource, - (TableSourceFactory) - factoryDiscovery - .createOptionalPluginInstance(pluginIdentifier) - .orElse(null), - null); - - source._1().setJobContext(jobContext); - ensureJobModeMatch(jobContext, source._1()); - - sources.add(new SourceTableInfo(source._1(), source._2())); - } - jarPaths.addAll(jars); - return sources; - } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_flink_schema_change.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_flink_schema_change.conf index 9dd595f54785..7ffbf1c1c614 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_flink_schema_change.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_flink_schema_change.conf @@ -20,7 +20,7 @@ env { # You can set engine configuration here - parallelism = 1 + parallelism = 5 job.mode = "STREAMING" checkpoint.interval = 5000 read_limit.bytes_per_second=7000000 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_flink_schema_change_exactly_once.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_flink_schema_change_exactly_once.conf index 09c88f312068..69816a41b9b6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_flink_schema_change_exactly_once.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_flink_schema_change_exactly_once.conf @@ -20,7 +20,7 @@ env { # You can set engine configuration here - parallelism = 1 + parallelism = 5 job.mode = "STREAMING" checkpoint.interval = 5000 read_limit.bytes_per_second=7000000 diff --git a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-13/src/main/java/org/apache/seatunnel/translation/flink/schema/SchemaOperator13.java b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-13/src/main/java/org/apache/seatunnel/translation/flink/schema/SchemaOperator13.java new file mode 100644 index 000000000000..3430c46b6188 --- /dev/null +++ b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-13/src/main/java/org/apache/seatunnel/translation/flink/schema/SchemaOperator13.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.translation.flink.schema; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import org.apache.seatunnel.api.source.SupportSchemaEvolution; + +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; + +import lombok.extern.slf4j.Slf4j; + +/** + * Flink 1.13-specific extension of {@link SchemaOperator} that resolves two issues present when the + * fallback timer is placed in the common module: + * + *
    + *
  1. No reflection: {@link ProcessingTimeService} and {@link ProcessingTimeCallback} are + * imported directly as strongly-typed Flink 1.13 APIs. There is no risk of silent breakage + * from method renames in future Flink versions. + *
  2. No dead flag path: the timer callback fires on the Flink task thread via + * {@code ProcessingTimeService.registerTimer}, so {@link #handleFallbackTimerOnTaskThread()} + * is always reachable even when no more source data arrives and {@code processElement} is + * never called again. This is the exact scenario this workaround targets on Flink 1.13. + *
+ * + *

The base {@link SchemaOperator} carries none of this timer infrastructure; Flink 1.15 and + * later use that base class directly because checkpointing behaves correctly there. + */ +@Slf4j +public class SchemaOperator13 extends SchemaOperator { + + /** + * Guards against double-registration. All accesses happen on the Flink task thread + * (processElement, timer callbacks, notifyCheckpointComplete) + */ + private boolean fallbackTimerPending = false; + + public SchemaOperator13(String jobId, SupportSchemaEvolution source, Config pluginConfig) { + super(jobId, source, pluginConfig); + } + + /** + * Registers a processing-time timer that will call {@link #handleFallbackTimerOnTaskThread()} + * on the Flink task thread after {@link #CHECKPOINT_STALL_TIMEOUT_MS} milliseconds. + * + *

Using {@link ProcessingTimeService#registerTimer} instead of a background {@code + * ScheduledExecutorService} achieves two goals: + * + *

    + *
  • The callback is delivered on the task thread, so {@code output.collect} and operator + * state are accessed safely without additional synchronisation. + *
  • No daemon thread overhead is introduced for Flink 1.14+ users who use the common + * module's no-op default. + *
+ * + *

If a timer is already pending this call is a no-op to prevent duplicate firings. + */ + @Override + protected void scheduleFallbackTimer() { + if (fallbackTimerPending) { + return; + } + fallbackTimerPending = true; + + ProcessingTimeService pts = getProcessingTimeService(); + long fireAt = pts.getCurrentProcessingTime() + CHECKPOINT_STALL_TIMEOUT_MS; + + pts.registerTimer( + fireAt, + (ProcessingTimeCallback) + timestamp -> { + fallbackTimerPending = false; + try { + handleFallbackTimerOnTaskThread(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error( + "Fallback schema-change timer interrupted for job {}", + jobId, + e); + } + }); + + log.debug( + "Registered Flink processing-time fallback timer to fire in {}ms for job {}", + CHECKPOINT_STALL_TIMEOUT_MS, + jobId); + } +} diff --git a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/schema/SchemaOperator.java b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/schema/SchemaOperator.java index 0f993dcda0ff..0ce26072b3fa 100644 --- a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/schema/SchemaOperator.java +++ b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/schema/SchemaOperator.java @@ -1,12 +1,12 @@ /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -54,7 +54,7 @@ /** * Operator placed after the source to handle schema evolution. * - *

schema change events are NOT processed synchronously in {@link #processElement}. Instead, they + *

Schema change events are NOT processed synchronously in {@link #processElement}. Instead, they * are buffered and deferred until an additional checkpoint cycle has completed after the first * checkpoint that observed the pending DDL. This wait ensures that when the sink executes ALTER * TABLE, all XA transactions from prior checkpoint cycles have been fully committed by the {@code @@ -64,6 +64,13 @@ * *

Per checkpoint cycle, at most ONE schema change is applied. If multiple DDLs arrive between * two checkpoints, they are processed across successive checkpoint cycles. + * + *

Flink 1.13 cannot continue checkpointing after some source subtasks have finished. When + * high-parallelism CDC jobs hit that condition, pending schema changes would otherwise stay blocked + * forever. Subclasses may override {@link #scheduleFallbackTimer()} to register a version-specific + * timer that detects the stall and re-enters the task thread via {@link + * #handleFallbackTimerOnTaskThread()} so the deferred DDL can still be applied safely. The base + * implementation is a no-op, keeping the common module free of version-specific overhead. */ @Slf4j public class SchemaOperator extends AbstractStreamOperator @@ -73,8 +80,14 @@ public class SchemaOperator extends AbstractStreamOperator private static final long SCHEMA_CHANGE_TIMEOUT_MS = 300_000L; private static final int CHECKPOINT_WAIT_ROUNDS = 1; + /** Exposed to subclasses so version-specific fallback timers can use the same threshold. */ + protected static final long CHECKPOINT_STALL_TIMEOUT_MS = 15_000L; + private final Map localSchemaState; - private String jobId; + + /** Exposed to subclasses for logging only. */ + protected String jobId; + private final SupportSchemaEvolution source; private final Config pluginConfig; private volatile Long lastProcessedEventTime; @@ -83,6 +96,13 @@ public class SchemaOperator extends AbstractStreamOperator private volatile boolean schemaChangePending = false; private long firstSeenCheckpointId = -1L; + /** + * Timestamp of the most recently completed checkpoint. Updated in {@link + * #notifyCheckpointComplete} and read by {@link #handleFallbackTimerOnTaskThread} to detect + * whether checkpoints have stalled. + */ + protected volatile long lastCheckpointCompletedMs = -1L; + private transient ListState localSchemaStateStore; private transient ListState lastProcessedEventTimeState; private transient ListState schemaChangePendingState; @@ -116,7 +136,8 @@ public void open() throws Exception { } @Override - public void processElement(StreamRecord streamRecord) { + public void processElement(StreamRecord streamRecord) + throws InterruptedException { SeaTunnelRow element = streamRecord.getValue(); if (!isSchemaEvolutionEnabled(pluginConfig)) { @@ -129,7 +150,7 @@ public void processElement(StreamRecord streamRecord) { && element.getOptions() != null) { Object object = element.getOptions().get("schema_change_event"); if (object instanceof SchemaChangeEvent) { - handleSchemaChangeDetected((SchemaChangeEvent) object, streamRecord.getTimestamp()); + handleSchemaChangeDetected((SchemaChangeEvent) object); return; } } @@ -143,7 +164,7 @@ public void processElement(StreamRecord streamRecord) { output.collect(streamRecord); } - private void handleSchemaChangeDetected(SchemaChangeEvent event, long timestamp) { + private void handleSchemaChangeDetected(SchemaChangeEvent event) { List supportedTypes = source.supports(); if (supportedTypes == null || supportedTypes.isEmpty()) { log.info("Source does not support any schema change types, skipping"); @@ -166,6 +187,7 @@ private void handleSchemaChangeDetected(SchemaChangeEvent event, long timestamp) pendingQueue.add(BufferedRecord.schemaChange(event)); schemaChangePending = true; + scheduleFallbackTimer(); } private void enqueueDataRecord(SeaTunnelRow row, long timestamp) { @@ -197,12 +219,12 @@ private TableIdentifier getPendingSchemaTableIdentifier() { * ensure safety: * *

    - *
  • first time seeing the DDL: record {@link #firstSeenCheckpointId} but do NOT - * broadcast the DDL yet. At this point the {@code FlinkGlobalCommitter} may still be - * running {@code XA COMMIT} for this checkpoint's prepared transactions, holding MDL - * locks on the sink table. - *
  • {@code checkpointId >= firstSeenCheckpointId + CHECKPOINT_WAIT_ROUNDS} : the XA - * COMMIT from the earlier checkpoint cycle is guaranteed to have finished (at least one + *
  • First time seeing the DDL: record {@link #firstSeenCheckpointId} but do NOT broadcast + * the DDL yet. At this point the {@code FlinkGlobalCommitter} may still be running {@code + * XA COMMIT} for this checkpoint's prepared transactions, holding MDL locks on the sink + * table. + *
  • {@code checkpointId >= firstSeenCheckpointId + CHECKPOINT_WAIT_ROUNDS}: the XA COMMIT + * from the earlier checkpoint cycle is guaranteed to have finished (at least one * additional checkpoint cycle has completed, which implies the committer ran). The sink's * ALTER TABLE will not encounter MDL lock, it is now safe to broadcast the DDL. *
@@ -210,20 +232,14 @@ private TableIdentifier getPendingSchemaTableIdentifier() { @Override public void notifyCheckpointComplete(long checkpointId) throws Exception { super.notifyCheckpointComplete(checkpointId); + lastCheckpointCompletedMs = System.currentTimeMillis(); if (!schemaChangePending || pendingQueue.isEmpty()) { return; } - BufferedRecord head = pendingQueue.peek(); - while (head != null && !head.isSchemaChange) { - output.collect(new StreamRecord<>(head.row, head.timestamp)); - pendingQueue.poll(); - head = pendingQueue.peek(); - } + BufferedRecord head = advancePastDataRecords(); if (head == null) { - schemaChangePending = false; - firstSeenCheckpointId = -1L; return; } @@ -266,6 +282,96 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { tableId, eventTime); + applyNextPendingSchemaChange(); + } + + /** + * Handles a checkpoint-stall fallback on the task thread. Must be called from the Flink task + * thread (e.g. via {@code ProcessingTimeService.registerTimer} callback) to keep {@code + * output.collect} and operator state accesses thread-safe. + * + *

Safety fence: the DDL is applied only when at least one checkpoint has already completed + * after the schema event ({@code firstSeenCheckpointId >= 0}). This preserves the guarantee + * that XA transactions from the earlier checkpoint cycle have finished before ALTER TABLE runs. + * If that fence has not been crossed yet, the fallback reschedules itself by calling {@link + * #scheduleFallbackTimer()} and returns without applying anything. + */ + protected void handleFallbackTimerOnTaskThread() throws InterruptedException { + if (!schemaChangePending || pendingQueue.isEmpty()) { + return; + } + + if (lastCheckpointCompletedMs > 0 + && System.currentTimeMillis() - lastCheckpointCompletedMs + < CHECKPOINT_STALL_TIMEOUT_MS) { + scheduleFallbackTimer(); + return; + } + + BufferedRecord head = advancePastDataRecords(); + if (head == null) { + return; + } + + if (firstSeenCheckpointId < 0) { + log.info( + "Fallback timer fired but no checkpoint has completed after schema event " + + "for table {} (epoch {}). Rescheduling fallback to preserve " + + "checkpoint-completion safety fence.", + head.schemaEvent.tableIdentifier(), + head.schemaEvent.getCreatedTime()); + scheduleFallbackTimer(); + return; + } + + log.warn( + "Checkpoint stall detected after first post-DDL checkpoint {}. " + + "Applying deferred DDL for table {} (epoch {}) via fallback timer. " + + "Note: data committed via normal Flink checkpoint lifecycle may be " + + "delayed until checkpoints resume.", + firstSeenCheckpointId, + head.schemaEvent.tableIdentifier(), + head.schemaEvent.getCreatedTime()); + + applyNextPendingSchemaChange(); + } + + /** + * Schedules a fallback timer that will call {@link #handleFallbackTimerOnTaskThread()} if + * checkpoints stall before the pending schema change can be applied. + * + *

The base implementation is a no-op: version-specific subclasses (e.g. {@code + * SchemaOperator13}) override this to register a timer via {@code ProcessingTimeService}, + * keeping the common module free of version-specific timer infrastructure and reflection. + */ + protected void scheduleFallbackTimer() { + // no-op by default; overridden in version-specific subclasses + } + + private BufferedRecord advancePastDataRecords() { + BufferedRecord head = pendingQueue.peek(); + while (head != null && !head.isSchemaChange) { + output.collect(new StreamRecord<>(head.row, head.timestamp)); + pendingQueue.poll(); + head = pendingQueue.peek(); + } + if (head == null) { + schemaChangePending = false; + firstSeenCheckpointId = -1L; + } + return head; + } + + private void applyNextPendingSchemaChange() throws InterruptedException { + BufferedRecord head = pendingQueue.peek(); + if (head == null || !head.isSchemaChange) { + return; + } + + SchemaChangeEvent event = head.schemaEvent; + TableIdentifier tableId = event.tableIdentifier(); + long eventTime = event.getCreatedTime(); + if (lastProcessedEventTime != null && eventTime <= lastProcessedEventTime) { log.warn( "Skipping outdated schema change event (epoch {} <= last processed {})", @@ -294,7 +400,6 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { "Schema change for table {} (epoch {}) confirmed by all sink subtasks.", tableId, eventTime); - pendingQueue.poll(); firstSeenCheckpointId = -1L; @@ -323,6 +428,7 @@ private void drainDataUntilNextSchemaChange() { "Released {} buffered data records. Another schema change pending, " + "waiting for next checkpoint.", released); + scheduleFallbackTimer(); return; } pendingQueue.poll(); @@ -474,11 +580,6 @@ private void sendSchemaChangeEventToDownstream(SchemaChangeEvent schemaChangeEve output.collect(new StreamRecord<>(broadcastRow)); } - @Override - public void close() throws Exception { - super.close(); - } - static class BufferedRecord { final boolean isSchemaChange; final SeaTunnelRow row; diff --git a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/schema/coordinator/LocalSchemaCoordinator.java b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/schema/coordinator/LocalSchemaCoordinator.java index 8d99f433a391..3cd96d43dc6f 100644 --- a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/schema/coordinator/LocalSchemaCoordinator.java +++ b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/schema/coordinator/LocalSchemaCoordinator.java @@ -41,6 +41,12 @@ * Local coordinator for schema change synchronization. This coordinator only manages temporary * communication between SchemaOperator and sink subtasks. All persistent state is managed by * BroadcastSchemaSinkOperator in Flink State. + * + *

Schema changes (DDL like ALTER TABLE) are database-level operations that only need to be + * executed once. In Flink's parallel execution model, SchemaOperator sends schema change events via + * output.collect() which routes to only ONE downstream subtask based on partitioning. Therefore, + * this coordinator completes schema change requests when ANY single subtask successfully applies + * the change, rather than waiting for all subtasks. */ @Slf4j public class LocalSchemaCoordinator { @@ -138,24 +144,27 @@ public void unregisterSinkSubtask(int subtaskId) { remaining, jobId); + // Check if any pending requests can now be completed + // (Since we only need 1 ACK for DDL, this typically won't change anything, + // but we keep it for edge cases where all subtasks close before any ACK) for (Map.Entry entry : pendingRequests.entrySet()) { String key = entry.getKey(); TimestampedPendingRequest request = entry.getValue(); Set applied = receivedAcks.get(key); - int expectedActive = Math.max(remaining, 1); - if (applied != null && applied.size() >= expectedActive) { + // If we already have at least 1 ACK, complete the request + if (applied != null && !applied.isEmpty()) { if (request.appliedPhaseCompleteAtomic.compareAndSet(false, true)) { - boolean allSuccess = request.allSuccess.get(); - request.future.complete(allSuccess); + boolean success = request.allSuccess.get(); + request.future.complete(success); log.info( - "After subtask {} unregistered, all {} active subtasks have applied " - + "schema change for table {} (epoch {}). Completing request with result: {}", + "After subtask {} unregistered, completing schema change request for " + + "table {} (epoch {}) with {} ACK(s). Result: {}", subtaskId, - expectedActive, request.tableId, request.epoch, - allSuccess); + applied.size(), + success); } } } @@ -218,8 +227,8 @@ public enum SchemaProcessingStatus { public boolean requestSchemaChange(TableIdentifier tableId, long epoch, long timeoutMs) throws InterruptedException, SchemaCoordinationException { String key = tableId.toString() + "#" + epoch; - int expectedAcks = activeSinkSubtasks.size(); - if (expectedAcks == 0) { + int totalSubtasks = activeSinkSubtasks.size(); + if (totalSubtasks == 0) { log.warn( "No active sink subtasks. Cannot coordinate schema change for table {} (epoch {}). " + "Assuming success to avoid deadlock.", @@ -227,11 +236,25 @@ public boolean requestSchemaChange(TableIdentifier tableId, long epoch, long tim epoch); return true; } + // Schema changes (DDL) are database-level operations that only need to execute once. + // Due to Flink's partitioning, only one subtask receives the schema change event, + // so we only need 1 ACK to confirm the DDL was applied successfully. + // + // Precondition: sink subtasks that do NOT receive the schema-change event directly + // (because Flink's partitioning routed it elsewhere) must have their local schema + // view refreshed through BroadcastSchemaSinkOperator's broadcast/state path. + // If that broadcast path is incomplete, those subtasks will silently apply the old + // schema to new-format rows — a data-corruption risk. Any change to the broadcast + // path must preserve this invariant, and a multi-table (≥2 tables, parallelism ≥2) + // E2E test should guard it so regressions are caught immediately. + int expectedAcks = 1; log.info( - "Requesting schema change for table {} (epoch {}). Waiting for all {} sink subtasks to apply after checkpoint completion.", + "Requesting schema change for table {} (epoch {}). Waiting for at least {} of {} " + + "sink subtasks to apply the DDL (database-level operation).", tableId, epoch, - expectedAcks); + expectedAcks, + totalSubtasks); long now = System.currentTimeMillis(); TimestampedPendingRequest request = @@ -312,31 +335,42 @@ public void notifySchemaChangeApplied( } appliedSubtasks.add(subtaskId); - int currentExpected = Math.min(request.expectedAcks, activeSinkSubtasks.size()); - currentExpected = Math.max(currentExpected, 1); + // Schema changes only need 1 successful application since they're database-level operations + int requiredAcks = request.expectedAcks; // This is now 1 log.info( - "Subtask {} applied schema change for table {} (epoch {}), success: {}. {}/{} subtasks applied.", + "Subtask {} applied schema change for table {} (epoch {}), success: {}. " + + "{} subtask(s) applied (need {} for completion).", subtaskId, tableId, epoch, success, appliedSubtasks.size(), - currentExpected); + requiredAcks); if (!success) { request.allSuccess.set(false); } - if (appliedSubtasks.size() >= currentExpected) { + // Complete when we have at least 1 successful ACK (DDL only needs to run once) + if (appliedSubtasks.size() >= requiredAcks && success) { if (request.appliedPhaseCompleteAtomic.compareAndSet(false, true)) { - boolean allSuccess = request.allSuccess.get(); - request.future.complete(allSuccess); + request.future.complete(true); log.info( - "All {} active subtasks have applied schema change for table {} (epoch {}). Completing request with result: {}", - currentExpected, + "Schema change for table {} (epoch {}) successfully applied by subtask {}. " + + "DDL execution complete (database-level operation).", + tableId, + epoch, + subtaskId); + } + } else if (appliedSubtasks.size() >= requiredAcks && !success) { + // If the only ACK we got was a failure, complete with failure + if (request.appliedPhaseCompleteAtomic.compareAndSet(false, true)) { + request.future.complete(false); + log.error( + "Schema change for table {} (epoch {}) failed on subtask {}.", tableId, epoch, - allSuccess); + subtaskId); } } } diff --git a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/source/FlinkSourceReader.java b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/source/FlinkSourceReader.java index 4c2a7b6d2e50..51bc49ad4cff 100644 --- a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/source/FlinkSourceReader.java +++ b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/source/FlinkSourceReader.java @@ -49,6 +49,8 @@ public class FlinkSourceReader implements SourceReader> { + private static final String SOURCE_KEEP_ALIVE_CONFIG = "schema-changes.source-keep-alive"; + private final Logger LOGGER = LoggerFactory.getLogger(FlinkSourceReader.class); private final org.apache.seatunnel.api.source.SourceReader sourceReader; @@ -65,6 +67,8 @@ public class FlinkSourceReader private final ScheduledExecutorService scheduledExecutor; + private final boolean sourceKeepAliveEnabled; + public FlinkSourceReader( org.apache.seatunnel.api.source.SourceReader sourceReader, org.apache.seatunnel.api.source.SourceReader.Context context, @@ -81,6 +85,9 @@ public FlinkSourceReader( this.sourceReader = sourceReader; this.context = context; this.flinkRowCollector = new FlinkRowCollector(envConfig, context.getMetricsContext()); + this.sourceKeepAliveEnabled = + envConfig.hasPath(SOURCE_KEEP_ALIVE_CONFIG) + && envConfig.getBoolean(SOURCE_KEEP_ALIVE_CONFIG); } @Override @@ -108,8 +115,11 @@ public InputStatus pollNext(ReaderOutput output) throws Exception return InputStatus.NOTHING_AVAILABLE; } } else { - // reduce CPU idle - Thread.sleep(DEFAULT_WAIT_TIME_MILLIS); + if (sourceKeepAliveEnabled) { + // Flink 1.13 requires idle source subtasks to stay alive so checkpoints continue. + Thread.sleep(DEFAULT_WAIT_TIME_MILLIS); + return InputStatus.NOTHING_AVAILABLE; + } } return inputStatus; } @@ -132,6 +142,12 @@ public CompletableFuture isAvailable() { @Override public void addSplits(List> splits) { + if (!splits.isEmpty() && context instanceof FlinkSourceReaderContext) { + if (sourceKeepAliveEnabled) { + ((FlinkSourceReaderContext) context).resetNoMoreElementEvent(); + inputStatus = InputStatus.MORE_AVAILABLE; + } + } sourceReader.addSplits( splits.stream().map(SplitWrapper::getSourceSplit).collect(Collectors.toList())); } @@ -144,7 +160,8 @@ public void notifyNoMoreSplits() { @Override public void handleSourceEvents(SourceEvent sourceEvent) { if (sourceEvent instanceof NoMoreElementEvent) { - inputStatus = InputStatus.END_OF_INPUT; + inputStatus = + sourceKeepAliveEnabled ? InputStatus.MORE_AVAILABLE : InputStatus.END_OF_INPUT; } if (sourceEvent instanceof SourceEventWrapper) { sourceReader.handleSourceEvent((((SourceEventWrapper) sourceEvent).getSourceEvent())); diff --git a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/source/FlinkSourceReaderContext.java b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/source/FlinkSourceReaderContext.java index 2b20e0d4047f..458b240d4d28 100644 --- a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/source/FlinkSourceReaderContext.java +++ b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/main/java/org/apache/seatunnel/translation/flink/source/FlinkSourceReaderContext.java @@ -101,6 +101,10 @@ public boolean isSendNoMoreElementEvent() { return isSendNoMoreElementEvent.get(); } + public void resetNoMoreElementEvent() { + isSendNoMoreElementEvent.set(false); + } + @Override public EventListener getEventListener() { return eventListener; diff --git a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/test/java/org/apache/seatunnel/translation/flink/schema/SchemaOperatorTest.java b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/test/java/org/apache/seatunnel/translation/flink/schema/SchemaOperatorTest.java index fd3b7d212616..f1873556963c 100644 --- a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/test/java/org/apache/seatunnel/translation/flink/schema/SchemaOperatorTest.java +++ b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/test/java/org/apache/seatunnel/translation/flink/schema/SchemaOperatorTest.java @@ -176,6 +176,76 @@ void testCoordinationFailureKeepsBufferedRecordsBlocked() throws Exception { assertTrue(pendingQueue.peek().isSchemaChange); } + /** + * Verifies that {@link SchemaOperator#handleFallbackTimerOnTaskThread()} correctly respects the + * checkpoint-completion safety fence even when called from a stall-detection timer. + * + *

The test invokes the handler directly (as if a processing-time timer fired) to keep the + * unit test independent of Flink's timer infrastructure. In production, the handler is called + * by {@link SchemaOperator13#scheduleFallbackTimer()} via {@code + * ProcessingTimeService.registerTimer}. + * + *

The base {@link SchemaOperator#scheduleFallbackTimer()} is a no-op; this test verifies + * only the handler logic, not the scheduling mechanism. + */ + @Test + void testFallbackTimerRespectsCheckpointSafetyFence() throws Exception { + LocalSchemaCoordinator coordinator = Mockito.mock(LocalSchemaCoordinator.class); + Mockito.when( + coordinator.requestSchemaChange( + Mockito.any(), Mockito.anyLong(), Mockito.anyLong())) + .thenReturn(true); + + OperatorTestContext context = createOperator(false); + setField(context.operator, "coordinator", coordinator); + + AlterTableAddColumnEvent event = createSchemaChangeEvent(); + SeaTunnelRow row = createDataRow("row-released-after-fallback"); + + context.operator.processElement(new StreamRecord<>(createSchemaRow(event), 400L)); + context.operator.processElement(new StreamRecord<>(row, 401L)); + + // Simulate timer firing before any checkpoint has completed (firstSeenCheckpointId < 0). + // The handler must NOT apply the DDL — it must call scheduleFallbackTimer() to wait for + // the checkpoint-completion safety fence (guards XA/MDL conflicts). + invokeNoArgMethod(context.operator, "handleFallbackTimerOnTaskThread"); + + assertTrue(context.output.records.isEmpty()); + assertTrue(getBooleanField(context.operator, "schemaChangePending")); + assertEquals(2, getPendingQueue(context.operator).size()); + assertEquals(-1L, getLongField(context.operator, "firstSeenCheckpointId")); + Mockito.verifyNoInteractions(coordinator); + + // Complete the first post-DDL checkpoint — sets firstSeenCheckpointId, not yet safe to + // apply (need one additional round, so notifyCheckpointComplete stops here). + context.operator.notifyCheckpointComplete(40L); + + assertTrue(context.output.records.isEmpty()); + assertEquals(40L, getLongField(context.operator, "firstSeenCheckpointId")); + assertTrue(getBooleanField(context.operator, "schemaChangePending")); + Mockito.verifyNoInteractions(coordinator); + + // Simulate checkpoint stall: move lastCheckpointCompletedMs into the past beyond + // CHECKPOINT_STALL_TIMEOUT_MS (15 s). This mirrors the Flink 1.13 behaviour where + // high-parallelism CDC jobs stop checkpointing after some source subtasks finish. + setField( + context.operator, + "lastCheckpointCompletedMs", + System.currentTimeMillis() - 20_000L); + + // Simulate timer firing again. firstSeenCheckpointId >= 0 and checkpoint has stalled, + // so the safety fence is satisfied — the DDL can now be applied. + invokeNoArgMethod(context.operator, "handleFallbackTimerOnTaskThread"); + + assertEquals(2, context.output.records.size()); + assertSchemaBroadcast(context.output.records.get(0), event); + assertEquals(row, context.output.records.get(1).getValue()); + assertFalse(getBooleanField(context.operator, "schemaChangePending")); + assertTrue(getPendingQueue(context.operator).isEmpty()); + Mockito.verify(coordinator) + .requestSchemaChange(event.tableIdentifier(), event.getCreatedTime(), 300_000L); + } + private static OperatorTestContext createOperator(boolean restored) throws Exception { return createOperator(new OperatorStateStoreStub(), restored); } @@ -285,6 +355,12 @@ private static void setField(Object target, Class owner, String fieldName, Ob field.set(target, value); } + private static Object invokeNoArgMethod(Object target, String methodName) throws Exception { + java.lang.reflect.Method method = target.getClass().getDeclaredMethod(methodName); + method.setAccessible(true); + return method.invoke(target); + } + private static Field findField(Class type, String fieldName) throws NoSuchFieldException { Class current = type; while (current != null) { diff --git a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/test/java/org/apache/seatunnel/translation/flink/sink/FlinkSinkWriterTest.java b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/test/java/org/apache/seatunnel/translation/flink/sink/FlinkSinkWriterTest.java index 50be1a39d279..943d48b43d8f 100644 --- a/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/test/java/org/apache/seatunnel/translation/flink/sink/FlinkSinkWriterTest.java +++ b/seatunnel-translation/seatunnel-translation-flink/seatunnel-translation-flink-common/src/test/java/org/apache/seatunnel/translation/flink/sink/FlinkSinkWriterTest.java @@ -20,6 +20,10 @@ import org.apache.seatunnel.api.common.metrics.MetricsContext; import org.apache.seatunnel.api.event.EventListener; import org.apache.seatunnel.api.sink.SinkWriter; +import org.apache.seatunnel.api.sink.SupportSchemaEvolutionSinkWriter; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.junit.jupiter.api.Assertions; @@ -28,7 +32,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; class FlinkSinkWriterTest { @@ -83,13 +89,54 @@ void testSnapshotStateWithoutPrepareCommitFallsBack() throws Exception { Assertions.assertEquals("state-3", states.get(0).getState()); } + @Test + void testSchemaChangeEventDoesNotForceCommit() throws Exception { + SchemaAwareRecordingSinkWriter delegate = new SchemaAwareRecordingSinkWriter(); + RecordingContext context = new RecordingContext(); + + FlinkSinkWriter flinkSinkWriter = + new FlinkSinkWriter<>(delegate, 7L, context); + + AlterTableAddColumnEvent event = + AlterTableAddColumnEvent.add( + TableIdentifier.of("catalog", "database", "table"), + PhysicalColumn.of( + "added_col", + org.apache.seatunnel.api.table.type.BasicType.STRING_TYPE, + 64L, + true, + null, + null)); + event.setJobId("job-under-test"); + SeaTunnelRow schemaEvent = new SeaTunnelRow(0); + Map options = new LinkedHashMap<>(); + options.put("schema_change_event", event); + options.put("schema_subtask_id", 0L); + schemaEvent.setOptions(options); + flinkSinkWriter.write(schemaEvent, null); + + SeaTunnelRow row = new SeaTunnelRow(1); + row.setField(0, "value"); + flinkSinkWriter.write(row, null); + + // Schema change should apply without forcing commit - commits happen via normal Flink + // lifecycle + Assertions.assertEquals(1, delegate.writtenRows.size()); + Assertions.assertEquals(Collections.emptyList(), delegate.prepareCommitCalls); + Assertions.assertEquals(1, delegate.appliedSchemaChanges.size()); + Assertions.assertEquals(event, delegate.appliedSchemaChanges.get(0)); + } + private static class RecordingSinkWriter implements SinkWriter { - private final List prepareCommitCalls = new ArrayList<>(); - private final List snapshotCalls = new ArrayList<>(); + protected final List prepareCommitCalls = new ArrayList<>(); + protected final List snapshotCalls = new ArrayList<>(); + protected final List writtenRows = new ArrayList<>(); @Override - public void write(SeaTunnelRow element) throws IOException {} + public void write(SeaTunnelRow element) throws IOException { + writtenRows.add(element); + } @Override public Optional prepareCommit() { @@ -116,6 +163,19 @@ public void abortPrepare() {} public void close() throws IOException {} } + private static class SchemaAwareRecordingSinkWriter extends RecordingSinkWriter + implements SupportSchemaEvolutionSinkWriter { + + private final List + appliedSchemaChanges = new ArrayList<>(); + + @Override + public void applySchemaChange( + org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent event) { + appliedSchemaChanges.add(event); + } + } + private static class RecordingContext implements SinkWriter.Context { @Override From 32f4ed380898e009b6725035894a9c596ebc850e Mon Sep 17 00:00:00 2001 From: Jast Date: Sat, 13 Jun 2026 19:38:01 +0800 Subject: [PATCH 013/375] [Fix][Connector-V2] Support SFTP keyfile option (#11078) --- docs/en/connectors/sink/SftpFile.md | 9 ++++-- docs/en/connectors/source/SftpFile.md | 3 +- docs/zh/connectors/sink/SftpFile.md | 11 +++++-- docs/zh/connectors/source/SftpFile.md | 3 +- .../seatunnel/file/sftp/config/SftpConf.java | 15 ++++++--- .../file/sftp/config/SftpFileBaseOptions.java | 5 +++ .../file/sftp/sink/SftpFileSinkFactory.java | 3 +- .../sftp/source/SftpFileSourceFactory.java | 1 + .../file/sftp/SftpFileFactoryTest.java | 32 ++++++++++++++++++- 9 files changed, 69 insertions(+), 13 deletions(-) diff --git a/docs/en/connectors/sink/SftpFile.md b/docs/en/connectors/sink/SftpFile.md index 683918c29908..4d6a0592a1e5 100644 --- a/docs/en/connectors/sink/SftpFile.md +++ b/docs/en/connectors/sink/SftpFile.md @@ -46,7 +46,8 @@ If you use SeaTunnel Engine, It automatically integrated the hadoop jar when you | host | string | yes | - | | | port | int | yes | - | | | user | string | yes | - | | -| password | string | yes | - | | +| password | string | no | - | Required when `keyfile` is not set. | +| keyfile | string | no | - | Private key file path used for SFTP public key authentication. | | path | string | yes | - | | | tmp_path | string | yes | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a FTP dir. | | custom_filename | boolean | no | false | Whether you need custom the filename | @@ -97,7 +98,11 @@ The target sftp user is required ### password [string] -The target sftp password is required +The target sftp password. Required when `keyfile` is not set. + +### keyfile [string] + +The private key file path used for SFTP public key authentication. ### path [string] diff --git a/docs/en/connectors/source/SftpFile.md b/docs/en/connectors/source/SftpFile.md index f32adda51038..3e95521f075a 100644 --- a/docs/en/connectors/source/SftpFile.md +++ b/docs/en/connectors/source/SftpFile.md @@ -83,7 +83,8 @@ The File does not have a specific type list, and we can indicate which SeaTunnel | host | String | Yes | - | The target sftp host is required | | port | Int | Yes | - | The target sftp port is required | | user | String | Yes | - | The target sftp username is required | -| password | String | Yes | - | The target sftp password is required | +| password | String | No | - | The target sftp password. Required when `keyfile` is not set. | +| keyfile | String | No | - | The private key file path used for SFTP public key authentication. | | path | String | Yes | - | The source file path. | | file_format_type | String | Yes | - | Please check #file_format_type below | | file_filter_pattern | String | No | - | Filter pattern, which used for filtering files. | diff --git a/docs/zh/connectors/sink/SftpFile.md b/docs/zh/connectors/sink/SftpFile.md index 7a4c791fdff5..884e5356fee7 100644 --- a/docs/zh/connectors/sink/SftpFile.md +++ b/docs/zh/connectors/sink/SftpFile.md @@ -45,7 +45,8 @@ import ChangeLog from '../changelog/connector-file-sftp.md'; | host | string | 是 | - | | | port | int | 是 | - | | | user | string | 是 | - | | -| password | string | 是 | - | | +| password | string | 否 | - | 未配置 `keyfile` 时需要配置。 | +| keyfile | string | 否 | - | 用于 SFTP 公钥认证的私钥文件路径。 | | path | string | 是 | - | | | tmp_path | string | 是 | /tmp/seatunnel | 结果文件将首先写入临时路径，然后使用`mv`将临时目录剪切到目标目录。需要一个FTP目录。 | | custom_filename | boolean | 否 | false | 是否需要自定义文件名 | @@ -94,7 +95,11 @@ import ChangeLog from '../changelog/connector-file-sftp.md'; ### password [string] -目标sftp密码，必填。 +目标sftp密码。未配置 `keyfile` 时需要配置。 + +### keyfile [string] + +用于 SFTP 公钥认证的私钥文件路径。 ### path [string] @@ -359,4 +364,4 @@ LocalFile { ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/source/SftpFile.md b/docs/zh/connectors/source/SftpFile.md index c225af5a58fc..ce71dd1cb573 100644 --- a/docs/zh/connectors/source/SftpFile.md +++ b/docs/zh/connectors/source/SftpFile.md @@ -83,7 +83,8 @@ import ChangeLog from '../changelog/connector-file-sftp.md'; | host | String | 是 | - | 目标sftp主机是必需的 | | port | Int | 是 | - | 目标sftp端口是必需的 | | user | String | 是 | - | 目标sftp用户名是必需的 | -| password | String | 是 | - | 目标sftp密码是必需的 | +| password | String | 否 | - | 目标sftp密码。未配置 `keyfile` 时需要配置。 | +| keyfile | String | 否 | - | 用于 SFTP 公钥认证的私钥文件路径。 | | path | String | 是 | - | 源文件路径。 | | file_format_type | String | 是 | - | 请查看下面的#file_format_type | | file_filter_pattern | String | 否 | - | 过滤模式，用于过滤文件。 | diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SftpConf.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SftpConf.java index 2849b5b7fa63..75d0e3e3f038 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SftpConf.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SftpConf.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf; +import org.apache.seatunnel.connectors.seatunnel.file.sftp.system.SFTPFileSystem; import java.util.HashMap; @@ -47,10 +48,16 @@ public static HadoopConf buildWithConfig(ReadonlyConfig config) { String defaultFS = String.format("sftp://%s:%s", host, port); HadoopConf hadoopConf = new SftpConf(defaultFS); HashMap sftpOptions = new HashMap<>(); - sftpOptions.put("fs.sftp.user." + host, config.get(SftpFileBaseOptions.SFTP_USER)); - sftpOptions.put( - "fs.sftp.password." + host + "." + config.get(SftpFileBaseOptions.SFTP_USER), - config.get(SftpFileBaseOptions.SFTP_PASSWORD)); + String user = config.get(SftpFileBaseOptions.SFTP_USER); + sftpOptions.put(SFTPFileSystem.FS_SFTP_USER_PREFIX + host, user); + config.getOptional(SftpFileBaseOptions.SFTP_PASSWORD) + .ifPresent( + password -> + sftpOptions.put( + SFTPFileSystem.FS_SFTP_PASSWORD_PREFIX + host + "." + user, + password)); + config.getOptional(SftpFileBaseOptions.SFTP_KEYFILE) + .ifPresent(keyfile -> sftpOptions.put(SFTPFileSystem.FS_SFTP_KEYFILE, keyfile)); hadoopConf.setExtraOptions(sftpOptions); return hadoopConf; } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SftpFileBaseOptions.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SftpFileBaseOptions.java index 5a614fbec73d..75e435c3f712 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SftpFileBaseOptions.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SftpFileBaseOptions.java @@ -27,6 +27,11 @@ public class SftpFileBaseOptions extends FileBaseOptions { .stringType() .noDefaultValue() .withDescription("SFTP server password"); + public static final Option SFTP_KEYFILE = + Options.key("keyfile") + .stringType() + .noDefaultValue() + .withDescription("SFTP private key file path"); public static final Option SFTP_USER = Options.key("user") .stringType() diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/sink/SftpFileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/sink/SftpFileSinkFactory.java index eb68f31158ee..0bfe1196dd86 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/sink/SftpFileSinkFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/sink/SftpFileSinkFactory.java @@ -52,7 +52,8 @@ public OptionRule optionRule() { .required(SftpFileSinkOptions.SFTP_HOST) .required(SftpFileSinkOptions.SFTP_PORT) .required(SftpFileSinkOptions.SFTP_USER) - .required(SftpFileSinkOptions.SFTP_PASSWORD) + .optional(SftpFileSinkOptions.SFTP_PASSWORD) + .optional(SftpFileSinkOptions.SFTP_KEYFILE) .optional(FileBaseSinkOptions.FILE_FORMAT_TYPE) .optional(FileBaseSinkOptions.SCHEMA_SAVE_MODE) .optional(FileBaseSinkOptions.DATA_SAVE_MODE) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java index 16e22ae664b9..4ce72d9eb4d1 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java @@ -51,6 +51,7 @@ public OptionRule optionRule() { .optional(SftpFileSourceOptions.SFTP_PORT) .optional(SftpFileSourceOptions.SFTP_USER) .optional(SftpFileSourceOptions.SFTP_PASSWORD) + .optional(SftpFileSourceOptions.SFTP_KEYFILE) .optional(FileBaseSourceOptions.FILE_FORMAT_TYPE) .conditional( FileBaseSourceOptions.FILE_FORMAT_TYPE, diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/SftpFileFactoryTest.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/SftpFileFactoryTest.java index 9ed5942d188b..b38f8c8794af 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/SftpFileFactoryTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/SftpFileFactoryTest.java @@ -17,17 +17,24 @@ package org.apache.seatunnel.connectors.seatunnel.file.sftp; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.Expression; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions; import org.apache.seatunnel.connectors.seatunnel.file.config.FileSyncMode; +import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf; +import org.apache.seatunnel.connectors.seatunnel.file.sftp.config.SftpConf; import org.apache.seatunnel.connectors.seatunnel.file.sftp.sink.SftpFileSinkFactory; import org.apache.seatunnel.connectors.seatunnel.file.sftp.source.SftpFileSourceFactory; +import org.apache.seatunnel.connectors.seatunnel.file.sftp.system.SFTPFileSystem; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.HashMap; +import java.util.Map; + class SftpFileFactoryTest { @Test @@ -48,6 +55,9 @@ void optionRule() { optionRule.getOptionalOptions().contains(FileBaseSourceOptions.SCAN_INTERVAL)); Assertions.assertTrue( optionRule.getOptionalOptions().contains(FileBaseSourceOptions.START_MODE)); + Assertions.assertTrue( + optionRule.getOptionalOptions().stream() + .anyMatch(option -> "keyfile".equals(option.key()))); Expression expectExpression = Expression.of(FileBaseSourceOptions.SYNC_MODE, FileSyncMode.UPDATE); @@ -60,6 +70,26 @@ void optionRule() { required.getOptions() .contains(FileBaseSourceOptions.TARGET_PATH)) .anyMatch(required -> expectExpression.equals(required.getExpression()))); - Assertions.assertNotNull((new SftpFileSinkFactory()).optionRule()); + OptionRule sinkOptionRule = (new SftpFileSinkFactory()).optionRule(); + Assertions.assertNotNull(sinkOptionRule); + Assertions.assertTrue( + sinkOptionRule.getOptionalOptions().stream() + .anyMatch(option -> "keyfile".equals(option.key()))); + } + + @Test + void buildHadoopConfWithKeyfile() { + Map configMap = new HashMap<>(); + configMap.put("host", "sftp.example.com"); + configMap.put("port", 22); + configMap.put("user", "seatunnel"); + configMap.put("password", "secret"); + configMap.put("keyfile", "/home/seatunnel/.ssh/id_rsa"); + + HadoopConf hadoopConf = SftpConf.buildWithConfig(ReadonlyConfig.fromMap(configMap)); + + Assertions.assertEquals( + "/home/seatunnel/.ssh/id_rsa", + hadoopConf.getExtraOptions().get(SFTPFileSystem.FS_SFTP_KEYFILE)); } } From fcf30486345eeb27015a7827974077ea79b65cb6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 13 Jun 2026 20:19:20 +0800 Subject: [PATCH 014/375] docs: add multi-table transform capability boundary guide (#10985) Co-authored-by: davidzollo --- ...multi-table-transform-and-join-boundary.md | 266 +++++++++++++++++ docs/sidebars.js | 3 +- ...multi-table-transform-and-join-boundary.md | 269 ++++++++++++++++++ 3 files changed, 537 insertions(+), 1 deletion(-) create mode 100644 docs/en/transforms/multi-table-transform-and-join-boundary.md create mode 100644 docs/zh/transforms/multi-table-transform-and-join-boundary.md diff --git a/docs/en/transforms/multi-table-transform-and-join-boundary.md b/docs/en/transforms/multi-table-transform-and-join-boundary.md new file mode 100644 index 000000000000..f10d52b8ebeb --- /dev/null +++ b/docs/en/transforms/multi-table-transform-and-join-boundary.md @@ -0,0 +1,266 @@ +--- +sidebar_position: 16 +--- + +# Multi-Table Transform Capability Boundary + +## Overview + +SeaTunnel's **multi-table transform** feature allows a single transform node to process multiple +tables flowing from an upstream source (typically a CDC connector) in one pipeline. This page +documents precisely what is supported, what is not, and what alternatives to use when you hit a +capability boundary. + +--- + +## 1. What Is a Multi-Table Transform? + +In a standard single-table pipeline, one Source feeds one Transform chain feeds one Sink. +In a multi-table pipeline, a single Source (e.g., MySQL-CDC) emits records from **many tables** +simultaneously, and each downstream Transform or Sink must declare which table(s) it applies to. + +``` +MySQL-CDC ──► FieldMapper (orders table) ──► Kafka Sink (orders topic) + │ + ├──► FieldMapper (users table) ──► Kafka Sink (users topic) + │ + └──► (unmatched tables pass through) ──► Elasticsearch Sink +``` + +--- + +## 2. Capability Boundary Table + +| Capability | Supported | Notes | +|---|---|---| +| Per-table field rename / map | ✅ Yes | Use `FieldMapper` with `plugin_input` and `table_match_regex` | +| Per-table column filtering | ✅ Yes | Use `Filter` with `plugin_input` and `table_match_regex` | +| Per-table type casting | ✅ Yes | Use `FieldMapper` with `define_sink_type` option | +| Per-table SQL transform (single table) | ✅ Yes | Use `SQL` transform with `plugin_input`; scope it with `table_match_regex` when needed | +| `TableMerge` — merge multiple tables into one | ✅ Yes | Tables must share compatible schema | +| `TableRename` — rename tables in the stream | ✅ Yes | Works well for routing to Sink by table name | +| Row-level filtering (filter by `rowkind`) | ✅ Yes | Use `FilterRowKind` transform | +| Cross-table SQL JOIN | ❌ Not supported | See Section 4 for alternatives | +| Aggregation across multiple CDC tables | ❌ Not supported | Aggregate downstream in OLAP engine | +| Generating new tables from a JOIN result | ❌ Not supported | Use a dedicated SQL engine | +| Applying one transform to ALL tables wildcard | ⚠️ Partial | `TableMerge` then single transform; schema must be compatible | +| Changing schema mid-stream (DDL events) | ⚠️ Limited | Depends on sink; some sinks handle schema evolution; transforms do not | +| Nested JSON field extraction per table | ✅ Yes | Use `JsonPath` transform with `plugin_input` and `table_match_regex` | + +--- + +## 3. TableMerge vs SQL Join + +These two are the most commonly confused features. + +### 3.1 `TableMerge` Transform + +`TableMerge` **merges the row streams of multiple tables into a single result table**. All +input tables must have the same (or compatible) schema. Use it to route all tables to one Sink. + +```json +{ + "plugin_name": "TableMerge", + "plugin_input": ["orders_2023", "orders_2024"], + "plugin_output": "all_orders", + "merge_by_field": true +} +``` + +**When to use**: Fan-in from multiple source tables that have the same structure (e.g., sharded +tables, multi-year partitions, or multi-database tables with identical schema). + +**When NOT to use**: When tables have different schemas that you need to correlate or enrich +from each other — that requires a JOIN. + +### 3.2 SQL JOIN (not natively supported in multi-table pipelines) + +A SQL JOIN correlates rows from two different tables based on a key. **SeaTunnel's `SQL` +transform does NOT support cross-table JOIN inside a multi-table streaming pipeline.** + +Attempting to JOIN records from two upstream tables inside a single SQL transform is not +supported and will result in a configuration error. + +**Recommended alternatives**: +- Write both tables to a shared data lake or warehouse (e.g., Hudi, Iceberg, ClickHouse), then + run the JOIN there +- Use Apache Flink with SeaTunnel's Flink connector for stateful JOIN operations +- Materialise the "dimension" table into a lookup cache (e.g., Redis, RocksDB) and use a custom + transform for enrichment + +--- + +## 4. Cross-Source JOIN Limitation + +SeaTunnel does **not** support streaming JOINs where the two input sides come from **different +sources** (e.g., joining MySQL-CDC with a PostgreSQL-CDC stream). + +| Scenario | Supported | +|---|---| +| Single-source multi-table pass-through | ✅ | +| Single-source TableMerge (same schema) | ✅ | +| Cross-source JOIN (MySQL-CDC + PG-CDC) | ❌ | +| Cross-source JOIN (CDC + JDBC batch) | ❌ | +| Same-source JOIN on two different tables | ❌ | + +**Workaround**: Write both sources to a common sink (Kafka, Iceberg, etc.) and perform the JOIN +downstream in a dedicated SQL engine (Flink, Spark, ClickHouse, etc.). + +--- + +## 5. Per-Table Transform Configuration Pattern + +When you need different transforms for different tables from the same source, declare separate +transform blocks that share the same `plugin_input` and use different `table_match_regex` +rules: + +```json +{ + "env": { + "job.name": "cdc-multi-table", + "job.mode": "STREAMING" + }, + "source": [ + { + "plugin_name": "MySQL-CDC", + "plugin_output": "cdc_stream", + "base-url": "jdbc:mysql://localhost:3306/mydb", + "username": "cdc_user", + "password": "password", + "database-names": ["mydb"], + "table-names": ["mydb.orders", "mydb.users", "mydb.products"] + } + ], + "transform": [ + { + "plugin_name": "FieldMapper", + "plugin_input": ["cdc_stream"], + "plugin_output": "orders_mapped", + "field_mapper": { "order_id": "id", "order_amount": "amount" }, + "table_match_regex": "mydb\\.orders" + }, + { + "plugin_name": "FieldMapper", + "plugin_input": ["cdc_stream"], + "plugin_output": "users_mapped", + "field_mapper": { "user_id": "id", "user_email": "email" }, + "table_match_regex": "mydb\\.users" + } + ], + "sink": [ + { + "plugin_name": "Kafka", + "plugin_input": ["orders_mapped"], + "topic": "orders" + }, + { + "plugin_name": "Kafka", + "plugin_input": ["users_mapped"], + "topic": "users" + }, + { + "plugin_name": "Kafka", + "plugin_input": ["cdc_stream"], + "topic": "products", + "table_match_regex": "mydb\\.products" + } + ] +} +``` + +--- + +## 6. Common Fields Example (Shared Schema) + +If multiple tables share a common set of fields, you can use `TableMerge` to combine them and +apply a single transform: + +```json +"transform": [ + { + "plugin_name": "TableMerge", + "plugin_input": ["cdc_stream"], + "plugin_output": "all_events", + "table_match_regex": "mydb\\.(orders|payments|refunds)" + }, + { + "plugin_name": "FieldMapper", + "plugin_input": ["all_events"], + "plugin_output": "all_events_mapped", + "field_mapper": { "created_at": "event_time", "event_type": "type" } + } +] +``` + +This works only when all three tables (`orders`, `payments`, `refunds`) share `created_at` and +`event_type` fields. If schemas differ, `TableMerge` will fail at runtime. + +--- + +## 7. EtLT Patterns with Multi-Table Transform + +**EtLT** (Extract, light-transform, Load, then Transform in the warehouse) is the recommended +pattern when SeaTunnel's transform layer cannot fulfil the full transformation requirement: + +``` +CDC Source + │ + ▼ +Light transforms (field rename, type cast, row filter) + │ + ▼ +Data Lake / Warehouse (Hudi / Iceberg / ClickHouse) + │ + ▼ +Heavy transforms (JOINs, aggregations, complex SQL) +in dbt / Flink SQL / Spark SQL +``` + +Use SeaTunnel's transform layer for: +- Field rename / filtering +- Type normalisation +- Row-level filtering +- Schema routing (different tables → different topics/tables) + +Offload to downstream: +- Cross-table JOINs +- Aggregations +- Pivot / unpivot +- Enrichment from dimension tables + +--- + +## 8. FAQ + +**Q: Can I apply one transform to ALL tables without specifying each one?** + +Not directly. Use `TableMerge` to combine tables with compatible schemas first, then apply a +single transform to the merged result. If schemas differ, you must use separate transform blocks, +typically with different `table_match_regex` rules. + +**Q: Does the `SQL` transform support `JOIN`?** + +No. The `SQL` transform only supports single-table queries (SELECT, WHERE, expressions). For +JOINs, use an external SQL engine after loading the data into a sink. + +**Q: What happens to tables that are not matched by any transform?** + +Unmatched tables continue flowing through the pipeline and can be captured by a Sink that +uses the right `plugin_input` and `table_match_regex`. + +**Q: Can I add a new table to an existing CDC pipeline without downtime?** + +This depends on the connector. MySQL-CDC supports dynamic table discovery in some configurations, +but adding a transform for a new table requires a pipeline restart. Use `stop-with-savepoint` +to minimise data loss (see [REST API v2 Reference](../engines/zeta/rest-api-v2.md)). + +--- + +## See Also + +- [TableMerge Transform Reference](table-merge.md) +- [TableRename Transform Reference](table-rename.md) +- [transform-multi-table Reference](transform-multi-table.md) +- [Multi-Table Architecture Overview](../architecture/features/multi-table.md) +- [CDC Pipeline Architecture](../architecture/cdc-pipeline-architecture.md) +- [REST API v2 Reference](../engines/zeta/rest-api-v2.md) diff --git a/docs/sidebars.js b/docs/sidebars.js index df0ce27e44de..a02d7fd42499 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -266,7 +266,8 @@ const sidebars = { "transforms/table-filter", "transforms/table-merge", "transforms/table-rename", - "transforms/transform-multi-table" + "transforms/transform-multi-table", + "transforms/multi-table-transform-and-join-boundary" ] }, { diff --git a/docs/zh/transforms/multi-table-transform-and-join-boundary.md b/docs/zh/transforms/multi-table-transform-and-join-boundary.md new file mode 100644 index 000000000000..37fcb42c6347 --- /dev/null +++ b/docs/zh/transforms/multi-table-transform-and-join-boundary.md @@ -0,0 +1,269 @@ +--- +sidebar_position: 16 +--- + +# 多表 Transform 能力边界 + +## 概述 + +SeaTunnel 的**多表 Transform**功能允许单个 Transform 节点在一条流水线中同时处理来自上游 +数据源（通常是 CDC 连接器）的多张表。本文档精确描述了哪些功能受支持、哪些不受支持，以及 +遇到能力边界时的替代方案。 + +--- + +## 1. 什么是多表 Transform？ + +在标准的单表流水线中，一个 Source 对应一条 Transform 链，再连接一个 Sink。在多表流水线中， +单个 Source（例如 MySQL-CDC）同时发送来自**多张表**的记录，下游的每个 Transform 或 Sink +需声明它适用于哪张（些）表。 + +``` +MySQL-CDC ──► FieldMapper (orders 表) ──► Kafka Sink (orders topic) + │ + ├──► FieldMapper (users 表) ──► Kafka Sink (users topic) + │ + └──► (未匹配的表直接透传) ──► Elasticsearch Sink +``` + +--- + +## 2. 能力边界一览 + +| 能力 | 是否支持 | 说明 | +|---|---|---| +| 按表字段重命名 / 映射 | ✅ 支持 | 使用带 `plugin_input` 和 `table_match_regex` 的 `FieldMapper` | +| 按表列过滤 | ✅ 支持 | 使用带 `plugin_input` 和 `table_match_regex` 的 `Filter` | +| 按表类型转换 | ✅ 支持 | 使用 `FieldMapper` 的 `define_sink_type` 选项 | +| 单表 SQL Transform | ✅ 支持 | 使用带 `plugin_input` 的 `SQL` Transform；必要时再用 `table_match_regex` 限定表范围 | +| `TableMerge`——将多表合并为一张 | ✅ 支持 | 各表需具有兼容的 Schema | +| `TableRename`——重命名流中的表 | ✅ 支持 | 适合按表名路由到 Sink | +| 行级过滤（按 `rowkind` 过滤） | ✅ 支持 | 使用 `FilterRowKind` Transform | +| 跨表 SQL JOIN | ❌ 不支持 | 参见第 4 节替代方案 | +| 多张 CDC 表的聚合 | ❌ 不支持 | 在下游 OLAP 引擎中执行聚合 | +| 通过 JOIN 生成新表 | ❌ 不支持 | 使用专用 SQL 引擎 | +| 通配符将同一 Transform 应用于所有表 | ⚠️ 部分支持 | 先用 `TableMerge` 合并，再做单 Transform；Schema 需兼容 | +| 流中 Schema 变更（DDL 事件） | ⚠️ 有限支持 | 取决于 Sink；部分 Sink 支持 Schema 演化，Transform 层不支持 | +| 按表 JSON 嵌套字段提取 | ✅ 支持 | 使用带 `plugin_input` 和 `table_match_regex` 的 `JsonPath` Transform | + +--- + +## 3. TableMerge 与 SQL Join 的区别 + +这是最容易混淆的两个特性。 + +### 3.1 `TableMerge` Transform + +`TableMerge` **将多张表的行流合并为一张结果表**。所有输入表必须具有相同（或兼容）的 +Schema。通常用于将多表路由至同一个 Sink。 + +```json +{ + "plugin_name": "TableMerge", + "plugin_input": ["orders_2023", "orders_2024"], + "plugin_output": "all_orders", + "merge_by_field": true +} +``` + +**适用场景**：多个来源表结构相同，需要合并（例如分片表、多年分区表，或多库同构表）。 + +**不适用场景**：需要关联或补全不同结构的表数据，这类场景需要 JOIN。 + +### 3.2 SQL JOIN（多表流水线中不支持） + +SQL JOIN 基于关联键将两张不同表的行进行关联。**SeaTunnel 的 `SQL` Transform 在多表流式 +流水线中不支持跨表 JOIN。** 在单个 SQL Transform 中尝试关联来自两张上游表的记录会导致 +配置报错。 + +**推荐替代方案**： + +- 将两张表写入共享数据湖或数仓（如 Hudi、Iceberg、ClickHouse），在目标端执行 JOIN +- 使用 Apache Flink 配合 SeaTunnel Flink 连接器进行有状态 JOIN +- 将“维度表”物化到查找缓存（如 Redis、RocksDB）中，通过自定义 Transform 进行数据补全 + +--- + +## 4. 跨 Source JOIN 的限制 + +SeaTunnel **不支持**两个输入侧来自**不同 Source** 的流式 JOIN（例如将 MySQL-CDC 与 +PostgreSQL-CDC 流进行 JOIN）。 + +| 场景 | 是否支持 | +|---|---| +| 单 Source 多表透传 | ✅ | +| 单 Source TableMerge（相同 Schema） | ✅ | +| 跨 Source JOIN（MySQL-CDC + PG-CDC） | ❌ | +| 跨 Source JOIN（CDC + JDBC 批量） | ❌ | +| 同 Source 两张不同表的 JOIN | ❌ | + +**解决方法**：将两个 Source 的数据写入公共 Sink（Kafka、Iceberg 等），再在专用 SQL 引擎 +（Flink、Spark、ClickHouse 等）中执行 JOIN。 + +--- + +## 5. 按表独立配置 Transform 的模式 + +当你需要对同一 Source 的不同表应用不同的 Transform 时，可声明多个共享相同 +`plugin_input`、但使用不同 `table_match_regex` 的 Transform 块： + +```json +{ + "env": { + "job.name": "cdc-multi-table", + "job.mode": "STREAMING" + }, + "source": [ + { + "plugin_name": "MySQL-CDC", + "plugin_output": "cdc_stream", + "base-url": "jdbc:mysql://localhost:3306/mydb", + "username": "cdc_user", + "password": "password", + "database-names": ["mydb"], + "table-names": ["mydb.orders", "mydb.users", "mydb.products"] + } + ], + "transform": [ + { + "plugin_name": "FieldMapper", + "plugin_input": ["cdc_stream"], + "plugin_output": "orders_mapped", + "field_mapper": { + "order_id": "id", + "order_amount": "amount" + }, + "table_match_regex": "mydb\\.orders" + }, + { + "plugin_name": "FieldMapper", + "plugin_input": ["cdc_stream"], + "plugin_output": "users_mapped", + "field_mapper": { + "user_id": "id", + "user_email": "email" + }, + "table_match_regex": "mydb\\.users" + } + ], + "sink": [ + { + "plugin_name": "Kafka", + "plugin_input": ["orders_mapped"], + "topic": "orders" + }, + { + "plugin_name": "Kafka", + "plugin_input": ["users_mapped"], + "topic": "users" + }, + { + "plugin_name": "Kafka", + "plugin_input": ["cdc_stream"], + "topic": "products", + "table_match_regex": "mydb\\.products" + } + ] +} +``` + +--- + +## 6. 公共字段示例（共享 Schema） + +如果多张表共享一组公共字段，可以先用 `TableMerge` 合并，再统一应用一个 Transform： + +```json +"transform": [ + { + "plugin_name": "TableMerge", + "plugin_input": ["cdc_stream"], + "plugin_output": "all_events", + "table_match_regex": "mydb\\.(orders|payments|refunds)" + }, + { + "plugin_name": "FieldMapper", + "plugin_input": ["all_events"], + "plugin_output": "all_events_mapped", + "field_mapper": { + "created_at": "event_time", + "event_type": "type" + } + } +] +``` + +此模式仅在三张表（`orders`、`payments`、`refunds`）都包含 `created_at` 和 +`event_type` 字段时有效。若各表 Schema 不同，`TableMerge` 会在运行时报错。 + +--- + +## 7. 多表 Transform 的 EtLT 模式 + +**EtLT**（Extract、轻量 transform、Load，再在数仓中 Transform）是当 SeaTunnel +Transform 层无法完成全量转换需求时的推荐架构模式： + +``` +CDC Source + │ + ▼ +轻量 Transform（字段重命名、类型转换、行过滤） + │ + ▼ +数据湖 / 数仓（Hudi / Iceberg / ClickHouse） + │ + ▼ +重型 Transform（JOIN、聚合、复杂 SQL） +在 dbt / Flink SQL / Spark SQL 中执行 +``` + +**适合在 SeaTunnel Transform 层完成的操作**： + +- 字段重命名 / 过滤 +- 类型标准化 +- 行级过滤 +- Schema 路由（不同表 -> 不同 Topic / 表） + +**建议下沉到下游处理的操作**： + +- 跨表 JOIN +- 聚合计算 +- Pivot / Unpivot +- 基于维度表的数据补全 + +--- + +## 8. 常见问题 + +**Q: 我能否不指定每张表，直接将一个 Transform 应用于所有表？** + +不能直接实现。可以先用 `TableMerge` 将 Schema 兼容的表合并，再对合并结果应用单一 +Transform。若各表 Schema 不同，则必须拆成多个 Transform 块，通常通过不同的 +`table_match_regex` 分别处理。 + +**Q: `SQL` Transform 支持 `JOIN` 吗？** + +不支持。`SQL` Transform 仅支持单表查询（SELECT、WHERE、表达式等）。如需执行 JOIN， +请先将数据加载至 Sink，再使用外部 SQL 引擎处理。 + +**Q: 未被任何 Transform 匹配的表会怎样？** + +未匹配的表会继续在流水线中流动，可被使用正确 `plugin_input` 和 `table_match_regex` +的 Sink 捕获。 + +**Q: 能否在不停机的情况下向现有 CDC 流水线新增表？** + +这取决于具体连接器。MySQL-CDC 在部分配置下支持动态发现新表，但为新表添加 Transform +仍需重启流水线。建议使用 `stop-with-savepoint` 将数据丢失降至最低（参见 +[REST API v2 参考文档](../engines/zeta/rest-api-v2.md)）。 + +--- + +## 参考文档 + +- [TableMerge Transform 参考](table-merge.md) +- [TableRename Transform 参考](table-rename.md) +- [transform-multi-table 参考](transform-multi-table.md) +- [多表能力概览](../architecture/features/multi-table.md) +- [CDC 流水线架构](../architecture/cdc-pipeline-architecture.md) +- [REST API v2 参考文档](../engines/zeta/rest-api-v2.md) From 30f4c98517a48ac1b39c190a51e9c36715696d37 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 13 Jun 2026 21:10:50 +0800 Subject: [PATCH 015/375] [Fix][E2E] Stabilize engine failover test and rebalance connector shards (#10949) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: davidzollo --- .github/workflows/backend.yml | 36 +++++ .../jdbc/AbstractSchemaChangeBaseIT.java | 2 +- .../e2e/connector/kafka/KafkaIT.java | 80 +++++++++-- ...kafka_to_kafka_exactly_once_streaming.conf | 6 +- .../engine/e2e/ClusterFailureNoRestoreIT.java | 38 ++++-- .../e2e/classloader/ClassLoaderITBase.java | 128 ++++++++++-------- .../SeaTunnelEngineClusterRoleTest.java | 1 + .../server/event/JobStateEventTest.java | 8 +- 8 files changed, 216 insertions(+), 83 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index e2938d3959c9..4f34cc94cb79 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -849,6 +849,13 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 1` + # Keep the longest Iceberg and HBase suites in a dedicated shard so this hosted runner + # does not lose heartbeat before the rest of the part-2 modules finish. + # Handle both first-position (no leading comma) and mid/last-position (leading comma). + run_it_modules=${run_it_modules//:connector-iceberg-e2e,/} + run_it_modules=${run_it_modules//,:connector-iceberg-e2e/} + run_it_modules=${run_it_modules//:connector-hbase-e2e,/} + run_it_modules=${run_it_modules//,:connector-hbase-e2e/} ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1008,6 +1015,35 @@ jobs: env: MAVEN_OPTS: -Xmx4096m + all-connectors-it-8: + needs: [ changes, sanity-check ] + if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' + runs-on: ${{ matrix.os }} + env: + RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} + RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} + strategy: + matrix: + java: [ '8', '11' ] + os: [ 'ubuntu-latest' ] + timeout-minutes: 210 + steps: + - uses: actions/checkout@v2 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + - name: free disk space + run: tools/github/free_disk_space.sh + - name: run connector-v2 integration test (part-8) + run: | + # These two suites dominated part-2 and triggered hosted-runner heartbeat loss. + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-iceberg-e2e,:connector-hbase-e2e -am -Pci + env: + MAVEN_OPTS: -Xmx4096m + jdbc-connectors-it-part-1: needs: [ changes, sanity-check ] if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java index 32a3d0076f1e..f95554f79d90 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java @@ -385,7 +385,7 @@ private void assertSchemaEvolutionForAddColumns(String sourceTable, String sinkT sourceDatabase.setTemplateName("add_columns").createAndInitialize(); given().pollDelay(Duration.ofSeconds(5)) .await() - .atMost(120, TimeUnit.SECONDS) + .atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kafka/KafkaIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kafka/KafkaIT.java index 6bcd10395673..f91c1827a188 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kafka/KafkaIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kafka/KafkaIT.java @@ -109,6 +109,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -121,6 +122,10 @@ @Slf4j public class KafkaIT extends TestSuiteBase implements TestResource { + private static final String EXACTLY_ONCE_SOURCE_TOPIC_VARIABLE = "sourceTopic"; + private static final String EXACTLY_ONCE_SINK_TOPIC_VARIABLE = "sinkTopic"; + private static final String EXACTLY_ONCE_CONSUMER_GROUP_VARIABLE = "consumerGroup"; + private static final String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:7.0.9"; private static final String KAFKA_HOST = "kafkaCluster"; @@ -135,6 +140,9 @@ public class KafkaIT extends TestSuiteBase implements TestResource { private List> nativeData; + /** Topics created dynamically during tests; cleaned up in {@link #tearDown()}. */ + private final List dynamicTopics = new CopyOnWriteArrayList<>(); + @BeforeAll @Override public void startUp() throws Exception { @@ -266,6 +274,14 @@ public void startUp() throws Exception { @AfterAll @Override public void tearDown() throws Exception { + if (!dynamicTopics.isEmpty()) { + try (AdminClient adminClient = createKafkaAdmin()) { + adminClient.deleteTopics(dynamicTopics).all().get(); + log.info("Deleted {} dynamic test topics", dynamicTopics.size()); + } catch (Exception e) { + log.warn("Failed to delete dynamic test topics: {}", e.getMessage()); + } + } if (producer != null) { producer.close(); } @@ -1579,11 +1595,16 @@ public void testKafkaProtobufToAssert(TestContainer container) value = {}) public void testRestoreKafkaToKafkaExactlyOnceOnStreaming(TestContainer container) throws InterruptedException, IOException { - - String producerTopic = "kafka_topic_exactly_once_1"; - String consumerTopic = "kafka_topic_exactly_once_2"; + String resourceSuffix = Long.toUnsignedString(System.nanoTime()); + String producerTopic = "kafka_topic_exactly_once_source_" + resourceSuffix; + String consumerTopic = "kafka_topic_exactly_once_sink_" + resourceSuffix; + String consumerGroup = "test_exactly_once_" + resourceSuffix; + List exactlyOnceVariables = + buildExactlyOnceStreamingVariables(producerTopic, consumerTopic, consumerGroup); + createKafkaTopic(producerTopic); + createKafkaTopic(consumerTopic); String sourceData = "Seatunnel Exactly Once Example"; - final String jobId = "18696753645413"; + final String jobId = Long.toUnsignedString(System.nanoTime()); long sinkStartOffset = endOffsetOnP0(consumerTopic); for (int i = 0; i < 10; i++) { ProducerRecord record = @@ -1596,7 +1617,9 @@ public void testRestoreKafkaToKafkaExactlyOnceOnStreaming(TestContainer containe () -> { try { container.executeJob( - "/kafka/kafka_to_kafka_exactly_once_streaming.conf", jobId); + "/kafka/kafka_to_kafka_exactly_once_streaming.conf", + jobId, + exactlyOnceVariables.toArray(new String[0])); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -1630,7 +1653,9 @@ public void testRestoreKafkaToKafkaExactlyOnceOnStreaming(TestContainer containe () -> { try { container.restoreJob( - "/kafka/kafka_to_kafka_exactly_once_streaming.conf", jobId); + "/kafka/kafka_to_kafka_exactly_once_streaming.conf", + jobId, + exactlyOnceVariables.toArray(new String[0])); } catch (Exception e) { throw new RuntimeException(e); } @@ -1655,9 +1680,14 @@ public void testRestoreKafkaToKafkaExactlyOnceOnStreaming(TestContainer containe type = EngineType.SPARK, value = {}) public void testKafkaToKafkaExactlyOnceOnStreaming(TestContainer container) { - - String producerTopic = "kafka_topic_exactly_once_1"; - String consumerTopic = "kafka_topic_exactly_once_2"; + String resourceSuffix = Long.toUnsignedString(System.nanoTime()); + String producerTopic = "kafka_topic_exactly_once_source_" + resourceSuffix; + String consumerTopic = "kafka_topic_exactly_once_sink_" + resourceSuffix; + String consumerGroup = "test_exactly_once_" + resourceSuffix; + List exactlyOnceVariables = + buildExactlyOnceStreamingVariables(producerTopic, consumerTopic, consumerGroup); + createKafkaTopic(producerTopic); + createKafkaTopic(consumerTopic); String sourceData = "Seatunnel Exactly Once Example"; long sinkStartOffset = endOffsetOnP0(consumerTopic); for (int i = 0; i < 10; i++) { @@ -1671,7 +1701,9 @@ public void testKafkaToKafkaExactlyOnceOnStreaming(TestContainer container) { CompletableFuture.supplyAsync( () -> { try { - container.executeJob("/kafka/kafka_to_kafka_exactly_once_streaming.conf"); + container.executeJob( + "/kafka/kafka_to_kafka_exactly_once_streaming.conf", + exactlyOnceVariables); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -2013,6 +2045,34 @@ private AdminClient createKafkaAdmin() { return AdminClient.create(props); } + /** + * Create a dedicated Kafka topic for the exactly-once tests so each method reads its own data + * and never reuses offsets from earlier runs in the same class. + */ + private void createKafkaTopic(String topicName) { + NewTopic topic = new NewTopic(topicName, 1, (short) 1); + topic.configs(Collections.singletonMap("retention.ms", "-1")); + try (AdminClient adminClient = createKafkaAdmin()) { + adminClient.createTopics(Collections.singletonList(topic)).all().get(); + dynamicTopics.add(topicName); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted while creating Kafka topic " + topicName, e); + } catch (ExecutionException e) { + throw new IllegalStateException("Failed to create Kafka topic " + topicName, e); + } + } + + /** Build the dynamic `-i key=value` variables for the exactly-once streaming template. */ + private List buildExactlyOnceStreamingVariables( + String sourceTopic, String sinkTopic, String consumerGroup) { + return Arrays.asList( + EXACTLY_ONCE_SOURCE_TOPIC_VARIABLE + "=" + sourceTopic, + EXACTLY_ONCE_SINK_TOPIC_VARIABLE + "=" + sinkTopic, + EXACTLY_ONCE_CONSUMER_GROUP_VARIABLE + "=" + consumerGroup); + } + private void initKafkaProducer() { Properties props = new Properties(); String bootstrapServers = kafkaContainer.getBootstrapServers(); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafka_to_kafka_exactly_once_streaming.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafka_to_kafka_exactly_once_streaming.conf index ddbc6034b037..ac82598d49b7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafka_to_kafka_exactly_once_streaming.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka/kafka_to_kafka_exactly_once_streaming.conf @@ -24,8 +24,8 @@ env { source { Kafka { bootstrap.servers = "kafkaCluster:9092" - topic = "kafka_topic_exactly_once_1" - consumer.group = "test_exactly_once" + topic = "${sourceTopic}" + consumer.group = "${consumerGroup}" # The default format is json, which is optional format = text start_mode = group_offsets @@ -42,7 +42,7 @@ transform {} sink{ kafka { format = text - topic = "kafka_topic_exactly_once_2" + topic = "${sinkTopic}" bootstrap.servers = "kafkaCluster:9092" semantics = EXACTLY_ONCE kafka.config = { diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterFailureNoRestoreIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterFailureNoRestoreIT.java index 3e46dbf6759c..b0ddf6a0e479 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterFailureNoRestoreIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterFailureNoRestoreIT.java @@ -56,6 +56,22 @@ public class ClusterFailureNoRestoreIT { private static final String TEST_TEMPLATE_FILE_NAME = "cluster_batch_fake_to_localfile_no_restore_template.conf"; + /** + * Keep the bounded fake source busy long enough to observe worker shutdown before the batch job + * converges on its own. + */ + private static final long NO_RESTORE_BATCH_ROW_NUM_PER_PARALLELISM = 20_000L; + + /** Wait for the batch job to enter the steady RUNNING state before shutting a worker down. */ + private static final long PRE_SHUTDOWN_RUNNING_TIMEOUT_SECONDS = 30L; + + /** + * Give the running batch topology a short warm-up window before shutting down a worker. The + * LocalFile sink used by this test commits files transactionally, so intermediate file lines + * are not a reliable progress signal while the job is still running. + */ + private static final long PRE_SHUTDOWN_RUNNING_GRACE_SECONDS = 5L; + private static final String DYNAMIC_TEST_CASE_NAME = "dynamic_test_case_name"; private static final String DYNAMIC_TEST_ROW_NUM_PER_PARALLELISM = @@ -68,7 +84,7 @@ public void testBatchJobWithoutCheckpointAndRetryConvergesAfterWorkerShutdown() throws Exception { String testCaseName = "testBatchJobWithoutCheckpointAndRetryConvergesAfterWorkerShutdown"; String testClusterName = "ClusterFailureNoRestoreIT_batch_no_restore"; - long testRowNumber = 10000; + long testRowNumber = NO_RESTORE_BATCH_ROW_NUM_PER_PARALLELISM; int testParallelism = 6; HazelcastInstanceImpl node1 = null; @@ -108,21 +124,13 @@ public void testBatchJobWithoutCheckpointAndRetryConvergesAfterWorkerShutdown() ClientJobProxy clientJobProxy = jobExecutionEnv.execute(); Awaitility.await() - .atMost(60, TimeUnit.SECONDS) - .pollInterval(500, TimeUnit.MILLISECONDS) + .atMost(PRE_SHUTDOWN_RUNNING_TIMEOUT_SECONDS, TimeUnit.SECONDS) .untilAsserted( - () -> { - Long lineNumberFromDir = - FileUtils.getFileLineNumberFromDir(testResources.getLeft()); - JobStatus status = clientJobProxy.getJobStatus(); - log.warn( - "\n====================={}=====================\n", - lineNumberFromDir); - Assertions.assertTrue(lineNumberFromDir > 1); - Assertions.assertFalse( - status.isEndState(), - "job finished before worker shutdown: " + status); - }); + () -> + Assertions.assertEquals( + JobStatus.RUNNING, clientJobProxy.getJobStatus())); + + TimeUnit.SECONDS.sleep(PRE_SHUTDOWN_RUNNING_GRACE_SECONDS); CompletableFuture waitForCompleteFuture = CompletableFuture.supplyAsync(clientJobProxy::waitForJobCompleteV2); diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java index bef36e35df0a..26cea8651ccb 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java @@ -39,7 +39,7 @@ import static io.restassured.RestAssured.given; import static org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; public abstract class ClassLoaderITBase extends SeaTunnelEngineContainer { @@ -49,6 +49,12 @@ public abstract class ClassLoaderITBase extends SeaTunnelEngineContainer { private static final String colon = ":"; + /** + * Disable-cache mode creates fresh classloaders for each submitted job, but the growth should + * stay linear relative to the cluster's initial classloader baseline. + */ + private static final int DISABLE_CACHE_MODE_CLASSLOADERS_PER_JOB = 2; + abstract boolean cacheMode(); private static final Path config = Paths.get(SEATUNNEL_HOME, "config"); @@ -60,16 +66,12 @@ public abstract class ClassLoaderITBase extends SeaTunnelEngineContainer { @Test public void testFakeSourceToInMemorySink() throws IOException, InterruptedException { LOG.info("test classloader with cache mode: {}", cacheMode()); + int initialClassLoaderCount = getClassLoaderCount(); for (int i = 0; i < 10; i++) { // load in memory sink which already leak thread with classloader Container.ExecResult execResult = executeJob(server, CONF_FILE); Assertions.assertEquals(0, execResult.getExitCode()); - Assertions.assertTrue(containsDaemonThread()); - if (cacheMode()) { - Assertions.assertTrue(3 >= getClassLoaderCount()); - } else { - Assertions.assertTrue(3 + 2 * i >= getClassLoaderCount()); - } + assertClassLoaderStateEventuallyStable(initialClassLoaderCount, i); } } @@ -99,46 +101,51 @@ public void testFakeSourceToInMemorySinkForRestApi() throws IOException, Interru Assertions.assertEquals( 1, response.jsonPath().getList("members").size()); }); + int initialClassLoaderCount = getClassLoaderCount(); for (int i = 0; i < 10; i++) { // load in memory sink which already leak thread with classloader - given().body( - "{\n" - + "\t\"env\": {\n" - + "\t\t\"parallelism\": 10,\n" - + "\t\t\"job.mode\": \"BATCH\"\n" - + "\t},\n" - + "\t\"source\": [\n" - + "\t\t{\n" - + "\t\t\t\"plugin_name\": \"FakeSource\",\n" - + "\t\t\t\"plugin_output\": \"fake\",\n" - + "\t\t\t\"parallelism\": 10,\n" - + "\t\t\t\"schema\": {\n" - + "\t\t\t\t\"fields\": {\n" - + "\t\t\t\t\t\"name\": \"string\",\n" - + "\t\t\t\t\t\"age\": \"int\",\n" - + "\t\t\t\t\t\"score\": \"double\"\n" - + "\t\t\t\t}\n" - + "\t\t\t}\n" - + "\t\t}\n" - + "\t],\n" - + "\t\"transform\": [],\n" - + "\t\"sink\": [\n" - + "\t\t{\n" - + "\t\t\t\"plugin_name\": \"InMemory\",\n" - + "\t\t\t\"plugin_input\": \"fake\"\n" - + "\t\t}\n" - + "\t]\n" - + "}") - .header("Content-Type", "application/json; charset=utf-8") - .post( - http - + server.getHost() - + colon - + server.getFirstMappedPort() - + RestConstant.CONTEXT_PATH - + RestConstant.REST_URL_SUBMIT_JOB) - .then() - .statusCode(200); + String jobId = + given().body( + "{\n" + + "\t\"env\": {\n" + + "\t\t\"parallelism\": 10,\n" + + "\t\t\"job.mode\": \"BATCH\"\n" + + "\t},\n" + + "\t\"source\": [\n" + + "\t\t{\n" + + "\t\t\t\"plugin_name\": \"FakeSource\",\n" + + "\t\t\t\"plugin_output\": \"fake\",\n" + + "\t\t\t\"parallelism\": 10,\n" + + "\t\t\t\"schema\": {\n" + + "\t\t\t\t\"fields\": {\n" + + "\t\t\t\t\t\"name\": \"string\",\n" + + "\t\t\t\t\t\"age\": \"int\",\n" + + "\t\t\t\t\t\"score\": \"double\"\n" + + "\t\t\t\t}\n" + + "\t\t\t}\n" + + "\t\t}\n" + + "\t],\n" + + "\t\"transform\": [],\n" + + "\t\"sink\": [\n" + + "\t\t{\n" + + "\t\t\t\"plugin_name\": \"InMemory\",\n" + + "\t\t\t\"plugin_input\": \"fake\"\n" + + "\t\t}\n" + + "\t]\n" + + "}") + .header("Content-Type", "application/json; charset=utf-8") + .post( + http + + server.getHost() + + colon + + server.getFirstMappedPort() + + RestConstant.CONTEXT_PATH + + RestConstant.REST_URL_SUBMIT_JOB) + .then() + .statusCode(200) + .extract() + .jsonPath() + .getString("jobId"); Awaitility.await() .atMost(2, TimeUnit.MINUTES) @@ -154,15 +161,30 @@ public void testFakeSourceToInMemorySinkForRestApi() throws IOException, Interru + "/FINISHED") .then() .statusCode(200) - .body("[0].jobStatus", equalTo("FINISHED"))); - Thread.sleep(5000); - Assertions.assertTrue(containsDaemonThread()); - if (cacheMode()) { - Assertions.assertTrue(3 >= getClassLoaderCount()); - } else { - Assertions.assertTrue(3 + 2 * i >= getClassLoaderCount()); - } + .body("jobId", hasItem(jobId))); + assertClassLoaderStateEventuallyStable(initialClassLoaderCount, i); + } + } + + private void assertClassLoaderStateEventuallyStable( + int initialClassLoaderCount, int iteration) { + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .untilAsserted( + () -> { + Assertions.assertTrue(containsDaemonThread()); + Assertions.assertTrue( + getClassLoaderCount() + <= getClassLoaderUpperBound( + initialClassLoaderCount, iteration)); + }); + } + + private int getClassLoaderUpperBound(int initialClassLoaderCount, int iteration) { + if (cacheMode()) { + return initialClassLoaderCount; } + return initialClassLoaderCount + DISABLE_CACHE_MODE_CLASSLOADERS_PER_JOB * (iteration + 1); } private int getClassLoaderCount() throws IOException, InterruptedException { diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java b/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java index 4da834720602..7f37656c64f1 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java +++ b/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java @@ -462,6 +462,7 @@ public void testWorkerIsFirstMemberThenGetJobDetailStatus() { .listJobStatus(true) .contains("RUNNING"))); jobClient.cancelJob(jobId); + // Master handoff can delay terminal status propagation on the slower JDK 8 CI lane. await().atMost(120000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/event/JobStateEventTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/event/JobStateEventTest.java index d028b8e5787e..9bf877b6cee4 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/event/JobStateEventTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/event/JobStateEventTest.java @@ -38,6 +38,12 @@ class JobStateEventTest extends AbstractSeaTunnelServerTest { + /** + * The commit-error batch job may restore several times before reaching the final FAILED state, + * so this event test needs the same longer timeout as the checkpoint restore regression test. + */ + private static final long FAILED_JOB_EVENT_TIMEOUT_SECONDS = 240L; + @Test void testJobStateEvent() { @@ -90,7 +96,7 @@ void testJobStateEvent() { long jobIdFailed = System.currentTimeMillis(); startJob(jobIdFailed, STREAM_CONF_WITH_ERROR_PATH, false); - await().atMost(60, TimeUnit.SECONDS) + await().atMost(FAILED_JOB_EVENT_TIMEOUT_SECONDS, TimeUnit.SECONDS) .untilAsserted( () -> Assertions.assertEquals( From 092b853d2e3d12fbc5673a1f3a4c827af46f2bd0 Mon Sep 17 00:00:00 2001 From: zuo <58384836+xxzuo@users.noreply.github.com> Date: Sat, 13 Jun 2026 22:46:33 +0800 Subject: [PATCH 016/375] [Docs][Core] Add Javadoc to TaskExecutionService inner classes (#10691) Co-authored-by: Daniel <48329107+DanielLeens@users.noreply.github.com> --- .../engine/server/TaskExecutionService.java | 273 +++++++++++++++++- 1 file changed, 266 insertions(+), 7 deletions(-) diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java index ae6e52a80dd5..a7b02766dc3a 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java @@ -112,39 +112,107 @@ import static org.apache.seatunnel.api.common.metrics.MetricTags.TASK_GROUP_LOCATION; import static org.apache.seatunnel.api.common.metrics.MetricTags.TASK_ID; -/** This class is responsible for the execution of the Task */ +/** + * This class is responsible for the execution of the Task. + * + *

TaskExecutionService manages the lifecycle of task execution in the SeaTunnel engine. It + * handles: + * + *

    + *
  • Task deployment and deserialization. + *
  • Task execution using cooperative multitasking (CooperativeTaskWorker) and blocking workers + * (BlockingWorker). + *
  • Class loader management for connector jars. + *
  • Task cancellation and cleanup. + *
  • Metrics collection and reporting. + *
+ * + *

The service supports two execution modes: + * + *

    + *
  • Thread-share mode: Tasks share a common thread pool and are executed cooperatively. + *
  • Blocking mode: Tasks run in dedicated threads for blocking operations. + *
+ * + *

Tasks are organized into TaskGroups, each tracked by a TaskGroupExecutionTracker that monitors + * execution state and handles completion/cancellation. + */ public class TaskExecutionService implements DynamicMetricsProvider { + /** The name of the Hazelcast instance this service runs on. */ private final String hzInstanceName; + + /** The NodeEngine implementation for this Hazelcast node. */ private final NodeEngineImpl nodeEngine; + + /** Service for managing class loaders for connector jars. */ private final ClassLoaderService classLoaderService; + + /** Logger for this service. */ private final ILogger logger; + + /** Flag indicating whether the service is running. */ private volatile boolean isRunning = true; + + /** Queue for tasks that can share threads (cooperative multitasking). */ private final LinkedBlockingDeque threadShareTaskQueue = new LinkedBlockingDeque<>(); + + /** Executor service for running task workers. */ private final ExecutorService executorService = newCachedThreadPool(new BlockingTaskThreadFactory()); + + /** Supplier for creating and running new BusWork threads. */ private final RunBusWorkSupplier runBusWorkSupplier = new RunBusWorkSupplier(executorService, threadShareTaskQueue); - // key: TaskID + + /** + * Cache of active execution contexts, keyed by TaskGroupLocation. Contains context for tasks + * currently being executed. + */ private final ConcurrentMap executionContexts = new ConcurrentHashMap<>(); + + /** + * Cache of finished execution contexts, keyed by TaskGroupLocation. Contains context for tasks + * that have completed but have not been cleaned up yet. + */ private final ConcurrentMap finishedExecutionContexts = new ConcurrentHashMap<>(); + /** + * Map of async function futures for each task group. Used to track and cancel async functions + * associated with a task group. + */ private final ConcurrentMap>> taskAsyncFunctionFuture = new ConcurrentHashMap<>(); + /** + * Map of cancellation futures for each task group. Used to cancel task group execution on + * request. + */ private final ConcurrentMap> cancellationFutures = new ConcurrentHashMap<>(); + + /** SeaTunnel configuration for this engine. */ private final SeaTunnelConfig seaTunnelConfig; + /** Scheduled executor for periodic tasks like metrics backup. */ private final ScheduledExecutorService scheduledExecutorService; + /** Client for managing connector packages on the server. */ private final ServerConnectorPackageClient serverConnectorPackageClient; + /** Service for reporting events. */ private final EventService eventService; + /** + * Creates a new TaskExecutionService. + * + * @param classLoaderService service for managing class loaders + * @param nodeEngine the Hazelcast node engine + * @param eventService service for reporting events + */ public TaskExecutionService( ClassLoaderService classLoaderService, NodeEngineImpl nodeEngine, @@ -174,20 +242,38 @@ public TaskExecutionService( this.eventService = eventService; } + /** + * Gets the Hazelcast node engine backing this task execution service. + * + * @return the Hazelcast node engine + */ public NodeEngineImpl getNodeEngine() { return nodeEngine; } + /** Starts the task execution service by creating initial cooperative task worker threads. */ public void start() { runBusWorkSupplier.runNewBusWork(false); } + /** + * Shuts down the task execution service. This method stops accepting new tasks and interrupts + * all running tasks. + */ public void shutdown() { isRunning = false; executorService.shutdownNow(); scheduledExecutorService.shutdown(); } + /** + * Gets the execution context for a task group. First checks active execution contexts, then + * falls back to finished execution contexts. + * + * @param taskGroupLocation the location of the task group + * @return the TaskGroupContext for the task group + * @throws TaskGroupContextNotFoundException if the task group is not found + */ public TaskGroupContext getExecutionContext(TaskGroupLocation taskGroupLocation) { TaskGroupContext taskGroupContext = executionContexts.get(taskGroupLocation); @@ -201,6 +287,14 @@ public TaskGroupContext getExecutionContext(TaskGroupLocation taskGroupLocation) return taskGroupContext; } + /** + * Gets the active execution context for a task group. Only checks active execution contexts, + * does not check finished contexts. + * + * @param taskGroupLocation the location of the task group + * @return the TaskGroupContext for the task group + * @throws TaskGroupContextNotFoundException if the task group is not found or not active + */ public TaskGroupContext getActiveExecutionContext(TaskGroupLocation taskGroupLocation) { TaskGroupContext taskGroupContext = executionContexts.get(taskGroupLocation); @@ -211,6 +305,13 @@ public TaskGroupContext getActiveExecutionContext(TaskGroupLocation taskGroupLoc return taskGroupContext; } + /** + * Submits tasks to the thread-share queue for cooperative execution. Each task is wrapped in a + * TaskTracker and initialized before being added to the queue. + * + * @param taskGroupExecutionTracker the tracker for the task group execution + * @param tasks the list of tasks to submit + */ private void submitThreadShareTask( TaskGroupExecutionTracker taskGroupExecutionTracker, List tasks) { Stream taskTrackerStream = @@ -236,6 +337,14 @@ private void submitThreadShareTask( } } + /** + * Submits tasks to the executor service for blocking execution. Each task runs in a dedicated + * thread using BlockingWorker. A CountDownLatch is used to ensure all workers have started + * before returning. + * + * @param taskGroupExecutionTracker the tracker for the task group execution + * @param tasks the list of tasks to submit + */ private void submitBlockingTask( TaskGroupExecutionTracker taskGroupExecutionTracker, List tasks) { MDCExecutorService mdcExecutorService = MDCTracer.tracing(executorService); @@ -265,18 +374,38 @@ private void submitBlockingTask( uncheckRun(startedLatch::await); } + /** + * Deploys a task from serialized data. + * + * @param taskImmutableInformation serialized task information + * @return the deployment state indicating success or failure + */ public TaskDeployState deployTask(@NonNull Data taskImmutableInformation) { TaskGroupImmutableInformation taskImmutableInfo = nodeEngine.getSerializationService().toObject(taskImmutableInformation); return deployTask(taskImmutableInfo); } + /** + * Gets a task by its location. + * + * @param taskLocation the location of the task + * @param the task type + * @return the task + */ public T getTask(@NonNull TaskLocation taskLocation) { TaskGroupContext executionContext = this.getActiveExecutionContext(taskLocation.getTaskGroupLocation()); return executionContext.getTaskGroup().getTask(taskLocation.getTaskID()); } + /** + * Deploys a task group from TaskGroupImmutableInformation. This method handles task + * deserialization, class loader setup, and task group creation. + * + * @param taskImmutableInfo the task group information + * @return the deployment state indicating success or failure + */ public TaskDeployState deployTask(@NonNull TaskGroupImmutableInformation taskImmutableInfo) { logger.info( String.format( @@ -369,6 +498,15 @@ public TaskDeployState deployTask(@NonNull TaskGroupImmutableInformation taskImm } } + /** + * Deploys a task group locally. This method initializes the task group, creates execution + * contexts, and submits tasks for execution based on the configured thread share mode. + * + * @param taskGroup the task group to deploy + * @param classLoaders map of task IDs to class loaders + * @param jars map of task IDs to connector jars + * @return a future that completes with the task execution state + */ public PassiveCompletableFuture deployLocalTask( @NonNull TaskGroup taskGroup, @NonNull ConcurrentHashMap classLoaders, @@ -456,6 +594,13 @@ public PassiveCompletableFuture deployLocalTask( return new PassiveCompletableFuture<>(resultFuture); } + /** + * Notifies the master node of the task execution state. This method retries indefinitely until + * successful or the service is shutdown. + * + * @param taskGroupLocation the location of the task group + * @param taskExecutionState the execution state to report + */ private void notifyTaskStatusToMaster( TaskGroupLocation taskGroupLocation, TaskExecutionState taskExecutionState) { long sleepTime = 1000; @@ -518,6 +663,13 @@ public void cancelTaskGroup(TaskGroupLocation taskGroupLocation) { } } + /** + * Executes a function asynchronously in the context of a task group. The function is tracked + * and can be cancelled when the task group is cancelled. + * + * @param taskGroupLocation the task group location + * @param task the Runnable to execute + */ public void asyncExecuteFunction(TaskGroupLocation taskGroupLocation, Runnable task) { String id = UUID.randomUUID().toString(); logger.fine("accept async execute function from " + taskGroupLocation + " with id " + id); @@ -538,6 +690,12 @@ public void asyncExecuteFunction(TaskGroupLocation taskGroupLocation, Runnable t }); } + /** + * Notifies the service to clean up the execution context for a finished task group. This is + * called when the task group context is no longer needed. + * + * @param taskGroupLocation the task group location to clean up + */ public void notifyCleanTaskGroupContext(TaskGroupLocation taskGroupLocation) { finishedExecutionContexts.remove(taskGroupLocation); } @@ -641,6 +799,10 @@ private HashMap collectLocalMetricsMap() return localMap; } + /** + * Prints task execution runtime information to the log. This includes thread pool status like + * active count, queue size, and task counts. Only logs when fine level logging is enabled. + */ public void printTaskExecutionRuntimeInfo() { if (logger.isFineEnabled()) { ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) executorService; @@ -662,14 +824,28 @@ public void printTaskExecutionRuntimeInfo() { } } + /** + * Reports an event to the event service. + * + * @param e the event to report + */ public void reportEvent(Event e) { eventService.reportEvent(e); } + /** + * Gets the SeaTunnel configuration. + * + * @return the SeaTunnel configuration + */ public SeaTunnelConfig getSeaTunnelConfig() { return seaTunnelConfig; } + /** + * Worker that executes blocking tasks in a dedicated thread. Each BlockingWorker runs a single + * task to completion, suitable for I/O-bound operations that may block. + */ private final class BlockingWorker implements Runnable { private final TaskTracker tracker; @@ -680,6 +856,21 @@ private BlockingWorker(TaskTracker tracker, CountDownLatch startedLatch) { this.startedLatch = startedLatch; } + /** + * Executes the blocking task in a dedicated thread. The task runs to completion (or + * failure/cancellation) without preemption. + * + *

Execution flow: + * + *

    + *
  1. Set up the class loader for the task + *
  2. Signal that the worker has started via CountDownLatch + *
  3. Initialize the task via {@link Task#init()} + *
  4. Execute the task repeatedly via {@link Task#call()} until done + *
  5. Handle interrupts and exceptions, notifying the execution tracker + *
  6. Clean up by calling {@link Task#close()} if not completed + *
+ */ @Override public void run() { TaskExecutionService.TaskGroupExecutionTracker taskGroupExecutionTracker = @@ -728,6 +919,12 @@ public void run() { } } + /** + * ThreadFactory for creating named threads used for SeaTunnel task execution. The shared + * executor service created with this factory may run blocking workers, cooperative workers, and + * asynchronous tasks. Threads are named with the pattern {@code + * hz.{instance}.seaTunnel.task.thread-{n}}. + */ private final class BlockingTaskThreadFactory implements ThreadFactory { private final AtomicInteger seq = new AtomicInteger(); @@ -742,8 +939,13 @@ public Thread newThread(@NonNull Runnable r) { } /** - * CooperativeTaskWorker is used to poll the task call method, When a task times out, a new - * BusWork will be created to take over the execution of the task + * Cooperative task worker that polls tasks from the queue and executes them cooperatively. Uses + * a TaskCallTimer to detect stuck tasks. When a task times out, a new BusWork will be created + * to take over the execution. + * + *

In cooperative mode, multiple tasks share a single worker thread. Each task yields control + * by returning {@link ProgressState#isDone()} == false, allowing other tasks to run. This is + * efficient for CPU-bound tasks that don't block. */ public final class CooperativeTaskWorker implements Runnable { @@ -765,6 +967,21 @@ public CooperativeTaskWorker( this.futureBlockingQueue = futureBlockingQueue; } + /** + * Main execution loop for the cooperative task worker. Continuously polls tasks from the + * queue and executes them. + * + *

The execution flow: + * + *

    + *
  1. Wait for a task from the queue or exclusive tracker + *
  2. Check if execution completed exceptionally, handle accordingly + *
  3. Start the task call timer for timeout detection + *
  4. Execute the task via {@link Task#call()} + *
  5. Stop the timer and check the result + *
  6. If task is done, mark it complete; otherwise, re-queue for next iteration + *
+ */ @SneakyThrows @Override public void run() { @@ -851,7 +1068,11 @@ public void run() { } } - /** Used to create a new BusWork and run */ + /** + * Supplier that creates and runs new CooperativeTaskWorker instances (BusWork) when needed. New + * workers are created either unconditionally or, when requested by the caller, only if the task + * queue currently contains pending tasks. + */ public final class RunBusWorkSupplier { ExecutorService executorService; @@ -863,6 +1084,12 @@ public RunBusWorkSupplier( this.taskQueue = taskqueue; } + /** + * Creates and submits a new CooperativeTaskWorker if conditions are met. + * + * @param checkTaskQueue if true, only creates a new worker if the task queue is not empty + * @return true if a new worker was created and submitted, false otherwise + */ public boolean runNewBusWork(boolean checkTaskQueue) { if (!checkTaskQueue || !taskQueue.isEmpty()) { BlockingQueue> futureBlockingQueue = new LinkedBlockingQueue<>(); @@ -877,8 +1104,8 @@ public boolean runNewBusWork(boolean checkTaskQueue) { } /** - * Internal utility class to track the overall state of tasklet execution. There's one instance - * of this class per job. + * Internal utility class to track the overall state of a TaskGroup execution. There's one + * instance of this class per TaskGroup. */ public final class TaskGroupExecutionTracker { @@ -915,6 +1142,12 @@ public final class TaskGroupExecutionTracker { })); } + /** + * Records an exception that occurred during task execution. Uses compareAndSet to ensure + * only the first exception is recorded. + * + * @param t the exception that occurred + */ void exception(Throwable t) { executionException.compareAndSet(null, t); } @@ -942,6 +1175,23 @@ private void cancelAsyncFunction(TaskGroupLocation taskGroupLocation) { } } + /** + * Marks a task as done and handles completion logic for the task group. + * + *

When the last task completes (completionLatch reaches zero): + * + *

    + *
  1. Recycle the class loader + *
  2. Move execution context from active to finished + *
  3. Cancel async functions and update metrics + *
  4. Complete the future with final state (FINISHED, CANCELED, or FAILED) + *
+ * + *

If an exception occurred and the task group is not cancelled, cancels all remaining + * tasks in the group. + * + * @param task the task that completed + */ void taskDone(Task task) { TaskGroupLocation taskGroupLocation = taskGroup.getTaskGroupLocation(); logger.info( @@ -1007,10 +1257,19 @@ boolean executionCompletedExceptionally() { } } + /** + * Gets the server connector package client for managing connector jars. + * + * @return the server connector package client + */ public ServerConnectorPackageClient getServerConnectorPackageClient() { return serverConnectorPackageClient; } + /** + * A Runnable wrapper that sets a custom thread name before executing the task and restores the + * original name afterward. + */ public static class NamedTaskWrapper implements Runnable { private final Runnable task; private final String threadName; From b4fe1ef25b67557db010ef6f3672e21feeb57b5e Mon Sep 17 00:00:00 2001 From: David Zollo Date: Sun, 14 Jun 2026 01:45:35 +0800 Subject: [PATCH 017/375] [Test][SqlServer-CDC] Add multi-table e2e testcase (#11054) Co-authored-by: Daniel <48329107+DanielLeens@users.noreply.github.com> --- .../cdc/sqlserver/SqlServerCDCIT.java | 69 +++++++++++ .../test/resources/ddl/column_type_test.sql | 112 ++++++++++++++++++ ...erver_with_multi_table_mode_two_table.conf | 60 ++++++++++ 3 files changed, 241 insertions(+) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/sqlservercdc_to_sqlserver_with_multi_table_mode_two_table.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java index 8df8f19a8f3a..fc48f3f81e40 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java @@ -98,12 +98,20 @@ public class SqlServerCDCIT extends TestSuiteBase implements TestResource { + "EXEC sys.sp_cdc_disable_db"; private static final String SOURCE_TABLE = DATABASE_NAME + "." + SCHEMA_NAME + "." + "full_types"; + // Additional source table used to verify multi-table CDC capture in one job. + private static final String SOURCE_TABLE_2 = + DATABASE_NAME + "." + SCHEMA_NAME + "." + "full_types_2"; private static final String SOURCE_TABLE_NO_PRIMARY_KEY = DATABASE_NAME + "." + SCHEMA_NAME + "." + "full_types_no_primary_key"; private static final String SOURCE_TABLE_CUSTOM_PRIMARY_KEY = DATABASE_NAME + "." + SCHEMA_NAME + "." + "full_types_custom_primary_key"; private static final String SINK_TABLE = DATABASE_NAME + "." + SCHEMA_NAME + "." + "full_types_sink"; + // Sink tables are derived from the source names with the configured sink_ prefix. + private static final String MULTI_TABLE_SINK_1 = + DATABASE_NAME + "." + SCHEMA_NAME + "." + "sink_full_types"; + private static final String MULTI_TABLE_SINK_2 = + DATABASE_NAME + "." + SCHEMA_NAME + "." + "sink_full_types_2"; private static final String SELECT_SOURCE_SQL = "select\n" @@ -259,6 +267,67 @@ public void test(TestContainer container) throws IOException, InterruptedExcepti }); } + /** + * Verifies that a single SqlServer CDC source can capture multiple tables and route them to + * different sink tables in the same database. + * + *

The sink tables are pre-created so this regression stays focused on multi-table routing + * instead of SQL Server auto-create type derivation while still exercising Jdbc table-mode + * writes. + */ + @TestTemplate + public void testSqlServerCdcMultiTableE2e(TestContainer container) { + initializeSqlServerTable(DATABASE_NAME); + + CompletableFuture.supplyAsync( + () -> { + try { + container.executeJob( + "/sqlservercdc_to_sqlserver_with_multi_table_mode_two_table.conf"); + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + }); + + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> + Assertions.assertAll( + () -> + Assertions.assertIterableEquals( + querySql(SELECT_SOURCE_SQL, SOURCE_TABLE), + querySql( + SELECT_SINK_SQL, + MULTI_TABLE_SINK_1)), + () -> + Assertions.assertIterableEquals( + querySql(SELECT_SOURCE_SQL, SOURCE_TABLE_2), + querySql( + SELECT_SINK_SQL, + MULTI_TABLE_SINK_2)))); + + updateSourceTable(SOURCE_TABLE); + updateSourceTable(SOURCE_TABLE_2); + + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> + Assertions.assertAll( + () -> + Assertions.assertIterableEquals( + querySql(SELECT_SOURCE_SQL, SOURCE_TABLE), + querySql( + SELECT_SINK_SQL, + MULTI_TABLE_SINK_1)), + () -> + Assertions.assertIterableEquals( + querySql(SELECT_SOURCE_SQL, SOURCE_TABLE_2), + querySql( + SELECT_SINK_SQL, + MULTI_TABLE_SINK_2)))); + } + @TestTemplate @DisabledOnContainer( value = {}, diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/ddl/column_type_test.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/ddl/column_type_test.sql index 64ceebcaa828..72cd6a1ac51a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/ddl/column_type_test.sql +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/ddl/column_type_test.sql @@ -75,6 +75,56 @@ INSERT INTO full_types VALUES (2, 'b',SYSDATETIMEOFFSET(),CAST('test_varbinary' AS varbinary(100)), 5.32); EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'full_types', @role_name = NULL, @supports_net_changes = 0; +CREATE TABLE full_types_2 ( + id int NOT NULL, + val_char char(3), + val_varchar varchar(1000), + val_text text, + val_nchar nchar(3), + val_nvarchar nvarchar(1000), + val_ntext ntext, + val_decimal decimal(6,3), + val_numeric numeric, + val_float float, + val_real real, + val_smallmoney smallmoney, + val_money money, + val_bit bit, + val_tinyint tinyint, + val_smallint smallint, + val_int int, + val_bigint bigint, + val_date date, + val_time time, + val_datetime2 datetime2, + val_datetime datetime, + val_smalldatetime smalldatetime, + val_xml xml, + val_datetimeoffset DATETIMEOFFSET(4), + val_varbinary varbinary(100), + val_udtdecimal UDTDECIMAL, + PRIMARY KEY (id) +); +INSERT INTO full_types_2 VALUES (0, + 'cč0', 'vcč', 'tč', N'cč', N'vcč', N'tč', + 1.123, 2, 3.323, 4.323, 5.323, 6.323, + 1, 22, 333, 4444, 55555, + '2018-07-13', '10:23:45', '2018-07-13 11:23:45.34', '2018-07-13 13:23:45.78', '2018-07-13 14:23:45', + 'b',SYSDATETIMEOFFSET(),CAST('test_varbinary' AS varbinary(100)), 5.32); +INSERT INTO full_types_2 VALUES (1, + 'cč1', 'vcč', 'tč', N'cč', N'vcč', N'tč', + 1.123, 2, 3.323, 4.323, 5.323, 6.323, + 1, 22, 333, 4444, 55555, + '2018-07-13', '10:23:45', '2018-07-13 11:23:45.34', '2018-07-13 13:23:45.78', '2018-07-13 14:23:45', + 'b',SYSDATETIMEOFFSET(),CAST('test_varbinary' AS varbinary(100)), 5.32); +INSERT INTO full_types_2 VALUES (2, + 'cč2', 'vcč', 'tč', N'cč', N'vcč', N'tč', + 1.123, 2, 3.323, 4.323, 5.323, 6.323, + 1, 22, 333, 4444, 55555, + '2018-07-13', '10:23:45', '2018-07-13 11:23:45.34', '2018-07-13 13:23:45.78', '2018-07-13 14:23:45', + 'b',SYSDATETIMEOFFSET(),CAST('test_varbinary' AS varbinary(100)), 5.32); +EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'full_types_2', @role_name = NULL, @supports_net_changes = 0; + CREATE TABLE full_types_no_primary_key ( id int NOT NULL, val_char char(3), @@ -203,3 +253,65 @@ CREATE TABLE full_types_sink ( val_udtdecimal UDTDECIMAL, PRIMARY KEY (id) ); + +CREATE TABLE sink_full_types ( + id int NOT NULL, + val_char char(3), + val_varchar varchar(1000), + val_text text, + val_nchar nchar(3), + val_nvarchar nvarchar(1000), + val_ntext ntext, + val_decimal decimal(6,3), + val_numeric numeric, + val_float float, + val_real real, + val_smallmoney smallmoney, + val_money money, + val_bit bit, + val_tinyint tinyint, + val_smallint smallint, + val_int int, + val_bigint bigint, + val_date date, + val_time time, + val_datetime2 datetime2, + val_datetime datetime, + val_smalldatetime smalldatetime, + val_xml xml, + val_datetimeoffset DATETIMEOFFSET(4), + val_varbinary varbinary(100), + val_udtdecimal UDTDECIMAL, + PRIMARY KEY (id) +); + +CREATE TABLE sink_full_types_2 ( + id int NOT NULL, + val_char char(3), + val_varchar varchar(1000), + val_text text, + val_nchar nchar(3), + val_nvarchar nvarchar(1000), + val_ntext ntext, + val_decimal decimal(6,3), + val_numeric numeric, + val_float float, + val_real real, + val_smallmoney smallmoney, + val_money money, + val_bit bit, + val_tinyint tinyint, + val_smallint smallint, + val_int int, + val_bigint bigint, + val_date date, + val_time time, + val_datetime2 datetime2, + val_datetime datetime, + val_smalldatetime smalldatetime, + val_xml xml, + val_datetimeoffset DATETIMEOFFSET(4), + val_varbinary varbinary(100), + val_udtdecimal UDTDECIMAL, + PRIMARY KEY (id) +); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/sqlservercdc_to_sqlserver_with_multi_table_mode_two_table.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/sqlservercdc_to_sqlserver_with_multi_table_mode_two_table.conf new file mode 100644 index 000000000000..f367f29e4b21 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/sqlservercdc_to_sqlserver_with_multi_table_mode_two_table.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set engine configuration here + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + SqlServer-CDC { + plugin_output = "customers" + username = "sa" + password = "Password!" + database-names = ["column_type_test"] + table-names = ["column_type_test.dbo.full_types", "column_type_test.dbo.full_types_2"] + url = "jdbc:sqlserver://sqlserver-host:1433;databaseName=column_type_test" + } +} + +transform { +} + +sink { + Jdbc { + plugin_input = "customers" + driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + url = "jdbc:sqlserver://sqlserver-host:1433;encrypt=false" + user = "sa" + password = "Password!" + # Pre-create both sink tables so this E2E only validates multi-table CDC routing. + # Keep table-mode writes enabled because Jdbc sink currently reserves query mode for + # generate_sink_sql = false. + generate_sink_sql = true + database = "column_type_test" + schema = "dbo" + tablePrefix = "sink_" + batch_size = 1 + primary_keys = ["id"] + } +} From 2826ea3dcd90350ddcb5391298f1bc52e4448a5d Mon Sep 17 00:00:00 2001 From: Kang Myeong Gwan Date: Mon, 15 Jun 2026 10:08:01 +0900 Subject: [PATCH 018/375] [Feature][Connector-V2] Add recursive_file_scan option for file connectors (#10505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 강명관 --- docs/en/connectors/source/CosFile.md | 7 +- docs/en/connectors/source/FtpFile.md | 8 +- docs/en/connectors/source/HdfsFile.md | 7 + docs/en/connectors/source/LocalFile.md | 9 +- docs/en/connectors/source/ObsFile.md | 2 +- docs/en/connectors/source/OssFile.md | 7 + docs/en/connectors/source/OssJindoFile.md | 6 + docs/en/connectors/source/S3File.md | 7 + docs/en/connectors/source/SftpFile.md | 7 + docs/zh/connectors/source/CosFile.md | 7 +- docs/zh/connectors/source/FtpFile.md | 8 +- docs/zh/connectors/source/HdfsFile.md | 7 + docs/zh/connectors/source/LocalFile.md | 8 +- docs/zh/connectors/source/ObsFile.md | 3 +- docs/zh/connectors/source/OssFile.md | 8 +- docs/zh/connectors/source/OssJindoFile.md | 3 +- docs/zh/connectors/source/S3File.md | 8 +- docs/zh/connectors/source/SftpFile.md | 8 +- .../file/config/FileBaseSourceOptions.java | 8 + .../source/reader/AbstractReadStrategy.java | 9 +- ...ultipleTableFileSourceSplitEnumerator.java | 10 +- .../reader/AbstractReadStrategyTest.java | 168 ++++++++++++++++++ .../source/reader/UpdateSyncModeTest.java | 77 ++++++++ ...pleTableFileSourceSplitEnumeratorTest.java | 114 +++++++++++- .../file/cos/source/CosFileSourceFactory.java | 1 + .../file/ftp/source/FtpFileSourceFactory.java | 1 + .../hdfs/source/HdfsFileSourceFactory.java | 2 + .../jindo/source/OssFileSourceFactory.java | 1 + .../local/source/LocalFileSourceFactory.java | 1 + .../file/obs/source/ObsFileSourceFactory.java | 1 + .../file/oss/source/OssFileSourceFactory.java | 1 + .../file/s3/source/S3FileSourceFactory.java | 1 + .../sftp/source/SftpFileSourceFactory.java | 1 + .../e2e/connector/file/ftp/FtpFileIT.java | 127 ++++++++++++- ...pdate_distcp_continuous_non_recursive.conf | 57 ++++++ ...tp_binary_update_non_recursive_distcp.conf | 52 ++++++ ..._update_non_recursive_strict_checksum.conf | 52 ++++++ ...ftp_file_text_non_recursive_to_assert.conf | 121 +++++++++++++ .../ftp_file_text_recursive_to_assert.conf | 121 +++++++++++++ .../e2e/connector/file/hdfs/HdfsFileIT.java | 139 +++++++++++++++ ...pdate_distcp_continuous_non_recursive.conf | 58 ++++++ ...fs_binary_update_non_recursive_distcp.conf | 53 ++++++ ..._update_non_recursive_strict_checksum.conf | 53 ++++++ .../hdfs_text_non_recursive_to_assert.conf | 113 ++++++++++++ .../hdfs_text_recursive_to_assert.conf | 113 ++++++++++++ .../src/test/resources/text/e2e.txt | 5 + .../e2e/connector/file/local/LocalFileIT.java | 128 +++++++++++++ ...pdate_distcp_continuous_non_recursive.conf | 47 +++++ ...le_binary_update_non_recursive_distcp.conf | 42 +++++ ..._update_non_recursive_strict_checksum.conf | 42 +++++ ...cal_file_text_non_recursive_to_assert.conf | 117 ++++++++++++ .../local_file_text_recursive_to_assert.conf | 117 ++++++++++++ .../e2e/connector/file/fstp/SftpFileIT.java | 137 ++++++++++++++ ...pdate_distcp_continuous_non_recursive.conf | 64 +++++++ ...tp_binary_update_non_recursive_distcp.conf | 59 ++++++ ..._update_non_recursive_strict_checksum.conf | 59 ++++++ ...ftp_file_text_non_recursive_to_assert.conf | 122 +++++++++++++ .../sftp_file_text_recursive_to_assert.conf | 122 +++++++++++++ 58 files changed, 2609 insertions(+), 27 deletions(-) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_distcp_continuous_non_recursive.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_distcp.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_strict_checksum.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_non_recursive_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_recursive_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_distcp_continuous_non_recursive.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_distcp.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_strict_checksum.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_non_recursive_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_recursive_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/text/e2e.txt create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_distcp_continuous_non_recursive.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_distcp.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_strict_checksum.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_non_recursive_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_recursive_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_distcp_continuous_non_recursive.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_distcp.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_strict_checksum.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_non_recursive_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_recursive_to_assert.conf diff --git a/docs/en/connectors/source/CosFile.md b/docs/en/connectors/source/CosFile.md index 4c71220b2230..d6a793f8cbd2 100644 --- a/docs/en/connectors/source/CosFile.md +++ b/docs/en/connectors/source/CosFile.md @@ -85,6 +85,7 @@ To use this connector you need put hadoop-cos-{hadoop.version}-{version}.jar and | file_filter_modified_end | string | no | - | | quote_char | string | no | " | | escape_char | string | no | - | +| recursive_file_scan | boolean | no | true | | sort_files_by_modification_time | boolean | no | false | ### path [string] @@ -437,6 +438,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. +### recursive_file_scan [boolean] + +Whether to scan subdirectories recursively. +If `false`, subdirectories will be ignored. + ### sort_files_by_modification_time [boolean] Whether to sort files by modification time in descending order. Default is `false`. @@ -548,4 +554,3 @@ sink { ## Changelog - diff --git a/docs/en/connectors/source/FtpFile.md b/docs/en/connectors/source/FtpFile.md index a04def1680bd..87589f59458c 100644 --- a/docs/en/connectors/source/FtpFile.md +++ b/docs/en/connectors/source/FtpFile.md @@ -89,8 +89,9 @@ If you use SeaTunnel Engine, It automatically integrated the hadoop jar when you | file_filter_modified_end | string | no | - | | quote_char | string | no | " | | escape_char | string | no | - | -| sort_files_by_modification_time | boolean | no | false | | metalake_type | string | no | gravitino | +| recursive_file_scan | boolean | no | true | +| sort_files_by_modification_time | boolean | no | false | ### host [string] @@ -539,6 +540,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. +### recursive_file_scan [boolean] + +Whether to scan subdirectories recursively. +If `false`, subdirectories will be ignored. + ### sort_files_by_modification_time [boolean] Whether to sort files by modification time in descending order. Default is `false`. diff --git a/docs/en/connectors/source/HdfsFile.md b/docs/en/connectors/source/HdfsFile.md index 8124c1a6c6cb..514f1485de5a 100644 --- a/docs/en/connectors/source/HdfsFile.md +++ b/docs/en/connectors/source/HdfsFile.md @@ -95,6 +95,8 @@ Read data from hdfs file system. | file_split_size | long | no | 134217728 | Split size in bytes when `enable_file_split=true`. For `text`/`csv`/`json`, the split end will be aligned to the next `row_delimiter`. For `parquet`, the split unit is RowGroup and will never break a RowGroup. | | quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. | | escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. | +| metalake_type | string | no | gravitino | The type of metalake service, currently supports `gravitino`. | +| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. | | sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. | ### file_format_type [string] @@ -321,6 +323,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. +### recursive_file_scan [boolean] + +Whether to scan subdirectories recursively. +If `false`, subdirectories will be ignored. + ### sort_files_by_modification_time [boolean] Whether to sort files by modification time in descending order. Default is `false`. diff --git a/docs/en/connectors/source/LocalFile.md b/docs/en/connectors/source/LocalFile.md index 44b208b8aceb..de5f1c43d7ac 100644 --- a/docs/en/connectors/source/LocalFile.md +++ b/docs/en/connectors/source/LocalFile.md @@ -92,7 +92,9 @@ If you use SeaTunnel Engine, It automatically integrated the hadoop jar when you | file_split_size | long | no | 134217728 | | quote_char | string | no | " | | escape_char | string | no | - | -| sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. | +| metalake_type | string | no | gravitino | +| recursive_file_scan | boolean | no | true | +| sort_files_by_modification_time | boolean | no | false | ### path [string] The source file path. @@ -535,6 +537,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. +### recursive_file_scan [boolean] + +Whether to scan subdirectories recursively. +If `false`, subdirectories will be ignored. + ### sort_files_by_modification_time [boolean] Whether to sort files by modification time in descending order. Default is `false`. diff --git a/docs/en/connectors/source/ObsFile.md b/docs/en/connectors/source/ObsFile.md index 4ecc0d5a2199..618885a346af 100644 --- a/docs/en/connectors/source/ObsFile.md +++ b/docs/en/connectors/source/ObsFile.md @@ -86,6 +86,7 @@ It only supports hadoop version **2.9.X+**. | file_filter_modified_end | string | no | - | File modification time filter. The connector will filter some files base on the last modification end time (not include end time). The default data format is `yyyy-MM-dd HH:mm:ss`. | | quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. | | escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. | +| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. | | sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. | ### Tips @@ -390,4 +391,3 @@ schema { ## Changelog - diff --git a/docs/en/connectors/source/OssFile.md b/docs/en/connectors/source/OssFile.md index 8636bce17745..6bc5fb63d51c 100644 --- a/docs/en/connectors/source/OssFile.md +++ b/docs/en/connectors/source/OssFile.md @@ -217,6 +217,8 @@ If you assign file type to `parquet` `orc`, schema option not required, connecto | file_filter_modified_end | string | no | - | File modification time filter. The connector will filter some files base on the last modification end time (not include end time). The default data format is `yyyy-MM-dd HH:mm:ss`. | | quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. | | escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. | +| metalake_type | string | no | gravitino | The type of metalake service, currently supports `gravitino`. | +| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. | | sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. | ### file_format_type [string] @@ -283,6 +285,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. +### recursive_file_scan [boolean] + +Whether to scan subdirectories recursively. +If `false`, subdirectories will be ignored. + ### sort_files_by_modification_time [boolean] Whether to sort files by modification time in descending order. Default is `false`. diff --git a/docs/en/connectors/source/OssJindoFile.md b/docs/en/connectors/source/OssJindoFile.md index 6cadda858c43..7a4f9654806f 100644 --- a/docs/en/connectors/source/OssJindoFile.md +++ b/docs/en/connectors/source/OssJindoFile.md @@ -87,6 +87,7 @@ It only supports hadoop version **2.9.X+**. | file_filter_modified_end | string | no | - | | quote_char | string | no | " | | escape_char | string | no | - | +| recursive_file_scan | boolean | no | true | | sort_files_by_modification_time | boolean | no | false | ### path [string] @@ -418,6 +419,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. +### recursive_file_scan [boolean] + +Whether to scan subdirectories recursively. +If `false`, subdirectories will be ignored. + ### sort_files_by_modification_time [boolean] Whether to sort files by modification time in descending order. Default is `false`. diff --git a/docs/en/connectors/source/S3File.md b/docs/en/connectors/source/S3File.md index b4ccfda5e957..71bd4cedd9ad 100644 --- a/docs/en/connectors/source/S3File.md +++ b/docs/en/connectors/source/S3File.md @@ -226,6 +226,8 @@ If you assign file type to `parquet` `orc`, schema option not required, connecto | common-options | | no | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details. | | quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. | | escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. | +| metalake_type | string | no | gravitino | The type of metalake service, currently supports `gravitino`. | +| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. | | sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. | ### file_format_type [string] @@ -419,6 +421,11 @@ When specified, the connector will fetch table schema from the external metadata For more information, please refer to [Metadata SPI](../../introduction/concepts/metadata-spi.md). +### recursive_file_scan [boolean] + +Whether to scan subdirectories recursively. +If `false`, subdirectories will be ignored. + ## Example 1. In this example, We read data from s3 path `s3a://seatunnel-test/seatunnel/text` and the file type is orc in this path. diff --git a/docs/en/connectors/source/SftpFile.md b/docs/en/connectors/source/SftpFile.md index 3e95521f075a..92bee70989d4 100644 --- a/docs/en/connectors/source/SftpFile.md +++ b/docs/en/connectors/source/SftpFile.md @@ -121,6 +121,8 @@ The File does not have a specific type list, and we can indicate which SeaTunnel | file_filter_modified_end | string | no | - | File modification time filter. The connector will filter some files base on the last modification end time (not include end time). The default data format is `yyyy-MM-dd HH:mm:ss`. | | quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. | | escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. | +| metalake_type | string | no | gravitino | The type of metalake service, currently supports `gravitino`. | +| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. | | sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. | ### file_filter_pattern [string] @@ -379,6 +381,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. +### recursive_file_scan [boolean] + +Whether to scan subdirectories recursively. +If `false`, subdirectories will be ignored. + ### sort_files_by_modification_time [boolean] Whether to sort files by modification time in descending order. Default is `false`. diff --git a/docs/zh/connectors/source/CosFile.md b/docs/zh/connectors/source/CosFile.md index f9622a1ab39f..6841dcbd4ea7 100644 --- a/docs/zh/connectors/source/CosFile.md +++ b/docs/zh/connectors/source/CosFile.md @@ -84,6 +84,7 @@ import ChangeLog from '../changelog/connector-file-cos.md'; | file_filter_modified_end | string | 否 | - | | quote_char | string | 否 | " | | escape_char | string | 否 | - | +| recursive_file_scan | boolean | 否 | true | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### path [string] @@ -431,10 +432,14 @@ abc.* 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 +### recursive_file_scan [boolean] + +是否递归扫描子目录。 +如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 + ### sort_files_by_modification_time [boolean] 是否按修改时间降序排序文件。默认值为 `false`。 - 启用后，文件将按修改时间排序（最新的在前）。适用于以下场景： - 读取具有不断演化的 schema 的文件，且希望 schema 推断使用最新的文件 - 需要按时间顺序处理文件 diff --git a/docs/zh/connectors/source/FtpFile.md b/docs/zh/connectors/source/FtpFile.md index 00cffcdfd446..088f1c57835b 100644 --- a/docs/zh/connectors/source/FtpFile.md +++ b/docs/zh/connectors/source/FtpFile.md @@ -85,6 +85,8 @@ import ChangeLog from '../changelog/connector-file-ftp.md'; | file_filter_modified_end | string | 否 | - | | quote_char | string | 否 | " | | escape_char | string | 否 | - | +| metalake_type | string | 否 | gravitino | +| recursive_file_scan | boolean | 否 | true | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### host [string] @@ -508,10 +510,14 @@ compare_mode = "len_mtime" 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 +### recursive_file_scan [boolean] + +是否递归扫描子目录。 +如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 + ### sort_files_by_modification_time [boolean] 是否按修改时间降序排序文件。默认值为 `false`。 - 启用后，文件将按修改时间排序（最新的在前）。适用于以下场景： - 读取具有不断演化的 schema 的文件，且希望 schema 推断使用最新的文件 - 需要按时间顺序处理文件 diff --git a/docs/zh/connectors/source/HdfsFile.md b/docs/zh/connectors/source/HdfsFile.md index e002cafec6eb..48814806db4c 100644 --- a/docs/zh/connectors/source/HdfsFile.md +++ b/docs/zh/connectors/source/HdfsFile.md @@ -95,6 +95,8 @@ import ChangeLog from '../changelog/connector-file-hadoop.md'; | file_split_size | long | 否 | 134217728 | `enable_file_split=true` 时生效，单位字节。`text`/`csv`/`json` 按 `file_split_size` 拆分并对齐到下一个 `row_delimiter`；`parquet` 以 RowGroup 为拆分单位，不会切开 RowGroup。 | | quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符，可保证包含逗号、换行符或引号的字段被正确解析。 | | escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 | +| metalake_type | string | 否 | gravitino | Metalake 服务类型，目前支持 `gravitino`。 | +| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### file_format_type [string] @@ -346,6 +348,11 @@ abc.* 更多信息请参考 [元数据 SPI](../../introduction/concepts/metadata-spi.md)。 +### recursive_file_scan [boolean] + +是否递归扫描子目录。 +如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 + ### 提示 > 如果您使用 spark/flink，为了使用此连接器，您必须确保您的 spark/flink 集群已经集成了 hadoop。测试过的 hadoop 版本是 2.x。如果您使用 SeaTunnel Engine，则在下载和安装 SeaTunnel Engine 时会自动集成 hadoop jar。您可以检查 `${SEATUNNEL_HOME}/lib` 下的 jar 包来确认这一点。 diff --git a/docs/zh/connectors/source/LocalFile.md b/docs/zh/connectors/source/LocalFile.md index 323171bbdef1..a6c4b1ce9aa3 100644 --- a/docs/zh/connectors/source/LocalFile.md +++ b/docs/zh/connectors/source/LocalFile.md @@ -92,6 +92,8 @@ import ChangeLog from '../changelog/connector-file-local.md'; | file_split_size | long | 否 | 134217728 | | quote_char | string | 否 | - | | escape_char | string | 否 | - | +| metalake_type | string | 否 | gravitino | Metalake 服务类型，目前支持 `gravitino`。 | +| recursive_file_scan | boolean | 否 | true | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### path [string] @@ -536,10 +538,14 @@ compare_mode = "len_mtime" 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 +### recursive_file_scan [boolean] + +是否递归扫描子目录。 +如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 + ### sort_files_by_modification_time [boolean] 是否按修改时间降序排序文件。默认值为 `false`。 - 启用后，文件将按修改时间排序（最新的在前）。适用于以下场景： - 读取具有不断演化的 schema 的文件，且希望 schema 推断使用最新的文件 - 需要按时间顺序处理文件 diff --git a/docs/zh/connectors/source/ObsFile.md b/docs/zh/connectors/source/ObsFile.md index 4a44ea486828..e06e48791e04 100644 --- a/docs/zh/connectors/source/ObsFile.md +++ b/docs/zh/connectors/source/ObsFile.md @@ -80,6 +80,7 @@ import ChangeLog from '../changelog/connector-file-obs.md'; | time_format | string | 否 | HH:mm:ss | 时间类型格式 | | quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符，可保证包含逗号、换行符或引号的字段被正确解析。 | | escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 | +| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### file_format_type [string] @@ -112,6 +113,7 @@ markdown 解析器提取各种元素，包括标题、段落、列表、代码 注意：Markdown 格式仅支持读取，不支持写入。 ### sort_files_by_modification_time [boolean] + 是否按修改时间降序排序文件。默认值为 `false`。 启用后，文件将按修改时间排序（最新的在前）。适用于以下场景： @@ -121,4 +123,3 @@ markdown 解析器提取各种元素，包括标题、段落、列表、代码 ## 变更日志 - diff --git a/docs/zh/connectors/source/OssFile.md b/docs/zh/connectors/source/OssFile.md index 67cc8a29049d..b2018d9cb1a9 100644 --- a/docs/zh/connectors/source/OssFile.md +++ b/docs/zh/connectors/source/OssFile.md @@ -216,6 +216,8 @@ schema { | file_filter_modified_end | string | 否 | - | 按照最后修改时间过滤文件。 要过滤的结束时间(不包括改时间),时间格式是：`yyyy-MM-dd HH:mm:ss` | | quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符，可保证包含逗号、换行符或引号的字段被正确解析。 | | escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 | +| metalake_type | string | 否 | gravitino | Metalake 服务类型，目前支持 `gravitino`。 | +| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### compress_codec [string] @@ -282,10 +284,14 @@ markdown 解析器提取各种元素，包括标题、段落、列表、代码 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 +### recursive_file_scan [boolean] + +是否递归扫描子目录。 +如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 + ### sort_files_by_modification_time [boolean] 是否按修改时间降序排序文件。默认值为 `false`。 - 启用后，文件将按修改时间排序（最新的在前）。适用于以下场景： - 读取具有不断演化的 schema 的文件，且希望 schema 推断使用最新的文件 - 需要按时间顺序处理文件 diff --git a/docs/zh/connectors/source/OssJindoFile.md b/docs/zh/connectors/source/OssJindoFile.md index 7e68d61b2117..a7559992fc13 100644 --- a/docs/zh/connectors/source/OssJindoFile.md +++ b/docs/zh/connectors/source/OssJindoFile.md @@ -80,12 +80,12 @@ import ChangeLog from '../changelog/connector-file-oss-jindo.md'; | file_filter_pattern | string | 否 | - | 文件过滤模式 | | quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符，可保证包含逗号、换行符或引号的字段被正确解析。 | | escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 | +| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### sort_files_by_modification_time [boolean] 是否按修改时间降序排序文件。默认值为 `false`。 - 启用后，文件将按修改时间排序（最新的在前）。适用于以下场景： - 读取具有不断演化的 schema 的文件，且希望 schema 推断使用最新的文件 - 需要按时间顺序处理文件 @@ -122,4 +122,3 @@ markdown 解析器提取各种元素，包括标题、段落、列表、代码 ## 变更日志 - diff --git a/docs/zh/connectors/source/S3File.md b/docs/zh/connectors/source/S3File.md index 6af96d6202cb..e91b7a0758e1 100644 --- a/docs/zh/connectors/source/S3File.md +++ b/docs/zh/connectors/source/S3File.md @@ -225,6 +225,8 @@ schema { | common-options | | 否 | - | 数据源插件通用参数，请参考[数据源通用选项](../common-options/source-common-options.md)了解详情。 | | quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符，可保证包含逗号、换行符或引号的字段被正确解析。 | | escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 | +| metalake_type | string | 否 | gravitino | Metalake 服务类型，目前支持 `gravitino`。 | +| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### delimiter/field_delimiter [string] @@ -247,10 +249,14 @@ schema { 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 +### recursive_file_scan [boolean] + +是否递归扫描子目录。 +如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 + ### sort_files_by_modification_time [boolean] 是否按修改时间降序排序文件。默认值为 `false`。 - 启用后，文件将按修改时间排序（最新的在前）。适用于以下场景： - 读取具有不断演化的 schema 的文件，且希望 schema 推断使用最新的文件 - 需要按时间顺序处理文件 diff --git a/docs/zh/connectors/source/SftpFile.md b/docs/zh/connectors/source/SftpFile.md index ce71dd1cb573..db3dce2ca70b 100644 --- a/docs/zh/connectors/source/SftpFile.md +++ b/docs/zh/connectors/source/SftpFile.md @@ -121,6 +121,8 @@ import ChangeLog from '../changelog/connector-file-sftp.md'; | file_filter_modified_end | string | 否 | - | 按照最后修改时间过滤文件。 要过滤的结束时间(不包括改时间),时间格式是：`yyyy-MM-dd HH:mm:ss` | | quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符，可保证包含逗号、换行符或引号的字段被正确解析。 | | escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符，使其不会结束字段。 | +| metalake_type | string | 否 | gravitino | Metalake 服务类型，目前支持 `gravitino`。 | +| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 | | sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后，在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 | ### file_filter_pattern [string] @@ -386,10 +388,14 @@ compare_mode = "len_mtime" 更多信息请参考 [元数据 SPI](../../introduction/concepts/metadata-spi.md)。 +### recursive_file_scan [boolean] + +是否递归扫描子目录。 +如果设置为 `false`，将忽略子目录，仅扫描指定路径下的文件。 + ### sort_files_by_modification_time [boolean] 是否按修改时间降序排序文件。默认值为 `false`。 - 启用后，文件将按修改时间排序（最新的在前）。适用于以下场景： - 读取具有不断演化的 schema 的文件，且希望 schema 推断使用最新的文件 - 需要按时间顺序处理文件 diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java index 99ec07fb51e3..8828eef0c729 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java @@ -225,6 +225,14 @@ public class FileBaseSourceOptions extends FileBaseOptions { .withDescription( "A single character that allows the quote or other special characters to appear inside a CSV field without ending the field."); + public static final Option RECURSIVE_FILE_SCAN = + Options.key("recursive_file_scan") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to recursively scan subdirectories. " + + "If false, subdirectories will be ignored."); + public static final Option SORT_FILES_BY_MOD_TIME = Options.key("sort_files_by_modification_time") .booleanType() diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java index 918e4641b54a..01cfb40e0aa9 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java @@ -131,6 +131,7 @@ public abstract class AbstractReadStrategy implements ReadStrategy { protected transient HadoopFileSystemProxy targetHadoopFileSystemProxy; protected transient boolean shareTargetFileSystemProxy; protected transient boolean checksumUnavailableWarned; + protected boolean recursiveFileScan = true; protected boolean sortFilesByModTime = FileBaseSourceOptions.SORT_FILES_BY_MOD_TIME.defaultValue(); @@ -190,7 +191,7 @@ private void collectFileInfoByPath( throws IOException { FileStatus[] stats = hadoopFileSystemProxy.listStatus(path); for (FileStatus fileStatus : stats) { - if (fileStatus.isDirectory()) { + if (fileStatus.isDirectory() && recursiveFileScan) { // skip hidden tmp directory, such as .hive-staging_hive if (!fileStatus.getPath().getName().startsWith(".")) { collectFileInfoByPath( @@ -359,6 +360,12 @@ public void setPluginConfig(Config pluginConfig) { pluginConfig.getString(FileBaseSourceOptions.SYNC_MODE.key()), FileBaseSourceOptions.SYNC_MODE.key()); } + + if (pluginConfig.hasPath(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key())) { + recursiveFileScan = + pluginConfig.getBoolean(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key()); + } + enableUpdateSync = syncMode == FileSyncMode.UPDATE; if (enableUpdateSync) { validateUpdateSyncConfig(pluginConfig); diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumerator.java index 93605c447160..34041efcbff8 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumerator.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumerator.java @@ -285,7 +285,7 @@ private void scanOnce() throws IOException { int queued = 0; Set activeKnownSplitIds = new HashSet<>(); for (TableScanContext ctx : tableScanContexts) { - List files = ctx.listFilesRecursively(ctx.rootPath); + List files = ctx.listFiles(ctx.rootPath); scanned += files.size(); for (FileStatus fileStatus : files) { if (!ctx.shouldProcess(fileStatus, jobStartTimeMillis, startMode)) { @@ -526,6 +526,7 @@ private static final class TableScanContext implements AutoCloseable { private final FileUpdateStrategy updateStrategy; private final FileCompareMode compareMode; private boolean checksumUnavailableWarned; + private final boolean recursiveFileScan; private final Pattern pattern; private final String fileBasePath; @@ -568,6 +569,7 @@ private TableScanContext( this.updateStrategy = config.get(FileBaseSourceOptions.UPDATE_STRATEGY); this.compareMode = config.get(FileBaseSourceOptions.COMPARE_MODE); + this.recursiveFileScan = config.get(FileBaseSourceOptions.RECURSIVE_FILE_SCAN); String targetPath = config.get(FileBaseSourceOptions.TARGET_PATH); Map targetHadoopConf = @@ -588,14 +590,14 @@ private List toSplits(FileStatus fileStatus) { return fileSplitStrategy.split(tableId, fileStatus.getPath().toString()); } - private List listFilesRecursively(String path) throws IOException { + private List listFiles(String path) throws IOException { List files = new ArrayList<>(); FileStatus[] statuses = sourceFs.listStatus(path); for (FileStatus status : statuses) { if (status.isDirectory()) { String name = status.getPath().getName(); - if (!name.startsWith(".")) { - files.addAll(listFilesRecursively(status.getPath().toString())); + if (recursiveFileScan && !name.startsWith(".")) { + files.addAll(listFiles(status.getPath().toString())); } continue; } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java index 69481f30d58d..a8b74b281899 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java @@ -47,6 +47,8 @@ import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; +import lombok.extern.slf4j.Slf4j; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -61,6 +63,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; +@Slf4j public class AbstractReadStrategyTest { @Test @@ -505,4 +508,169 @@ private static void assertSetCatalogTableWithEmptyFileNames( SeaTunnelRowType actualRowType = readStrategy.getActualSeaTunnelRowTypeInfo(); Assertions.assertArrayEquals(new String[] {"id", "dt"}, actualRowType.getFieldNames()); } + + @DisabledOnOs(OS.WINDOWS) + @Test + public void testNonRecursiveFileScan() throws Exception { + String baseDir = "/tmp/test_recursive"; + String file1 = baseDir + "/file1.txt"; + String file2 = baseDir + "/file2.txt"; + String subdirFile = baseDir + "/subdir/file3.txt"; + + try { + createTestFiles(file1, file2, subdirFile); + + Map config = new HashMap<>(); + config.put(FileBaseSourceOptions.FILE_PATH.key(), baseDir); + config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "text"); + config.put(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key(), false); + + Config pluginConfig = ConfigFactory.parseMap(config); + + try (TextReadStrategy strategy = new TextReadStrategy()) { + LocalFileSystemConf.LocalConf localConf = + new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT); + strategy.init(localConf); + strategy.setPluginConfig(pluginConfig); + + List fileNames = strategy.getFileNamesByPath(baseDir); + + Assertions.assertEquals(2, fileNames.size()); + Assertions.assertTrue(fileNames.stream().noneMatch(f -> f.contains("subdir"))); + Assertions.assertTrue(fileNames.stream().anyMatch(f -> f.endsWith("file1.txt"))); + Assertions.assertTrue(fileNames.stream().anyMatch(f -> f.endsWith("file2.txt"))); + } + + } finally { + deleteTestDirectory(baseDir); + } + } + + @DisabledOnOs(OS.WINDOWS) + @Test + public void testRecursiveFileScanDefault() throws Exception { + String baseDir = "/tmp/test_default"; + String file1 = baseDir + "/file1.txt"; + String subdirFile = baseDir + "/subdir/file2.txt"; + + try { + createTestFiles(file1, subdirFile); + + Map config = new HashMap<>(); + config.put(FileBaseSourceOptions.FILE_PATH.key(), baseDir); + config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "text"); + + Config pluginConfig = ConfigFactory.parseMap(config); + + try (TextReadStrategy strategy = new TextReadStrategy()) { + LocalFileSystemConf.LocalConf localConf = + new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT); + strategy.init(localConf); + strategy.setPluginConfig(pluginConfig); + + List fileNames = strategy.getFileNamesByPath(baseDir); + + Assertions.assertEquals(2, fileNames.size()); + Assertions.assertTrue(fileNames.stream().anyMatch(f -> f.contains("file1.txt"))); + Assertions.assertTrue(fileNames.stream().anyMatch(f -> f.contains("subdir"))); + Assertions.assertTrue( + fileNames.stream().anyMatch(f -> f.contains("subdir/file2.txt"))); + } + } finally { + deleteTestDirectory(baseDir); + } + } + + @DisabledOnOs(OS.WINDOWS) + @Test + public void testNonRecursiveEmptyDirectory() throws Exception { + String baseDir = "/tmp/test_empty"; + String subdirDir = baseDir + "/subdir"; + String subdirFile = baseDir + "/subdir/file.txt"; + createTestFiles(subdirFile); + + try { + Configuration conf = new Configuration(); + Path path = new Path(subdirDir); + org.apache.hadoop.fs.FileSystem fs = path.getFileSystem(conf); + fs.mkdirs(path); + + Map config = new HashMap<>(); + config.put(FileBaseSourceOptions.FILE_PATH.key(), baseDir); + config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "text"); + config.put(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key(), false); + + Config pluginConfig = ConfigFactory.parseMap(config); + + try (TextReadStrategy strategy = new TextReadStrategy()) { + LocalFileSystemConf.LocalConf localConf = + new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT); + strategy.init(localConf); + strategy.setPluginConfig(pluginConfig); + + List fileNames = strategy.getFileNamesByPath(baseDir); + + Assertions.assertEquals(0, fileNames.size()); + } + } finally { + deleteTestDirectory(baseDir); + } + } + + @DisabledOnOs(OS.WINDOWS) + @Test + public void testNonRecursiveWithOnlySubdirectories() throws Exception { + String baseDir = "/tmp/test_only_subdir"; + String subdirFile1 = baseDir + "/subdir1/file1.txt"; + String subdirFile2 = baseDir + "/subdir2/file2.txt"; + + try { + createTestFiles(subdirFile1, subdirFile2); + + Map config = new HashMap<>(); + config.put(FileBaseSourceOptions.FILE_PATH.key(), baseDir); + config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "text"); + config.put(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key(), false); + + Config pluginConfig = ConfigFactory.parseMap(config); + + try (TextReadStrategy strategy = new TextReadStrategy()) { + LocalFileSystemConf.LocalConf localConf = + new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT); + strategy.init(localConf); + strategy.setPluginConfig(pluginConfig); + + List fileNames = strategy.getFileNamesByPath(baseDir); + + Assertions.assertEquals(0, fileNames.size()); + } + } finally { + deleteTestDirectory(baseDir); + } + } + + private void createTestFiles(String... filePaths) throws IOException { + Configuration conf = new Configuration(); + for (String filePath : filePaths) { + Path path = new Path(filePath); + org.apache.hadoop.fs.FileSystem fs = path.getFileSystem(conf); + + fs.mkdirs(path.getParent()); + + try (org.apache.hadoop.fs.FSDataOutputStream out = fs.create(path)) { + out.writeBytes("test content"); + } + } + } + + private void deleteTestDirectory(String dirPath) { + try { + Path path = new Path(dirPath); + Configuration conf = new Configuration(); + org.apache.hadoop.fs.FileSystem fs = path.getFileSystem(conf); + fs.delete(path, true); + } catch (Exception e) { + log.error("Warning: Failed to delete test directory: " + dirPath, e); + } + } } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/UpdateSyncModeTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/UpdateSyncModeTest.java index fff09538f514..d058b707d47f 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/UpdateSyncModeTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/UpdateSyncModeTest.java @@ -203,6 +203,73 @@ void testStrictChecksumCopyWhenSameLengthButDifferentContent() throws Exception } } + @Test + void testUpdateModeNonRecursiveScanOnlyComparesTopLevelFiles() throws Exception { + Path sourceDir = tempDir.resolve("src"); + Path targetDir = tempDir.resolve("dst"); + Path topLevelSourceFile = sourceDir.resolve("root.bin"); + Path nestedSourceFile = sourceDir.resolve("subdir/nested.bin"); + Path topLevelTargetFile = targetDir.resolve("root.bin"); + Path nestedTargetFile = targetDir.resolve("subdir/nested.bin"); + + writeFile(topLevelSourceFile, "root".getBytes()); + writeFile(nestedSourceFile, "nested".getBytes()); + writeFile(topLevelTargetFile, "root".getBytes()); + writeFile(nestedTargetFile, "nested".getBytes()); + setMtime(topLevelSourceFile, 2_000); + setMtime(topLevelTargetFile, 1_000); + setMtime(nestedSourceFile, 2_000); + setMtime(nestedTargetFile, 1_000); + + try (BinaryReadStrategy strategy = new BinaryReadStrategy()) { + strategy.setPluginConfig( + updateConfig( + sourceDir.toUri().toString(), + targetDir.toUri().toString(), + "distcp", + "len_mtime", + false)); + strategy.init(new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT)); + + List files = strategy.getFileNamesByPath(sourceDir.toUri().toString()); + Assertions.assertEquals(1, files.size()); + Assertions.assertTrue(files.get(0).endsWith("/root.bin")); + } + } + + @Test + void testUpdateModeNonRecursiveScanSkipsNestedChanges() throws Exception { + Path sourceDir = tempDir.resolve("src"); + Path targetDir = tempDir.resolve("dst"); + Path topLevelSourceFile = sourceDir.resolve("root.bin"); + Path nestedSourceFile = sourceDir.resolve("subdir/nested.bin"); + Path topLevelTargetFile = targetDir.resolve("root.bin"); + Path nestedTargetFile = targetDir.resolve("subdir/nested.bin"); + + writeFile(topLevelSourceFile, "root".getBytes()); + writeFile(nestedSourceFile, "nested".getBytes()); + writeFile(topLevelTargetFile, "root".getBytes()); + writeFile(nestedTargetFile, "nested".getBytes()); + setMtime(topLevelSourceFile, 1_000); + setMtime(topLevelTargetFile, 1_000); + setMtime(nestedSourceFile, 2_000); + setMtime(nestedTargetFile, 1_000); + + try (BinaryReadStrategy strategy = new BinaryReadStrategy()) { + strategy.setPluginConfig( + updateConfig( + sourceDir.toUri().toString(), + targetDir.toUri().toString(), + "distcp", + "len_mtime", + false)); + strategy.init(new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT)); + + List files = strategy.getFileNamesByPath(sourceDir.toUri().toString()); + Assertions.assertTrue(files.isEmpty(), "Nested-only changes should be skipped"); + } + } + private static void writeFile(Path path, byte[] content) throws IOException { Files.createDirectories(path.getParent()); Files.write(path, content); @@ -214,6 +281,15 @@ private static void setMtime(Path path, long millis) throws IOException { private static Config updateConfig( String sourcePath, String targetPath, String updateStrategy, String compareMode) { + return updateConfig(sourcePath, targetPath, updateStrategy, compareMode, true); + } + + private static Config updateConfig( + String sourcePath, + String targetPath, + String updateStrategy, + String compareMode, + boolean recursiveFileScan) { Map configMap = new HashMap<>(); configMap.put("path", sourcePath); configMap.put("file_format_type", "binary"); @@ -221,6 +297,7 @@ private static Config updateConfig( configMap.put("target_path", targetPath); configMap.put("update_strategy", updateStrategy); configMap.put("compare_mode", compareMode); + configMap.put("recursive_file_scan", recursiveFileScan); return ConfigFactory.parseMap(configMap); } } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumeratorTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumeratorTest.java index dd3bcb423a39..c9178aa19411 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumeratorTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumeratorTest.java @@ -43,9 +43,12 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.FileTime; +import java.time.Duration; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; @@ -382,7 +385,7 @@ void testContinuousDiscoveryRequiresPositiveScanInterval() throws Exception { config.put(FileBaseSourceOptions.TARGET_PATH.key(), dstDir.toString()); config.put(FileBaseSourceOptions.UPDATE_STRATEGY.key(), "distcp"); config.put(FileBaseSourceOptions.COMPARE_MODE.key(), "len_mtime"); - config.put(FileBaseSourceOptions.SCAN_INTERVAL.key(), "0S"); + config.put(FileBaseSourceOptions.SCAN_INTERVAL.key(), Duration.ZERO); ReadonlyConfig readonlyConfig = ReadonlyConfig.fromMap(config); @@ -421,10 +424,93 @@ void testContinuousDiscoveryRequiresPositiveScanInterval() throws Exception { "continuous mode should require a positive scan_interval"); } + @Test + void testContinuousDiscoveryWithNonRecursiveFileScan() throws Exception { + Path srcDir = Files.createDirectories(tempDir.resolve("src_recursive_disabled")); + Path dstDir = Files.createDirectories(tempDir.resolve("dst_recursive_disabled")); + Path rootFile = srcDir.resolve("root.bin"); + Path nestedFile = Files.createDirectories(srcDir.resolve("nested")).resolve("nested.bin"); + Files.write(rootFile, "abc".getBytes()); + Files.write(nestedFile, "def".getBytes()); + + EnumeratorWithContext enumeratorWithContext = createEnumerator(srcDir, dstDir, false); + try { + enumeratorWithContext.enumerator.scanOnceForTest(); + Assertions.assertEquals( + 1, enumeratorWithContext.enumerator.currentUnassignedSplitSize()); + + List filePaths = assignAndCaptureFilePaths(enumeratorWithContext); + Assertions.assertEquals(1, filePaths.size()); + Assertions.assertTrue(filePaths.get(0).endsWith(rootFile.toString())); + } finally { + enumeratorWithContext.enumerator.close(); + } + } + + @Test + void testContinuousDiscoveryWithDefaultRecursiveFileScan() throws Exception { + Path srcDir = Files.createDirectories(tempDir.resolve("src_recursive_default")); + Path dstDir = Files.createDirectories(tempDir.resolve("dst_recursive_default")); + Path rootFile = srcDir.resolve("root.bin"); + Path nestedFile = Files.createDirectories(srcDir.resolve("nested")).resolve("nested.bin"); + Files.write(rootFile, "abc".getBytes()); + Files.write(nestedFile, "def".getBytes()); + + EnumeratorWithContext enumeratorWithContext = createEnumerator(srcDir, dstDir); + try { + enumeratorWithContext.enumerator.scanOnceForTest(); + Assertions.assertEquals( + 2, enumeratorWithContext.enumerator.currentUnassignedSplitSize()); + + List filePaths = assignAndCaptureFilePaths(enumeratorWithContext); + Assertions.assertTrue( + filePaths.stream().anyMatch(path -> path.endsWith(rootFile.toString()))); + Assertions.assertTrue( + filePaths.stream().anyMatch(path -> path.endsWith(nestedFile.toString()))); + } finally { + enumeratorWithContext.enumerator.close(); + } + } + + @Test + void testContinuousDiscoveryWithRecursiveFileScan() throws Exception { + Path srcDir = Files.createDirectories(tempDir.resolve("src_recursive_enabled")); + Path dstDir = Files.createDirectories(tempDir.resolve("dst_recursive_enabled")); + Path rootFile = srcDir.resolve("root.bin"); + Path nestedFile = Files.createDirectories(srcDir.resolve("nested")).resolve("nested.bin"); + Files.write(rootFile, "abc".getBytes()); + Files.write(nestedFile, "def".getBytes()); + + EnumeratorWithContext enumeratorWithContext = createEnumerator(srcDir, dstDir, true); + try { + enumeratorWithContext.enumerator.scanOnceForTest(); + Assertions.assertEquals( + 2, enumeratorWithContext.enumerator.currentUnassignedSplitSize()); + + List filePaths = assignAndCaptureFilePaths(enumeratorWithContext); + Assertions.assertTrue( + filePaths.stream().anyMatch(path -> path.endsWith(rootFile.toString()))); + Assertions.assertTrue( + filePaths.stream().anyMatch(path -> path.endsWith(nestedFile.toString()))); + } finally { + enumeratorWithContext.enumerator.close(); + } + } + private EnumeratorWithContext createEnumerator(Path srcDir, Path dstDir) throws IOException { return createEnumerator(srcDir, dstDir, "earliest"); } + private EnumeratorWithContext createEnumerator( + Path srcDir, Path dstDir, boolean recursiveFileScan) throws IOException { + return createEnumerator( + srcDir, + dstDir, + "earliest", + new FileSourceState(Collections.emptySet()), + recursiveFileScan); + } + private EnumeratorWithContext createEnumerator(Path srcDir, Path dstDir, String startMode) throws IOException { return createEnumerator( @@ -434,6 +520,16 @@ private EnumeratorWithContext createEnumerator(Path srcDir, Path dstDir, String private EnumeratorWithContext createEnumerator( Path srcDir, Path dstDir, String startMode, FileSourceState checkpointState) throws IOException { + return createEnumerator(srcDir, dstDir, startMode, checkpointState, null); + } + + private EnumeratorWithContext createEnumerator( + Path srcDir, + Path dstDir, + String startMode, + FileSourceState checkpointState, + Boolean recursiveFileScan) + throws IOException { Map config = new HashMap<>(); config.put(FileBaseSourceOptions.FILE_PATH.key(), srcDir.toString()); config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "binary"); @@ -443,6 +539,9 @@ private EnumeratorWithContext createEnumerator( config.put(FileBaseSourceOptions.TARGET_PATH.key(), dstDir.toString()); config.put(FileBaseSourceOptions.UPDATE_STRATEGY.key(), "distcp"); config.put(FileBaseSourceOptions.COMPARE_MODE.key(), "len_mtime"); + if (recursiveFileScan != null) { + config.put(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key(), recursiveFileScan); + } ReadonlyConfig readonlyConfig = ReadonlyConfig.fromMap(config); @@ -478,6 +577,19 @@ private EnumeratorWithContext createEnumerator( return new EnumeratorWithContext(enumerator, context); } + private static List assignAndCaptureFilePaths( + EnumeratorWithContext enumeratorWithContext) { + enumeratorWithContext.enumerator.handleSplitRequest(0); + ArgumentCaptor> splitsCaptor = + ArgumentCaptor.forClass((Class) java.util.List.class); + Mockito.verify(enumeratorWithContext.context) + .assignSplit(Mockito.eq(0), splitsCaptor.capture()); + return splitsCaptor.getValue().stream() + .map(FileSourceSplit::getFilePath) + .sorted() + .collect(Collectors.toList()); + } + private static final class EnumeratorWithContext { private final ContinuousMultipleTableFileSourceSplitEnumerator enumerator; private final SourceSplitEnumerator.Context context; diff --git a/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java index 68704b920e9c..50dfd7e91ea5 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java @@ -101,6 +101,7 @@ public OptionRule optionRule() { FileBaseSourceOptions.MARKDOWN_RAG_METADATA_ENABLED) .optional(FileBaseSourceOptions.QUOTE_CHAR) .optional(FileBaseSourceOptions.ESCAPE_CHAR) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java index 79499cbf4707..4c48a411fb49 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java @@ -122,6 +122,7 @@ public OptionRule optionRule() { FileBaseSourceOptions.SYNC_MODE, FileSyncMode.UPDATE, FileBaseSourceOptions.TARGET_PATH) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java index de9a088b7ec0..215a8b8b748b 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java @@ -25,6 +25,7 @@ import org.apache.seatunnel.api.table.factory.Factory; import org.apache.seatunnel.api.table.factory.TableSourceFactory; import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; +import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions; import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat; import org.apache.seatunnel.connectors.seatunnel.file.config.FileSyncMode; import org.apache.seatunnel.connectors.seatunnel.file.config.FileSystemType; @@ -133,6 +134,7 @@ public OptionRule optionRule() { .optional(HdfsFileSourceOptions.QUOTE_CHAR) .optional(HdfsFileSourceOptions.ESCAPE_CHAR) .optional(ConnectorCommonOptions.METALAKE_TYPE) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java index d07ab8fffe73..75939c1ffb89 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java @@ -96,6 +96,7 @@ public OptionRule optionRule() { FileBaseSourceOptions.MARKDOWN_RAG_METADATA_ENABLED) .optional(FileBaseSourceOptions.QUOTE_CHAR) .optional(FileBaseSourceOptions.ESCAPE_CHAR) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java index 5cd72b651312..919eaecd3100 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java @@ -129,6 +129,7 @@ public OptionRule optionRule() { FileBaseSourceOptions.SYNC_MODE, FileSyncMode.UPDATE, FileBaseSourceOptions.TARGET_PATH) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java index 1f9773795257..44e079c6c1b1 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java @@ -85,6 +85,7 @@ public OptionRule optionRule() { FileBaseSourceOptions.MARKDOWN_RAG_METADATA_ENABLED) .optional(FileBaseSourceOptions.QUOTE_CHAR) .optional(FileBaseSourceOptions.ESCAPE_CHAR) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java index b620f0628971..8fc59b4db2c8 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java @@ -106,6 +106,7 @@ public OptionRule optionRule() { .optional(FileBaseSourceOptions.QUOTE_CHAR) .optional(FileBaseSourceOptions.ESCAPE_CHAR) .optional(ConnectorCommonOptions.METALAKE_TYPE) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java index f982fcce4abc..89194eff0c6c 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java @@ -122,6 +122,7 @@ public OptionRule optionRule() { .optional(FileBaseSourceOptions.QUOTE_CHAR) .optional(FileBaseSourceOptions.ESCAPE_CHAR) .optional(ConnectorCommonOptions.METALAKE_TYPE) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java index 4ce72d9eb4d1..acf72f17e9bd 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java @@ -110,6 +110,7 @@ public OptionRule optionRule() { FileBaseSourceOptions.SYNC_MODE, FileSyncMode.UPDATE, FileBaseSourceOptions.TARGET_PATH) + .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN) .build(); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/ftp/FtpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/ftp/FtpFileIT.java index da8f53399534..8bd3d510ca99 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/ftp/FtpFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/ftp/FtpFileIT.java @@ -171,6 +171,26 @@ public void startUp() throws Exception { ContainerUtil.copyFileIntoContainers( "/excel/e2e.xlsx", ftpHomeDir + "/e2e.xlsx", ftpContainer); + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + ftpHomeDir + "/tmp/seatunnel/read/recursive/e2e.txt", + ftpContainer); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + ftpHomeDir + "/tmp/seatunnel/read/recursive/subdir/e2e.txt", + ftpContainer); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + ftpHomeDir + "/tmp/seatunnel/read/recursive/subdir/deeper/e2e.txt", + ftpContainer); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + ftpHomeDir + "/tmp/seatunnel/read/recursive/subdir/deeper/final/e2e.txt", + ftpContainer); + ftpContainer.execInContainer("sh", "-c", "chmod -R 777 " + ftpHomeDir + "/"); ftpContainer.execInContainer("sh", "-c", "chown -R ftp:ftp " + ftpHomeDir + "/"); } @@ -302,6 +322,95 @@ public void testFtpBinaryUpdateModeContinuousDiscoveryDistcp(TestContainer conta deleteFileFromContainer(ftpHomeDir + "/tmp/seatunnel/continuous"); } + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.FLINK, EngineType.SPARK}, + disabledReason = "Continuous discovery is a long-running job; only run in zeta engine.") + public void testFtpBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan( + TestContainer container) throws IOException, InterruptedException { + resetContinuousTestPath(); + + String jobId = String.valueOf(JobIdGenerator.newJobId()); + CompletableFuture jobFuture = + CompletableFuture.supplyAsync( + () -> { + try { + return container.executeJob( + "/text/ftp_binary_update_distcp_continuous_non_recursive.conf", + jobId); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + putFtpFile("/tmp/seatunnel/continuous/src/root.bin", "root"); + putFtpFile("/tmp/seatunnel/continuous/src/subdir/nested.bin", "nested"); + + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertEquals( + "root", + readFtpFile("/tmp/seatunnel/continuous/dst/root.bin"))); + + Thread.sleep(3000); + Assertions.assertFalse(isFtpFileExists("/tmp/seatunnel/continuous/dst/subdir/nested.bin")); + + Container.ExecResult cancelResult = container.cancelJob(jobId); + Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr()); + + Container.ExecResult execResult; + try { + execResult = jobFuture.get(120, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException("Wait continuous job exit failed.", e); + } + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + + deleteFileFromContainer(ftpHomeDir + "/tmp/seatunnel/continuous"); + } + + @TestTemplate + public void testFtpBinaryUpdateModeDistcpWithNonRecursiveScan(TestContainer container) + throws IOException, InterruptedException { + resetUpdateTestPath(); + putFtpFile("/tmp/seatunnel/update/src/root.bin", "root-updated-v2"); + putFtpFile("/tmp/seatunnel/update/src/subdir/nested.bin", "nest-updated-v2"); + putFtpFile("/tmp/seatunnel/update/dst/root.bin", "root-stale-v1"); + putFtpFile("/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-stale-v1"); + + Container.ExecResult execResult = + container.executeJob("/text/ftp_binary_update_non_recursive_distcp.conf"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + Assertions.assertEquals( + "root-updated-v2", readFtpFile("/tmp/seatunnel/update/dst/root.bin")); + Assertions.assertEquals( + "nest-stale-v1", readFtpFile("/tmp/seatunnel/update/dst/subdir/nested.bin")); + + deleteFileFromContainer(ftpHomeDir + "/tmp/seatunnel/update"); + } + + @TestTemplate + public void testFtpBinaryUpdateModeStrictChecksumSkipsNestedChangesWithNonRecursiveScan( + TestContainer container) throws IOException, InterruptedException { + resetUpdateTestPath(); + putFtpFile("/tmp/seatunnel/update/src/root.bin", "root-same-v1"); + putFtpFile("/tmp/seatunnel/update/src/subdir/nested.bin", "nest-new-v1"); + putFtpFile("/tmp/seatunnel/update/dst/root.bin", "root-same-v1"); + putFtpFile("/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-old-v1"); + + Container.ExecResult execResult = + container.executeJob("/text/ftp_binary_update_non_recursive_strict_checksum.conf"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + Assertions.assertEquals("root-same-v1", readFtpFile("/tmp/seatunnel/update/dst/root.bin")); + Assertions.assertEquals( + "nest-old-v1", readFtpFile("/tmp/seatunnel/update/dst/subdir/nested.bin")); + + deleteFileFromContainer(ftpHomeDir + "/tmp/seatunnel/update"); + } + @TestTemplate public void testFtpToAssertForJsonFilter(TestContainer container) throws IOException, InterruptedException { @@ -391,14 +500,9 @@ public void testFtpFileReadAndWrite(TestContainer container) helper.execute("/excel/fake_source_to_ftp_root_path_excel.conf"); // test ftp source support multipleTable - String homePath = ftpHomeDir; - String sink01 = "/tmp/seatunnel/json/sink/multiplesource/fake01"; - String sink02 = "/tmp/seatunnel/json/sink/multiplesource/fake02"; - deleteFileFromContainer(homePath + sink01); - deleteFileFromContainer(homePath + sink02); - helper.execute("/json/ftp_file_json_to_assert_with_multipletable.conf"); - Assertions.assertEquals(getFileListFromContainer(homePath + sink01).size(), 1); - Assertions.assertEquals(getFileListFromContainer(homePath + sink02).size(), 1); + // test read recursive file path + helper.execute("/text/ftp_file_text_recursive_to_assert.conf"); + helper.execute("/text/ftp_file_text_non_recursive_to_assert.conf"); } @TestTemplate @@ -550,6 +654,13 @@ private String readFtpFile(String ftpPath) throws IOException, InterruptedExcept return catResult.getStdout() == null ? "" : catResult.getStdout().trim(); } + private boolean isFtpFileExists(String ftpPath) throws IOException, InterruptedException { + String containerPath = ftpHomeDir + ftpPath; + Container.ExecResult result = + ftpContainer.execInContainer("sh", "-c", "test -f '" + containerPath + "'"); + return result.getExitCode() == 0; + } + private long getFtpFileMtimeSeconds(String ftpPath) throws IOException, InterruptedException { String containerPath = ftpHomeDir + ftpPath; Container.ExecResult result = diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_distcp_continuous_non_recursive.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_distcp_continuous_non_recursive.conf new file mode 100644 index 000000000000..3d88995203ab --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_distcp_continuous_non_recursive.conf @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + FtpFile { + host = "ftp" + port = 21 + user = seatunnel + password = pass + + path = "/tmp/seatunnel/continuous/src" + file_format_type = "binary" + + discovery_mode = "continuous" + scan_interval = "1S" + start_mode = "earliest" + + sync_mode = "update" + target_path = "/tmp/seatunnel/continuous/dst" + update_strategy = "distcp" + compare_mode = "len_mtime" + recursive_file_scan = false + } +} + +sink { + FtpFile { + host = "ftp" + port = 21 + user = seatunnel + password = pass + + path = "/tmp/seatunnel/continuous/dst" + tmp_path = "/tmp/seatunnel/continuous/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_distcp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_distcp.conf new file mode 100644 index 000000000000..45e1cb414d34 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_distcp.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FtpFile { + host = "ftp" + port = 21 + user = seatunnel + password = pass + + path = "/tmp/seatunnel/update/src" + file_format_type = "binary" + + sync_mode = "update" + target_path = "/tmp/seatunnel/update/dst" + update_strategy = "distcp" + compare_mode = "len_mtime" + recursive_file_scan = false + } +} + +sink { + FtpFile { + host = "ftp" + port = 21 + user = seatunnel + password = pass + + path = "/tmp/seatunnel/update/dst" + tmp_path = "/tmp/seatunnel/update/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_strict_checksum.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_strict_checksum.conf new file mode 100644 index 000000000000..e1df788853a0 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_strict_checksum.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FtpFile { + host = "ftp" + port = 21 + user = seatunnel + password = pass + + path = "/tmp/seatunnel/update/src" + file_format_type = "binary" + + sync_mode = "update" + target_path = "/tmp/seatunnel/update/dst" + update_strategy = "strict" + compare_mode = "checksum" + recursive_file_scan = false + } +} + +sink { + FtpFile { + host = "ftp" + port = 21 + user = seatunnel + password = pass + + path = "/tmp/seatunnel/update/dst" + tmp_path = "/tmp/seatunnel/update/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_non_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_non_recursive_to_assert.conf new file mode 100644 index 000000000000..274b0a91545c --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_non_recursive_to_assert.conf @@ -0,0 +1,121 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + FtpFile { + host = "ftp" + port = 21 + user = seatunnel + password = pass + path = "/tmp/seatunnel/read/recursive" + file_format_type = "text" + recursive_file_scan = false + plugin_output = "ftp" + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + } +} + +sink { + Assert { + plugin_input = "ftp" + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 5 + }, + { + rule_type = MIN_ROW + rule_value = 5 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_recursive_to_assert.conf new file mode 100644 index 000000000000..3bf4b871fdb6 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_recursive_to_assert.conf @@ -0,0 +1,121 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + FtpFile { + host = "ftp" + port = 21 + user = seatunnel + password = pass + path = "/tmp/seatunnel/read/recursive" + file_format_type = "text" + recursive_file_scan = true + plugin_output = "ftp" + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + } +} + +sink { + Assert { + plugin_input = "ftp" + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 20 + }, + { + rule_type = MIN_ROW + rule_value = 20 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/hdfs/HdfsFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/hdfs/HdfsFileIT.java index f8c6d97fe6a1..0577eaf42dbc 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/hdfs/HdfsFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/hdfs/HdfsFileIT.java @@ -23,6 +23,7 @@ import org.apache.seatunnel.e2e.common.container.TestContainer; import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer; import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; +import org.apache.seatunnel.e2e.common.util.ContainerUtil; import org.apache.seatunnel.e2e.common.util.JobIdGenerator; import org.awaitility.Awaitility; @@ -170,6 +171,24 @@ public void testHdfsReadEmptyTextDirectory(TestContainer container) Assertions.assertEquals(0, readResult.getExitCode()); } + @TestTemplate + public void testHdfsReadRecursiveTextDirectory(TestContainer container) + throws IOException, InterruptedException { + resetRecursiveTestPath(); + putHdfsClasspathFile("/text/e2e.txt", "/recursive/e2e.txt"); + putHdfsClasspathFile("/text/e2e.txt", "/recursive/subdir/e2e.txt"); + putHdfsClasspathFile("/text/e2e.txt", "/recursive/subdir/deeper/e2e.txt"); + putHdfsClasspathFile("/text/e2e.txt", "/recursive/subdir/deeper/final/e2e.txt"); + + org.testcontainers.containers.Container.ExecResult recursiveResult = + container.executeJob("/hdfs_text_recursive_to_assert.conf"); + Assertions.assertEquals(0, recursiveResult.getExitCode()); + + org.testcontainers.containers.Container.ExecResult nonRecursiveResult = + container.executeJob("/hdfs_text_non_recursive_to_assert.conf"); + Assertions.assertEquals(0, nonRecursiveResult.getExitCode()); + } + @TestTemplate public void testHdfsBinaryUpdateModeDistcp(TestContainer container) throws IOException, InterruptedException { @@ -275,6 +294,91 @@ public void testHdfsBinaryUpdateModeContinuousDiscoveryDistcp(TestContainer cont nameNode.execInContainer("bash", "-c", "hdfs dfs -rm -r -f /continuous || true"); } + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.FLINK, EngineType.SPARK}, + disabledReason = "Continuous discovery is a long-running job; only run in zeta engine.") + public void testHdfsBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan( + TestContainer container) throws IOException, InterruptedException { + resetContinuousTestPath(); + + String jobId = String.valueOf(JobIdGenerator.newJobId()); + CompletableFuture jobFuture = + CompletableFuture.supplyAsync( + () -> { + try { + return container.executeJob( + "/hdfs_binary_update_distcp_continuous_non_recursive.conf", + jobId); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + createHdfsDirectories("/continuous/src/subdir"); + putHdfsFile("/continuous/src/root.bin", "root"); + putHdfsFile("/continuous/src/subdir/nested.bin", "nested"); + + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertEquals( + "root", readHdfsFile("/continuous/dst/root.bin"))); + + Thread.sleep(3000); + Assertions.assertFalse(isHdfsFileExists("/continuous/dst/subdir/nested.bin")); + + org.testcontainers.containers.Container.ExecResult cancelResult = + container.cancelJob(jobId); + Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr()); + + org.testcontainers.containers.Container.ExecResult execResult; + try { + execResult = jobFuture.get(120, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException("Wait continuous job exit failed.", e); + } + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + + nameNode.execInContainer("bash", "-c", "hdfs dfs -rm -r -f /continuous || true"); + } + + @TestTemplate + public void testHdfsBinaryUpdateModeDistcpWithNonRecursiveScan(TestContainer container) + throws IOException, InterruptedException { + resetUpdateTestPath(); + createHdfsDirectories("/update/src/subdir", "/update/dst/subdir"); + putHdfsFile("/update/src/root.bin", "root-updated-v2"); + putHdfsFile("/update/src/subdir/nested.bin", "nest-updated-v2"); + putHdfsFile("/update/dst/root.bin", "root-stale-v1"); + putHdfsFile("/update/dst/subdir/nested.bin", "nest-stale-v1"); + + org.testcontainers.containers.Container.ExecResult execResult = + container.executeJob("/hdfs_binary_update_non_recursive_distcp.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + Assertions.assertEquals("root-updated-v2", readHdfsFile("/update/dst/root.bin")); + Assertions.assertEquals("nest-stale-v1", readHdfsFile("/update/dst/subdir/nested.bin")); + } + + @TestTemplate + public void testHdfsBinaryUpdateModeStrictChecksumSkipsNestedChangesWithNonRecursiveScan( + TestContainer container) throws IOException, InterruptedException { + resetUpdateTestPath(); + createHdfsDirectories("/update/src/subdir", "/update/dst/subdir"); + putHdfsFile("/update/src/root.bin", "root-same-v1"); + putHdfsFile("/update/src/subdir/nested.bin", "nest-new-v1"); + putHdfsFile("/update/dst/root.bin", "root-same-v1"); + putHdfsFile("/update/dst/subdir/nested.bin", "nest-old-v1"); + + org.testcontainers.containers.Container.ExecResult execResult = + container.executeJob("/hdfs_binary_update_non_recursive_strict_checksum.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + Assertions.assertEquals("root-same-v1", readHdfsFile("/update/dst/root.bin")); + Assertions.assertEquals("nest-old-v1", readHdfsFile("/update/dst/subdir/nested.bin")); + } + private void resetUpdateTestPath() throws IOException, InterruptedException { nameNode.execInContainer("bash", "-c", "hdfs dfs -rm -r -f /update || true"); org.testcontainers.containers.Container.ExecResult mkdirResult = @@ -304,6 +408,14 @@ private void resetSplitTestPath() throws IOException, InterruptedException { Assertions.assertEquals(0, mkdirResult.getExitCode()); } + private void resetRecursiveTestPath() throws IOException, InterruptedException { + nameNode.execInContainer("bash", "-c", "hdfs dfs -rm -r -f /recursive || true"); + org.testcontainers.containers.Container.ExecResult mkdirResult = + nameNode.execInContainer( + "hdfs", "dfs", "-mkdir", "-p", "/recursive/subdir/deeper/final"); + Assertions.assertEquals(0, mkdirResult.getExitCode()); + } + private void putHdfsFile(String hdfsPath, String content) throws IOException, InterruptedException { String command = "printf '" + content + "' | hdfs dfs -put -f - " + hdfsPath; @@ -312,6 +424,27 @@ private void putHdfsFile(String hdfsPath, String content) Assertions.assertEquals(0, putResult.getExitCode()); } + private void createHdfsDirectories(String... hdfsPaths) + throws IOException, InterruptedException { + String command = "hdfs dfs -mkdir -p " + String.join(" ", hdfsPaths); + org.testcontainers.containers.Container.ExecResult mkdirResult = + nameNode.execInContainer("bash", "-c", command); + Assertions.assertEquals(0, mkdirResult.getExitCode()); + } + + private void putHdfsClasspathFile(String resourcePath, String hdfsPath) + throws IOException, InterruptedException { + String tempFileName = + "/tmp/" + + java.util.UUID.randomUUID() + + "-" + + java.nio.file.Paths.get(resourcePath).getFileName(); + ContainerUtil.copyFileIntoContainers(resourcePath, tempFileName, nameNode); + org.testcontainers.containers.Container.ExecResult putResult = + nameNode.execInContainer("hdfs", "dfs", "-put", "-f", tempFileName, hdfsPath); + Assertions.assertEquals(0, putResult.getExitCode()); + } + private void putHdfsSequentialLinesFile(String hdfsPath, int lineCount) throws IOException, InterruptedException { String command = @@ -331,6 +464,12 @@ private String readHdfsFile(String hdfsPath) throws IOException, InterruptedExce return catResult.getStdout() == null ? "" : catResult.getStdout().trim(); } + private boolean isHdfsFileExists(String hdfsPath) throws IOException, InterruptedException { + org.testcontainers.containers.Container.ExecResult result = + nameNode.execInContainer("hdfs", "dfs", "-test", "-f", hdfsPath); + return result.getExitCode() == 0; + } + private long getHdfsFileMtimeSeconds(String hdfsPath) throws IOException, InterruptedException { org.testcontainers.containers.Container.ExecResult statResult = nameNode.execInContainer("bash", "-c", "hdfs dfs -stat %Y " + hdfsPath); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_distcp_continuous_non_recursive.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_distcp_continuous_non_recursive.conf new file mode 100644 index 000000000000..4f91e7407d3e --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_distcp_continuous_non_recursive.conf @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + HdfsFile { + fs.defaultFS = "hdfs://namenode1:9000" + path = "/continuous/src" + file_format_type = "binary" + + discovery_mode = "continuous" + scan_interval = "1S" + start_mode = "earliest" + + sync_mode = "update" + target_path = "/continuous/dst" + update_strategy = "distcp" + compare_mode = "len_mtime" + recursive_file_scan = false + + hadoop_conf = { + "dfs.replication" = 1 + } + } +} + +sink { + HdfsFile { + fs.defaultFS = "hdfs://namenode1:9000" + path = "/continuous/dst" + tmp_path = "/continuous/tmp" + file_format_type = "binary" + data_save_mode = "APPEND_DATA" + + hadoop_conf = { + "dfs.replication" = 1 + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_distcp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_distcp.conf new file mode 100644 index 000000000000..8ff41bbedf24 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_distcp.conf @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + HdfsFile { + fs.defaultFS = "hdfs://namenode1:9000" + path = "/update/src" + file_format_type = "binary" + + sync_mode = "update" + target_path = "/update/dst" + update_strategy = "distcp" + compare_mode = "len_mtime" + recursive_file_scan = false + + hadoop_conf = { + "dfs.replication" = 1 + } + } +} + +sink { + HdfsFile { + fs.defaultFS = "hdfs://namenode1:9000" + path = "/update/dst" + tmp_path = "/update/tmp" + file_format_type = "binary" + data_save_mode = "APPEND_DATA" + + hadoop_conf = { + "dfs.replication" = 1 + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_strict_checksum.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_strict_checksum.conf new file mode 100644 index 000000000000..cb02dd2909f9 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_strict_checksum.conf @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + HdfsFile { + fs.defaultFS = "hdfs://namenode1:9000" + path = "/update/src" + file_format_type = "binary" + + sync_mode = "update" + target_path = "/update/dst" + update_strategy = "strict" + compare_mode = "checksum" + recursive_file_scan = false + + hadoop_conf = { + "dfs.replication" = 1 + } + } +} + +sink { + HdfsFile { + fs.defaultFS = "hdfs://namenode1:9000" + path = "/update/dst" + tmp_path = "/update/tmp" + file_format_type = "binary" + data_save_mode = "APPEND_DATA" + + hadoop_conf = { + "dfs.replication" = 1 + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_non_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_non_recursive_to_assert.conf new file mode 100644 index 000000000000..0ec366364fca --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_non_recursive_to_assert.conf @@ -0,0 +1,113 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + HdfsFile { + fs.defaultFS = "hdfs://namenode1:9000" + path = "/recursive" + file_format_type = "text" + recursive_file_scan = false + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + hadoop_conf = { + "dfs.replication" = 1 + } + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 5 + }, + { + rule_type = MIN_ROW + rule_value = 5 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_recursive_to_assert.conf new file mode 100644 index 000000000000..f6d713d63eef --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_recursive_to_assert.conf @@ -0,0 +1,113 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + HdfsFile { + fs.defaultFS = "hdfs://namenode1:9000" + path = "/recursive" + file_format_type = "text" + recursive_file_scan = true + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + hadoop_conf = { + "dfs.replication" = 1 + } + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 20 + }, + { + rule_type = MIN_ROW + rule_value = 20 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/text/e2e.txt b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/text/e2e.txt new file mode 100644 index 000000000000..9871cd85eb66 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/text/e2e.txt @@ -0,0 +1,5 @@ +uDDrwsQQYONTNeUBIOnLAgunvDqLBObroRzdEdvDgRmgaeFyFH5456857591576298739157764687713794636442057612252MTDnafalse3313846190943192276641872220071936002.4798444E389.52375328387482E307vcIGF2023-06-0776258155390368615610.7646252373186602912023-05-08 16:08:51ipToEdierOAbwQfQzObWqiRhjkWYaMKdCbjurhstsWrAVlRyyR2905930362869031292782506910815576701385108050hArFutrue12631169122166306155952414159791708165.949173E372.1775762383875058E307kMlgO2023-05-2027214280267865241887.6424416000104182532023-10-20 03:49:02 +QIpzzZNFkLwARZDSdwdBzkegCdIRVYJnuXgxNXytAJxxaTzmDF16603816781145850255103997497062535321459349811xaTOktrue5327578191749099325840234439082792961.955231E381.5072154481920294E308GDWOu2023-05-0581449039533149712064.4515003874168475032023-07-06 22:34:11sfgxhqvOLzjdTSNcNaWfEnZqvQraSSuMPazCGhPmSrGuxggqGh111449466287130860562118177510004750271267350957FDhTstrue96247293946402921952995131535667203.3240283E384.473485404447698E307YFdwf2023-02-0429456519357128996647.9939318900994572132023-01-12 02:29:58 +xVJPgVlosBlTYSkmJCqKHMXzbZkNQKInuVMZeYGhsmzUmcLyPx137745493211075991209783701051546835517166168384qcYaifalse8318050110096656524405690917018449922.9617934E371.8901064340036343E307jaKMq2023-05-1275317114043170470995.9654034735914367862023-05-18 08:09:22raGGBnHsNwMZKemkFErUbedNjSllNcKOVUGdTpXcHGSVphHsNE86377304018502081846122308810391870441519757437JCRZStrue1829974183977114228752256792969205767.9090967E371.6286963710372255E308NBHUB2023-05-0732934086493941743464.6503746053883129532023-05-06 04:35:55 +dBgFeTKkCfnxCljyGfNEurEzCVgwpsHgmcOfYXiQHxeeQNjQuq1961913761867016982512369059615238191571813320BTfhbfalse652666522281866957533025299230722.1456136E381.2398422714159417E308YOiwg2023-10-2433001899362876139955.7235198795513055732023-06-23 13:46:46jsvmHLHlXCGFKwuqlTwAjdMckElrmqgBWvOuuKuWxcinFZWSky19959088245502706421265289671411088181469730839vUyULtrue952655754382886132164227350822215681.9033253E381.0966562906060974E308XFeKf2023-09-1731084757529957096723.2394423349193989032023-06-15 17:04:50 +obtYzIHOTKsABVtirEKEMYUYobsYlDJcFbpQUYvGxCcKlnswEG8096984004544201585383739017658796661353001394xchcntrue853141253976762312923177914159380482.8480754E381.055208146200822E308MSkTD2023-11-2420361788179232141281.9718823433892185262023-10-25 11:47:50gdCWZMGESyarjQPopBhDwKnOyDvaUDgQOEDRCmfUAagfnDDPqV8473436731118772451890654127233667151574025969ewJzLtrue6321769209768782446484076920790579202.7134378E381.1883616449174808E308STvOu2023-10-0821793351767634029460.2897683013563753232023-08-12 23:57:38 \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java index c0cc2d77c25e..213771b4e96a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java @@ -324,6 +324,22 @@ public class LocalFileIT extends TestSuiteBase { "-c", "mkdir -p /seatunnel/read/markdown && printf '# E2E Markdown RAG\\n' > /seatunnel/read/markdown/e2e.md"); + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", "/seatunnel/read/recursive/e2e.txt", container); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", "/seatunnel/read/recursive/subdir/e2e.txt", container); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + "/seatunnel/read/recursive/subdir/deeper/e2e.txt", + container); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + "/seatunnel/read/recursive/subdir/deeper/final/e2e.txt", + container); + container.execInContainer("mkdir", "-p", "/tmp/fake_empty"); }; @@ -430,6 +446,10 @@ public void testLocalFileReadAndWrite(TestContainer container) helper.execute("/excel/local_excel_multi_zip_to_assert.conf"); helper.execute("/excel/local_excel_xls_gz_to_assert.conf"); helper.execute("/excel/local_excel_xlsx_gz_to_assert.conf"); + + // test read recursive file path + helper.execute("/text/local_file_text_recursive_to_assert.conf"); + helper.execute("/text/local_file_text_non_recursive_to_assert.conf"); } @TestTemplate @@ -560,6 +580,108 @@ public void testLocalFileBinaryUpdateModeContinuousDiscovery(TestContainer conta baseContainer.execInContainer("sh", "-c", "rm -rf /tmp/seatunnel/continuous"); } + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.FLINK, EngineType.SPARK}, + disabledReason = + "Continuous discovery is a long-running job. Local filesystem is not shared between engine master/workers in Flink/Spark E2E.") + public void testLocalFileBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan( + TestContainer container) throws IOException, InterruptedException { + resetContinuousTestPath(); + + String jobId = String.valueOf(JobIdGenerator.newJobId()); + CompletableFuture jobFuture = + CompletableFuture.supplyAsync( + () -> { + try { + return container.executeJob( + "/binary/local_file_binary_update_distcp_continuous_non_recursive.conf", + jobId); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + putLocalFile("/tmp/seatunnel/continuous/src/root.bin", "root"); + putLocalFile("/tmp/seatunnel/continuous/src/subdir/nested.bin", "nested"); + + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertEquals( + "root", + readLocalFile("/tmp/seatunnel/continuous/dst/root.bin"))); + + Thread.sleep(3000); + Assertions.assertFalse( + isLocalFileExists("/tmp/seatunnel/continuous/dst/subdir/nested.bin")); + + Container.ExecResult cancelResult = container.cancelJob(jobId); + Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr()); + + Container.ExecResult execResult; + try { + execResult = jobFuture.get(120, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException("Wait continuous job exit failed.", e); + } + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + + baseContainer.execInContainer("sh", "-c", "rm -rf /tmp/seatunnel/continuous"); + } + + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.FLINK, EngineType.SPARK}, + disabledReason = + "sync_mode=update needs to compare source/target on the same filesystem. Local filesystem is not shared between engine master/workers in Flink/Spark E2E.") + public void testLocalFileBinaryUpdateModeDistcpWithNonRecursiveScan(TestContainer container) + throws IOException, InterruptedException { + resetUpdateTestPath(); + putLocalFile("/tmp/seatunnel/update/src/root.bin", "root-updated-v2"); + putLocalFile("/tmp/seatunnel/update/src/subdir/nested.bin", "nest-updated-v2"); + putLocalFile("/tmp/seatunnel/update/dst/root.bin", "root-stale-v1"); + putLocalFile("/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-stale-v1"); + + TestHelper helper = new TestHelper(container); + helper.execute("/binary/local_file_binary_update_non_recursive_distcp.conf"); + + Assertions.assertEquals( + "root-updated-v2", readLocalFile("/tmp/seatunnel/update/dst/root.bin")); + Assertions.assertEquals( + "nest-stale-v1", readLocalFile("/tmp/seatunnel/update/dst/subdir/nested.bin")); + + baseContainer.execInContainer("sh", "-c", "rm -rf /tmp/seatunnel/update"); + } + + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.FLINK, EngineType.SPARK}, + disabledReason = + "sync_mode=update needs to compare source/target on the same filesystem. Local filesystem is not shared between engine master/workers in Flink/Spark E2E.") + public void testLocalFileBinaryUpdateModeStrictChecksumSkipsNestedChangesWithNonRecursiveScan( + TestContainer container) throws IOException, InterruptedException { + resetUpdateTestPath(); + putLocalFile("/tmp/seatunnel/update/src/root.bin", "root-same-v1"); + putLocalFile("/tmp/seatunnel/update/src/subdir/nested.bin", "nest-new-v1"); + putLocalFile("/tmp/seatunnel/update/dst/root.bin", "root-same-v1"); + putLocalFile("/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-old-v1"); + + TestHelper helper = new TestHelper(container); + helper.execute("/binary/local_file_binary_update_non_recursive_strict_checksum.conf"); + + Assertions.assertEquals( + "root-same-v1", readLocalFile("/tmp/seatunnel/update/dst/root.bin")); + Assertions.assertEquals( + "nest-old-v1", readLocalFile("/tmp/seatunnel/update/dst/subdir/nested.bin")); + + baseContainer.execInContainer("sh", "-c", "rm -rf /tmp/seatunnel/update"); + } + @TestTemplate @DisabledOnContainer( value = {TestContainerId.SPARK_2_4}, @@ -666,6 +788,12 @@ private String readLocalFile(String filePath) throws IOException, InterruptedExc return result.getStdout() == null ? "" : result.getStdout().trim(); } + private boolean isLocalFileExists(String filePath) throws IOException, InterruptedException { + Container.ExecResult result = + baseContainer.execInContainer("sh", "-c", "test -f '" + filePath + "'"); + return result.getExitCode() == 0; + } + private long getLocalFileMtimeSeconds(String filePath) throws IOException, InterruptedException { Container.ExecResult result = diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_distcp_continuous_non_recursive.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_distcp_continuous_non_recursive.conf new file mode 100644 index 000000000000..2a90543b7230 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_distcp_continuous_non_recursive.conf @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + LocalFile { + path = "/tmp/seatunnel/continuous/src" + file_format_type = "binary" + + discovery_mode = "continuous" + scan_interval = "1S" + start_mode = "earliest" + + sync_mode = "update" + target_path = "/tmp/seatunnel/continuous/dst" + update_strategy = "distcp" + compare_mode = "len_mtime" + recursive_file_scan = false + } +} + +sink { + LocalFile { + path = "/tmp/seatunnel/continuous/dst" + tmp_path = "/tmp/seatunnel/continuous/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_distcp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_distcp.conf new file mode 100644 index 000000000000..860e4dfeb659 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_distcp.conf @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + LocalFile { + path = "/tmp/seatunnel/update/src" + file_format_type = "binary" + + sync_mode = "update" + target_path = "/tmp/seatunnel/update/dst" + update_strategy = "distcp" + compare_mode = "len_mtime" + recursive_file_scan = false + } +} + +sink { + LocalFile { + path = "/tmp/seatunnel/update/dst" + tmp_path = "/tmp/seatunnel/update/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_strict_checksum.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_strict_checksum.conf new file mode 100644 index 000000000000..fe53193e1051 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_strict_checksum.conf @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + LocalFile { + path = "/tmp/seatunnel/update/src" + file_format_type = "binary" + + sync_mode = "update" + target_path = "/tmp/seatunnel/update/dst" + update_strategy = "strict" + compare_mode = "checksum" + recursive_file_scan = false + } +} + +sink { + LocalFile { + path = "/tmp/seatunnel/update/dst" + tmp_path = "/tmp/seatunnel/update/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_non_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_non_recursive_to_assert.conf new file mode 100644 index 000000000000..36419729eaf0 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_non_recursive_to_assert.conf @@ -0,0 +1,117 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + LocalFile { + path = "/seatunnel/read/recursive" + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + file_format_type = "text" + recursive_file_scan = false + plugin_output = "fake" + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 5 + }, + { + rule_type = MIN_ROW + rule_value = 5 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_recursive_to_assert.conf new file mode 100644 index 000000000000..19d3a8a2169b --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_recursive_to_assert.conf @@ -0,0 +1,117 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + LocalFile { + path = "/seatunnel/read/recursive" + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + file_format_type = "text" + recursive_file_scan = true + plugin_output = "fake" + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 20 + }, + { + rule_type = MIN_ROW + rule_value = 20 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java index c11282f140d8..658cfaab0082 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java @@ -125,6 +125,27 @@ public void startUp() throws Exception { "/text/e2e.txt", "/home/seatunnel/tmp/seatunnel/read/wildcard/e2e.txt", sftpContainer); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + "/home/seatunnel/tmp/seatunnel/read/recursive/e2e.txt", + sftpContainer); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + "/home/seatunnel/tmp/seatunnel/read/recursive/subdir/e2e.txt", + sftpContainer); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + "/home/seatunnel/tmp/seatunnel/read/recursive/subdir/deeper/e2e.txt", + sftpContainer); + + ContainerUtil.copyFileIntoContainers( + "/text/e2e.txt", + "/home/seatunnel/tmp/seatunnel/read/recursive/subdir/deeper/final/e2e.txt", + sftpContainer); + Container.ExecResult chownResult = sftpContainer.execInContainer( "sh", "-c", "chown -R seatunnel /home/seatunnel/tmp/"); @@ -335,6 +356,115 @@ public void testSftpBinaryUpdateModeContinuousDiscoveryDistcp(TestContainer cont } } + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.FLINK, EngineType.SPARK}, + disabledReason = "Continuous discovery is a long-running job; only run in zeta engine.") + public void testSftpBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan( + TestContainer container) throws IOException, InterruptedException { + resetContinuousTestPath(); + try { + String jobId = String.valueOf(JobIdGenerator.newJobId()); + CompletableFuture jobFuture = + CompletableFuture.supplyAsync( + () -> { + try { + return container.executeJob( + "/text/sftp_binary_update_distcp_continuous_non_recursive.conf", + jobId); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/continuous/src/root.bin", "root"); + putSftpFile( + SFTP_CONTAINER_HOME + "/tmp/seatunnel/continuous/src/subdir/nested.bin", + "nested"); + + Awaitility.await() + .atMost(120, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertEquals( + "root", + readSftpFile( + SFTP_CONTAINER_HOME + + "/tmp/seatunnel/continuous/dst/root.bin"))); + + Thread.sleep(3000); + Assertions.assertFalse( + isSftpFileExists( + SFTP_CONTAINER_HOME + + "/tmp/seatunnel/continuous/dst/subdir/nested.bin")); + + Container.ExecResult cancelResult = container.cancelJob(jobId); + Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr()); + + Container.ExecResult execResult; + try { + execResult = jobFuture.get(120, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException("Wait continuous job exit failed.", e); + } + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + } finally { + deleteFileFromContainer(SFTP_CONTAINER_HOME + "/tmp/seatunnel/continuous"); + } + } + + @TestTemplate + public void testSftpBinaryUpdateModeDistcpWithNonRecursiveScan(TestContainer container) + throws IOException, InterruptedException { + resetUpdateTestPath(); + putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/src/root.bin", "root-updated-v2"); + putSftpFile( + SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/src/subdir/nested.bin", + "nest-updated-v2"); + putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/root.bin", "root-stale-v1"); + putSftpFile( + SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/subdir/nested.bin", + "nest-stale-v1"); + + TestHelper helper = new TestHelper(container); + helper.execute("/text/sftp_binary_update_non_recursive_distcp.conf"); + + Assertions.assertEquals( + "root-updated-v2", + readSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/root.bin")); + Assertions.assertEquals( + "nest-stale-v1", + readSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/subdir/nested.bin")); + + deleteFileFromContainer(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update"); + } + + @TestTemplate + public void testSftpBinaryUpdateModeStrictChecksumSkipsNestedChangesWithNonRecursiveScan( + TestContainer container) throws IOException, InterruptedException { + resetUpdateTestPath(); + putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/src/root.bin", "root-same-v1"); + putSftpFile( + SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/src/subdir/nested.bin", "nest-new-v1"); + putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/root.bin", "root-same-v1"); + putSftpFile( + SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-old-v1"); + + TestHelper helper = new TestHelper(container); + helper.execute("/text/sftp_binary_update_non_recursive_strict_checksum.conf"); + + Assertions.assertEquals( + "root-same-v1", + readSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/root.bin")); + Assertions.assertEquals( + "nest-old-v1", + readSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/subdir/nested.bin")); + + deleteFileFromContainer(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update"); + } + @TestTemplate public void testMultipleTableAndSaveMode(TestContainer container) throws IOException, InterruptedException { @@ -446,6 +576,13 @@ private String readSftpFile(String containerPath) throws IOException, Interrupte return catResult.getStdout() == null ? "" : catResult.getStdout().trim(); } + private boolean isSftpFileExists(String containerPath) + throws IOException, InterruptedException { + Container.ExecResult result = + sftpContainer.execInContainer("sh", "-c", "test -f '" + containerPath + "'"); + return result.getExitCode() == 0; + } + private void waitUntilContainerTimeAfter(long epochSeconds) { Awaitility.await() .atMost(10, TimeUnit.SECONDS) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_distcp_continuous_non_recursive.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_distcp_continuous_non_recursive.conf new file mode 100644 index 000000000000..de3f99a7f71d --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_distcp_continuous_non_recursive.conf @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 + + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + + path = "tmp/seatunnel/continuous/src" + file_format_type = "binary" + + discovery_mode = "continuous" + scan_interval = "1S" + start_mode = "earliest" + + sync_mode = "update" + target_path = "tmp/seatunnel/continuous/dst" + update_strategy = "distcp" + compare_mode = "len_mtime" + recursive_file_scan = false + } +} + +sink { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + + path = "tmp/seatunnel/continuous/dst" + tmp_path = "tmp/seatunnel/continuous/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_distcp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_distcp.conf new file mode 100644 index 000000000000..babebe240e9a --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_distcp.conf @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + + path = "tmp/seatunnel/update/src" + file_format_type = "binary" + + sync_mode = "update" + target_path = "tmp/seatunnel/update/dst" + update_strategy = "distcp" + compare_mode = "len_mtime" + recursive_file_scan = false + } +} + +sink { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + + path = "tmp/seatunnel/update/dst" + tmp_path = "tmp/seatunnel/update/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_strict_checksum.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_strict_checksum.conf new file mode 100644 index 000000000000..34ef8272cdd7 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_strict_checksum.conf @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + + path = "tmp/seatunnel/update/src" + file_format_type = "binary" + + sync_mode = "update" + target_path = "tmp/seatunnel/update/dst" + update_strategy = "strict" + compare_mode = "checksum" + recursive_file_scan = false + } +} + +sink { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + + path = "tmp/seatunnel/update/dst" + tmp_path = "tmp/seatunnel/update/tmp" + file_format_type = "binary" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_non_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_non_recursive_to_assert.conf new file mode 100644 index 000000000000..42b7bbc30fc7 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_non_recursive_to_assert.conf @@ -0,0 +1,122 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + path = "tmp/seatunnel/read/recursive" + file_format_type = "text" + recursive_file_scan = false + plugin_output = "sftp" + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + } +} + +sink { + Assert { + plugin_input = "sftp" + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 5 + }, + { + rule_type = MIN_ROW + rule_value = 5 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_recursive_to_assert.conf new file mode 100644 index 000000000000..f81e4625f494 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_recursive_to_assert.conf @@ -0,0 +1,122 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + path = "tmp/seatunnel/read/recursive" + file_format_type = "text" + recursive_file_scan = true + plugin_output = "sftp" + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + } +} + +sink { + Assert { + plugin_input = "sftp" + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 20 + }, + { + rule_type = MIN_ROW + rule_value = 20 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} From 73d96a736c30a3e11ab98583b7f6f4a17a5434f7 Mon Sep 17 00:00:00 2001 From: Jast Date: Tue, 16 Jun 2026 09:41:01 +0800 Subject: [PATCH 019/375] [Fix][Zeta] Fix checkpoint barrier trigger test race (#11071) --- .../server/checkpoint/CheckpointBarrierTriggerErrorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointBarrierTriggerErrorTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointBarrierTriggerErrorTest.java index ebcf2827242a..ff80c9a64624 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointBarrierTriggerErrorTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointBarrierTriggerErrorTest.java @@ -51,6 +51,7 @@ public class CheckpointBarrierTriggerErrorTest extends AbstractSeaTunnelServerTe @Test public void testCheckpointBarrierTriggerError() throws NoSuchFieldException, IllegalAccessException { + COUNTER.set(0); long jobId = System.currentTimeMillis(); startJob(jobId, CONF_PATH); @@ -62,11 +63,10 @@ public void testCheckpointBarrierTriggerError() JobStatus.RUNNING)); CheckpointManager spiedCheckpointManager = spy(getCheckpointManager(jobId)); - setCheckpointManager(spiedCheckpointManager); - doAnswer(this::mockException) .when(spiedCheckpointManager) .sendOperationToMemberNode(Mockito.any(CheckpointBarrierTriggerOperation.class)); + setCheckpointManager(spiedCheckpointManager); await().atMost(120000, TimeUnit.MILLISECONDS) .untilAsserted( From 2dc6c970f84290fae89e79d3b3c38224f0036747 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Tue, 16 Jun 2026 16:21:53 +0800 Subject: [PATCH 020/375] [Fix][E2E] Stabilize multiple flaky tests (#11102) --- .../cdc/sqlserver/SqlServerCDCIT.java | 39 +++++++- .../e2e/connector/tdengine/TDengineIT.java | 24 ++++- .../e2e/classloader/ClassLoaderITBase.java | 2 +- .../client/ConnectorPackageClientTest.java | 91 ++++++++++--------- .../SeaTunnelEngineClusterRoleTest.java | 22 ++++- 5 files changed, 127 insertions(+), 51 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java index fc48f3f81e40..f45cc8f7e35d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java @@ -620,18 +620,51 @@ private void executeSqlFile(String sqlFile) { final String ddlFile = String.format("ddl/%s.sql", sqlFile); final URL ddlTestFile = TestSuiteBase.class.getClassLoader().getResource(ddlFile); Assertions.assertNotNull(ddlTestFile, "Cannot locate " + ddlFile); - try (Connection connection = getJdbcConnection(); - Statement statement = connection.createStatement()) { + try { List statements = parseStatements(Files.readAllLines(Paths.get(ddlTestFile.toURI()))); + String currentDatabase = null; for (String stmt : statements) { - statement.execute(stmt); + String trimmed = stmt.trim(); + if (trimmed.toUpperCase().startsWith("USE ")) { + currentDatabase = trimmed.substring(4).replaceAll(";\\s*$", "").trim(); + continue; + } + executeWithDeadlockRetry(stmt, currentDatabase); } } catch (Exception e) { throw new RuntimeException(e); } } + private void executeWithDeadlockRetry(String sql, String database) { + Awaitility.await( + "Executing: " + + sql.substring(0, Math.min(80, sql.length())) + .replaceAll("\\s+", " ")) + .atMost(60, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + if (database != null) { + statement.execute("USE " + database); + } + statement.execute(sql); + return true; + } catch (SQLException e) { + if (e.getMessage() != null + && (e.getMessage().contains("deadlock") + || e.getMessage().contains("Deadlock"))) { + log.warn("Deadlock detected, will retry: {}", sql); + return false; + } + throw new RuntimeException(e); + } + }); + } + private void initializeSqlServerTable(String sqlFile) { final String ddlFile = String.format("ddl/%s.sql", sqlFile); final URL ddlTestFile = TestSuiteBase.class.getClassLoader().getResource(ddlFile); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tdengine/TDengineIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tdengine/TDengineIT.java index 12935b378ddc..6c60ac6bf148 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tdengine/TDengineIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tdengine/TDengineIT.java @@ -108,16 +108,16 @@ public void startUp() throws Exception { @SneakyThrows private int generateTestDataSet() { int rowCount; + waitForDatabaseReady(connection1, "CREATE DATABASE power KEEP 3650"); try (Statement stmt = connection1.createStatement()) { - stmt.execute("CREATE DATABASE power KEEP 3650"); stmt.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT, off BOOL, nc NCHAR(10)) " + "TAGS (location BINARY(64), groupId INT)"); String sql = getSQL(); rowCount = stmt.executeUpdate(sql); } + waitForDatabaseReady(connection2, "CREATE DATABASE power2 KEEP 3650"); try (Statement stmt = connection2.createStatement()) { - stmt.execute("CREATE DATABASE power2 KEEP 3650"); stmt.execute( "CREATE STABLE power2.meters2 (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT, off BOOL, nc NCHAR(10)) " + "TAGS (location BINARY(64), groupId INT)"); @@ -134,8 +134,8 @@ private int generateTestDataSet() { "CREATE STABLE power2.meters4 (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT, off BOOL, nc NCHAR(10)) " + "TAGS (location BINARY(64), groupId INT)"); } + waitForDatabaseReady(connection2, "CREATE DATABASE power3 KEEP 3650"); try (Statement stmt = connection2.createStatement()) { - stmt.execute("CREATE DATABASE power3 KEEP 3650"); stmt.execute( "CREATE STABLE power3.meters5 (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT, off BOOL, nc NCHAR(10)) " + "TAGS (location BINARY(64), groupId INT)"); @@ -143,6 +143,24 @@ private int generateTestDataSet() { return rowCount; } + /** + * Waits for TDengine dnode to be fully online before creating the database. The REST adapter + * (port 6041) may accept connections before the dnode registers with the management node, + * causing "Out of dnodes" errors on CREATE DATABASE. + */ + private void waitForDatabaseReady(Connection connection, String createDbSql) { + given().ignoreExceptions() + .await() + .pollInterval(2, TimeUnit.SECONDS) + .atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try (Statement stmt = connection.createStatement()) { + stmt.execute(createDbSql); + } + }); + } + @TestTemplate public void testTDengine(TestContainer container) throws Exception { Container.ExecResult execResult = diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java index 26cea8651ccb..f0ff5d78bd63 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java @@ -182,7 +182,7 @@ private void assertClassLoaderStateEventuallyStable( private int getClassLoaderUpperBound(int initialClassLoaderCount, int iteration) { if (cacheMode()) { - return initialClassLoaderCount; + return initialClassLoaderCount + DISABLE_CACHE_MODE_CLASSLOADERS_PER_JOB; } return initialClassLoaderCount + DISABLE_CACHE_MODE_CLASSLOADERS_PER_JOB * (iteration + 1); } diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/ConnectorPackageClientTest.java b/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/ConnectorPackageClientTest.java index a2273782a3b1..89b58995928f 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/ConnectorPackageClientTest.java +++ b/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/ConnectorPackageClientTest.java @@ -191,50 +191,55 @@ public void testUploadConnectorPluginJars() throws MalformedURLException { SeaTunnelHazelcastClient seaTunnelHazelcastClient = new SeaTunnelHazelcastClient(clientConfig); - Common.setDeployMode(DeployMode.CLIENT); - String filePath = ContentFormatUtilTest.getResource("/client_test.conf"); - Config seaTunnelJobConfig = ConfigBuilder.of(Paths.get(filePath)); - ReadonlyConfig envOptions = ReadonlyConfig.fromConfig(seaTunnelJobConfig.getConfig("env")); - JobConfig jobConfig = new JobConfig(); - jobConfig.setName("testUploadConnectorPluginJars"); - jobConfig.setJobContext(new JobContext(JOB_ID)); - fillJobConfig(jobConfig, envOptions); - - ConnectorPackageClient connectorPackageClient = - new ConnectorPackageClient(seaTunnelHazelcastClient); - Path connectorDir = Common.connectorDir(); - File[] files = - connectorDir - .toFile() - .listFiles( - new FileFilter() { - @Override - public boolean accept(File pathname) { - return pathname.getName().endsWith(".jar") - && (StringUtils.startsWithIgnoreCase( - pathname.getName(), - "connector-fake") - || StringUtils.startsWithIgnoreCase( - pathname.getName(), - "connector-file")); - } - }); - if (files != null) { - for (File file : files) { - ConnectorJarIdentifier connectorJarIdentifier = - connectorPackageClient.uploadConnectorPluginJar( - JOB_ID, file.toURI().toURL()); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - Assertions.assertTrue( - StringUtils.isNotBlank( - connectorJarIdentifier.getStoragePath())); - Assertions.assertEquals( - ConnectorJarType.CONNECTOR_PLUGIN_JAR, - connectorJarIdentifier.getType()); - }); + try { + Common.setDeployMode(DeployMode.CLIENT); + String filePath = ContentFormatUtilTest.getResource("/client_test.conf"); + Config seaTunnelJobConfig = ConfigBuilder.of(Paths.get(filePath)); + ReadonlyConfig envOptions = + ReadonlyConfig.fromConfig(seaTunnelJobConfig.getConfig("env")); + JobConfig jobConfig = new JobConfig(); + jobConfig.setName("testUploadConnectorPluginJars"); + jobConfig.setJobContext(new JobContext(JOB_ID)); + fillJobConfig(jobConfig, envOptions); + + ConnectorPackageClient connectorPackageClient = + new ConnectorPackageClient(seaTunnelHazelcastClient); + Path connectorDir = Common.connectorDir(); + File[] files = + connectorDir + .toFile() + .listFiles( + new FileFilter() { + @Override + public boolean accept(File pathname) { + return pathname.getName().endsWith(".jar") + && (StringUtils.startsWithIgnoreCase( + pathname.getName(), + "connector-fake") + || StringUtils.startsWithIgnoreCase( + pathname.getName(), + "connector-file")); + } + }); + if (files != null) { + for (File file : files) { + ConnectorJarIdentifier connectorJarIdentifier = + connectorPackageClient.uploadConnectorPluginJar( + JOB_ID, file.toURI().toURL()); + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + Assertions.assertTrue( + StringUtils.isNotBlank( + connectorJarIdentifier.getStoragePath())); + Assertions.assertEquals( + ConnectorJarType.CONNECTOR_PLUGIN_JAR, + connectorJarIdentifier.getType()); + }); + } } + } finally { + seaTunnelHazelcastClient.shutdown(); } } diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java b/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java index 7f37656c64f1..82f63ae9d89e 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java +++ b/seatunnel-engine/seatunnel-engine-client/src/test/java/org/apache/seatunnel/engine/client/SeaTunnelEngineClusterRoleTest.java @@ -332,6 +332,10 @@ public void pendingJobCancel() { @Test public void testStartMasterNodeWithTcpIp() { SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); + seaTunnelConfig + .getHazelcastConfig() + .setClusterName( + ContentFormatUtilTest.getClusterName("Test_testStartMasterNodeWithTcpIp")); HazelcastInstanceImpl instance = SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig); Assertions.assertNotNull(instance); @@ -343,6 +347,11 @@ public void testStartMasterNodeWithTcpIp() { public void testStartMasterNodeWithMulticastJoin() { SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); seaTunnelConfig.setHazelcastConfig(Config.loadFromString(getMulticastConfig())); + seaTunnelConfig + .getHazelcastConfig() + .setClusterName( + ContentFormatUtilTest.getClusterName( + "Test_testStartMasterNodeWithMulticastJoin")); HazelcastInstanceImpl instance = SeaTunnelServerStarter.createMasterHazelcastInstance(seaTunnelConfig); Assertions.assertNotNull(instance); @@ -353,6 +362,11 @@ public void testStartMasterNodeWithMulticastJoin() { @Test public void testCannotOnlyStartWorkerNodeWithTcpIp() { SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); + seaTunnelConfig + .getHazelcastConfig() + .setClusterName( + ContentFormatUtilTest.getClusterName( + "Test_testCannotOnlyStartWorkerNodeWithTcpIp")); Assertions.assertThrows( IllegalStateException.class, () -> { @@ -364,6 +378,11 @@ public void testCannotOnlyStartWorkerNodeWithTcpIp() { public void testCannotOnlyStartWorkerNodeWithMulticastJoin() { SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); seaTunnelConfig.setHazelcastConfig(Config.loadFromString(getMulticastConfig())); + seaTunnelConfig + .getHazelcastConfig() + .setClusterName( + ContentFormatUtilTest.getClusterName( + "Test_testCannotOnlyStartWorkerNodeWithMulticastJoin")); Assertions.assertThrows( IllegalStateException.class, () -> { @@ -462,8 +481,9 @@ public void testWorkerIsFirstMemberThenGetJobDetailStatus() { .listJobStatus(true) .contains("RUNNING"))); jobClient.cancelJob(jobId); - // Master handoff can delay terminal status propagation on the slower JDK 8 CI lane. await().atMost(120000, TimeUnit.MILLISECONDS) + .pollDelay(5, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) .untilAsserted( () -> { String status = jobClient.getJobStatus(jobId); From de4cf3387a75ac58827eb94772dbbf057f29589a Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Tue, 16 Jun 2026 16:28:07 +0800 Subject: [PATCH 021/375] [Fix][Docs] Remove duplicate connector FAQ sidebar entry (#11105) Co-authored-by: Shenghang --- docs/sidebars.js | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sidebars.js b/docs/sidebars.js index a02d7fd42499..d5980484d303 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -142,7 +142,6 @@ const sidebars = { "connectors/connector-isolated-dependency", "connectors/connector-faq", "connectors/cdc-production-cookbook", - "connectors/connector-faq", { "type": "category", "label": "Source", From 5e88b7f50cabc0d8037c511e49b8e670f08c42f6 Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Wed, 17 Jun 2026 17:05:17 +0800 Subject: [PATCH 022/375] [Docs] Fix Typesense sink common options link (#11111) Co-authored-by: DanielCarter-stack <254644355+DanielCarter-stack@users.noreply.github.com> --- docs/en/connectors/sink/Typesense.md | 4 ++-- docs/zh/connectors/sink/Typesense.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/connectors/sink/Typesense.md b/docs/en/connectors/sink/Typesense.md index ee12cf91ec34..18888d8df0e7 100644 --- a/docs/en/connectors/sink/Typesense.md +++ b/docs/en/connectors/sink/Typesense.md @@ -56,7 +56,7 @@ The maximum size of document batches. ### common options -Common parameters for Sink plugins. Refer to [Common Sink Options](../common-options/source-common-options.md) for more details. +Common parameters for Sink plugins. Refer to [Common Sink Options](../common-options/sink-common-options.md) for more details. ### schema_save_mode @@ -95,4 +95,4 @@ sink { ## Changelog - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/Typesense.md b/docs/zh/connectors/sink/Typesense.md index b88532b183f3..5056c91bc941 100644 --- a/docs/zh/connectors/sink/Typesense.md +++ b/docs/zh/connectors/sink/Typesense.md @@ -56,7 +56,7 @@ typesense 安全认证的 api_key。 ### common options -Sink插件常用参数，请参考 [Sink常用选项](../common-options/sink-common-options.md) 了解详情 +Sink 插件常用参数，请参考 [Sink 常用选项](../common-options/sink-common-options.md) 了解详情。 ### schema_save_mode @@ -97,4 +97,4 @@ sink { ## 变更日志 - \ No newline at end of file + From 7b987716c9b6f8fb6059ce0347516018ea7cca98 Mon Sep 17 00:00:00 2001 From: Jast Date: Thu, 18 Jun 2026 08:22:43 +0800 Subject: [PATCH 023/375] [Docs] Fix RocketMQ transform links (#11118) --- docs/en/connectors/sink/RocketMQ.md | 6 +++--- docs/en/connectors/source/RocketMQ.md | 8 ++++---- docs/zh/connectors/sink/RocketMQ.md | 6 +++--- docs/zh/connectors/source/RocketMQ.md | 5 ++--- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/en/connectors/sink/RocketMQ.md b/docs/en/connectors/sink/RocketMQ.md index 141ad6a87b97..6a6e09cb5fe9 100644 --- a/docs/en/connectors/sink/RocketMQ.md +++ b/docs/en/connectors/sink/RocketMQ.md @@ -95,7 +95,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/category/transform + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -194,7 +194,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/category/transform + # please go to https://seatunnel.apache.org/docs/transforms } sink { Rocketmq { @@ -208,4 +208,4 @@ sink { ## Changelog - \ No newline at end of file + diff --git a/docs/en/connectors/source/RocketMQ.md b/docs/en/connectors/source/RocketMQ.md index 776fa795c36d..279ad13c1ce9 100644 --- a/docs/en/connectors/source/RocketMQ.md +++ b/docs/en/connectors/source/RocketMQ.md @@ -108,7 +108,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/category/transform + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -160,7 +160,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/category/transform + # please go to https://seatunnel.apache.org/docs/transforms } sink { Console { @@ -215,7 +215,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/category/transform + # please go to https://seatunnel.apache.org/docs/transforms } sink { @@ -274,7 +274,7 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/category/transform + # please go to https://seatunnel.apache.org/docs/transforms } sink { diff --git a/docs/zh/connectors/sink/RocketMQ.md b/docs/zh/connectors/sink/RocketMQ.md index 7502fcd5dd12..5bd55404b16e 100644 --- a/docs/zh/connectors/sink/RocketMQ.md +++ b/docs/zh/connectors/sink/RocketMQ.md @@ -93,7 +93,7 @@ source { transform { #如果你想了解更多关于如何配置seatunnel的信息，并查看转换插件的完整列表， - #请前往https://seatunnel.apache.org/docs/category/transform + #请前往https://seatunnel.apache.org/docs/transforms } sink { @@ -192,7 +192,7 @@ source { transform { #如果你想了解更多关于如何配置seatunnel的信息，并查看转换插件的完整列表， - #请前往https://seatunnel.apache.org/docs/category/transform + #请前往https://seatunnel.apache.org/docs/transforms } sink { Rocketmq { @@ -206,4 +206,4 @@ sink { ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/source/RocketMQ.md b/docs/zh/connectors/source/RocketMQ.md index 052452b768c9..4bb0d158b65b 100644 --- a/docs/zh/connectors/source/RocketMQ.md +++ b/docs/zh/connectors/source/RocketMQ.md @@ -108,7 +108,7 @@ source { transform { # 如果您想了解有关如何配置 seatunnel 的更多信息并查看完整的转换插件列表， - # 请访问 https://seatunnel.apache.org/docs/category/transform + # 请访问 https://seatunnel.apache.org/docs/transforms } sink { @@ -159,7 +159,7 @@ source { transform { # 如果您想了解有关如何配置 seatunnel 的更多信息并查看完整的转换插件列表， - # 请访问 https://seatunnel.apache.org/docs/category/transform + # 请访问 https://seatunnel.apache.org/docs/transforms } sink { @@ -217,4 +217,3 @@ sink { ## 变更日志 - From a73b3d05c11f0a116efe3909f2d831ccecdd2af5 Mon Sep 17 00:00:00 2001 From: Jast Date: Thu, 18 Jun 2026 08:23:24 +0800 Subject: [PATCH 024/375] [Docs] Fix Kafka source example text (#11119) --- docs/en/connectors/source/Kafka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/connectors/source/Kafka.md b/docs/en/connectors/source/Kafka.md index be6cac4df7da..1964ac784ff3 100644 --- a/docs/en/connectors/source/Kafka.md +++ b/docs/en/connectors/source/Kafka.md @@ -131,7 +131,7 @@ transform { ### Simple -> This example reads the data of kafka's topic_1, topic_2, topic_3 and prints it to the client.And if you have not yet installed and deployed SeaTunnel, you need to follow the instructions in Install SeaTunnel to install and deploy SeaTunnel. And if you have not yet installed and deployed SeaTunnel, you need to follow the instructions in [Install SeaTunnel](../../getting-started/locally/deployment.md) to install and deploy SeaTunnel. And then follow the instructions in [Quick Start With SeaTunnel Engine](../../getting-started/locally/quick-start-seatunnel-engine.md) to run this job. +> This example reads data from Kafka topics topic_1, topic_2, and topic_3 and prints it to the client. If you have not installed and deployed SeaTunnel yet, follow [Install SeaTunnel](../../getting-started/locally/deployment.md) first. Then follow [Quick Start With SeaTunnel Engine](../../getting-started/locally/quick-start-seatunnel-engine.md) to run this job. > In batch mode, during the enumerator sharding process, it will fetch the latest offset for each partition and use it as the stopping point. ```hocon From 9140b90954d43f9d4debe2b50dcf7a4595be23c2 Mon Sep 17 00:00:00 2001 From: Jast Date: Thu, 18 Jun 2026 08:24:46 +0800 Subject: [PATCH 025/375] [Fix][Connector-V2] Fix HTTP cursor pagination no-progress loop (#11098) --- .../http/source/HttpSourceReader.java | 8 +++- .../HttpSourceReaderInternalPollNextTest.java | 48 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java index 2bc66a379633..3ed214ce6778 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java @@ -464,6 +464,7 @@ private void collect(Collector output, String data) throws IOExcep // cursor pagination if (HttpPaginationType.CURSOR.getCode().equals(pageInfo.getPageType())) { + String currentCursor = pageInfo.getCursor(); // get cursor value from response JSON with fileName String cursorResponseField = pageInfo.getPageCursorResponseField(); ReadContext context = JsonPath.using(jsonConfiguration).parse(data); @@ -473,8 +474,11 @@ private void collect(Collector output, String data) throws IOExcep newCursor = cursorList.get(0); } pageInfo.setCursor(newCursor); - // if not present cursor, then no more data - noMoreElementFlag = Strings.isNullOrEmpty(newCursor); + // If the response cursor is empty or unchanged, the next request cannot make + // progress. + noMoreElementFlag = + Strings.isNullOrEmpty(newCursor) + || Objects.equals(currentCursor, newCursor); } else { // if not set page pagination is default // Determine whether the task is completed by specifying the presence of the 'total diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java index 057eeb851b44..13ac4dc8ac13 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java @@ -16,6 +16,7 @@ */ package org.apache.seatunnel.connectors.seatunnel.http; +import org.apache.seatunnel.api.source.Boundedness; import org.apache.seatunnel.api.source.Collector; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; @@ -42,10 +43,13 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class HttpSourceReaderInternalPollNextTest { @@ -135,6 +139,50 @@ public void testPageNumberPlaceHolderRequestBodyUpdate() throws Exception { httpSourceReader.close(); } + @Test + public void testCursorPaginationStopsWhenCursorDoesNotAdvance() throws Exception { + httpParameter.setBody("{\"scrollId\":\"${scrollId}\",\"capacity\":100}"); + + PageInfo pageInfo = new PageInfo(); + pageInfo.setPageType(HttpPaginationType.CURSOR.getCode()); + pageInfo.setPageCursorFieldName("scrollId"); + pageInfo.setPageCursorResponseField("$.scrollId"); + pageInfo.setUsePlaceholderReplacement(true); + + when(context.getBoundedness()).thenReturn(Boundedness.BOUNDED); + AtomicInteger requestCount = new AtomicInteger(); + when(httpClientProvider.execute( + anyString(), anyString(), any(), any(), any(), anyBoolean())) + .thenAnswer( + invocation -> { + int currentRequest = requestCount.incrementAndGet(); + if (currentRequest == 1) { + return new HttpResponse( + 200, + "{\"scrollId\":\"cursor-1\",\"data\":[{\"key1\":\"v1\",\"key2\":\"v2\"}]}"); + } + if (currentRequest == 2) { + return new HttpResponse( + 200, "{\"scrollId\":\"cursor-1\",\"data\":[]}"); + } + throw new AssertionError( + "Cursor pagination should stop when the cursor does not advance"); + }); + + httpSourceReader = + new HttpSourceReader( + httpParameter, context, deserializationSchema, null, "$.data", pageInfo); + httpSourceReader.open(); + httpSourceReader.setHttpClient(httpClientProvider); + + httpSourceReader.internalPollNext(collector); + + verify(httpClientProvider, times(2)) + .execute(anyString(), anyString(), any(), any(), any(), anyBoolean()); + verify(context, times(1)).signalNoMoreElement(); + httpSourceReader.close(); + } + @AfterEach public void tearDown() throws Exception { mock.close(); From c69824769969c52754da3c8f0ac58cf22e15ce24 Mon Sep 17 00:00:00 2001 From: Jast Date: Thu, 18 Jun 2026 08:26:13 +0800 Subject: [PATCH 026/375] [Fix][Connector-V2] Fix Oracle CDC schema cache matching (#11110) Co-authored-by: zhangshenghang --- .../cdc/oracle/utils/OracleSchema.java | 22 ++++++++++++++++--- .../cdc/oracle/utils/OracleUtilsTest.java | 14 ++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleSchema.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleSchema.java index f2713e348115..a2db1d9ceb5f 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleSchema.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleSchema.java @@ -74,14 +74,15 @@ private TableChange readTableSchema(JdbcConnection jdbc, TableId tableId) { null, false); for (TableId id : tables.tableIds()) { - if (tableMap.containsKey(id)) { + TableId tableMapId = resolveTableId(id, tableId, tableMap); + if (tableMap.containsKey(tableMapId)) { Table table = CatalogTableUtils.mergeCatalogTableConfig( - tables.forTable(id), tableMap.get(id)); + tables.forTable(id), tableMap.get(tableMapId)); TableChanges.TableChange tableChange = new TableChanges.TableChange( TableChanges.TableChangeType.CREATE, table); - schemasByTableId.put(id, tableChange); + schemasByTableId.put(tableMapId, tableChange); } } } catch (SQLException e) { @@ -96,4 +97,19 @@ private TableChange readTableSchema(JdbcConnection jdbc, TableId tableId) { return schemasByTableId.get(tableId); } + + static TableId resolveTableId( + TableId readTableId, TableId requestedTableId, Map tableMap) { + if (tableMap.containsKey(readTableId)) { + return readTableId; + } + + TableId readTableIdWithRequestedCatalog = + new TableId(requestedTableId.catalog(), readTableId.schema(), readTableId.table()); + if (tableMap.containsKey(readTableIdWithRequestedCatalog)) { + return readTableIdWithRequestedCatalog; + } + + return readTableId; + } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleUtilsTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleUtilsTest.java index 10c253da83e9..a3856a7ff70d 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleUtilsTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleUtilsTest.java @@ -26,6 +26,9 @@ import io.debezium.relational.TableId; +import java.util.Collections; +import java.util.Map; + public class OracleUtilsTest { @Test public void testSplitScanQuery() { @@ -70,4 +73,15 @@ public void testSplitScanQuery() { Assertions.assertEquals( "SELECT * FROM \"schema1\".\"table1\" WHERE \"id\" >= ?", splitScanSQL); } + + @Test + public void testResolveTableIdWithRequestedCatalog() { + TableId requestedTableId = TableId.parse("ORCLPDB.LIB_B.T_B1"); + TableId readTableIdWithoutCatalog = new TableId(null, "LIB_B", "T_B1"); + Map tableMap = Collections.singletonMap(requestedTableId, null); + + Assertions.assertEquals( + requestedTableId, + OracleSchema.resolveTableId(readTableIdWithoutCatalog, requestedTableId, tableMap)); + } } From 34940bf66cd4fd6b07f28fe4b5eb3e6211587b71 Mon Sep 17 00:00:00 2001 From: Shuai Liu <390105636@qq.com> Date: Thu, 18 Jun 2026 08:41:31 +0800 Subject: [PATCH 027/375] [Feature][Connector-V2][CDC] Add include/exclude filtering for schema change event types #11044 (#11108) Co-authored-by: Claude Opus 4.8 (1M context) --- docs/en/connectors/source/MySQL-CDC.md | 39 +++ docs/en/connectors/source/Oracle-CDC.md | 38 +++ docs/en/connectors/source/SqlServer-CDC.md | 39 +++ docs/zh/connectors/source/MySQL-CDC.md | 38 +++ docs/zh/connectors/source/Oracle-CDC.md | 37 +++ docs/zh/connectors/source/SqlServer-CDC.md | 38 +++ .../cdc/base/option/SourceOptions.java | 25 ++ .../base/schema/SchemaChangeEventFilter.java | 174 ++++++++++ .../base/schema/SchemaChangeEventType.java | 95 ++++++ .../BaseChangeStreamTableSourceFactory.java | 3 + ...SeaTunnelRowDebeziumDeserializeSchema.java | 18 + .../schema/SchemaChangeEventFilterTest.java | 308 ++++++++++++++++++ .../mysql/source/MySqlIncrementalSource.java | 2 + .../source/MySqlIncrementalSourceFactory.java | 2 + .../source/OracleIncrementalSource.java | 2 + .../OracleIncrementalSourceFactory.java | 4 +- .../source/SqlServerIncrementalSource.java | 2 + .../SqlServerIncrementalSourceFactory.java | 4 +- .../cdc/mysql/MysqlCDCWithSchemaChangeIT.java | 121 +++++++ .../test/resources/ddl/add_columns_filter.sql | 24 ++ .../resources/ddl/drop_columns_filter.sql | 29 ++ .../src/test/resources/ddl/shop.sql | 8 + ...dc_to_mysql_with_schema_change_filter.conf | 53 +++ 23 files changed, 1101 insertions(+), 2 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventFilter.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventType.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventFilterTest.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/add_columns_filter.sql create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/drop_columns_filter.sql create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_schema_change_filter.conf diff --git a/docs/en/connectors/source/MySQL-CDC.md b/docs/en/connectors/source/MySQL-CDC.md index 2210231123aa..5ee7a0c64736 100644 --- a/docs/en/connectors/source/MySQL-CDC.md +++ b/docs/en/connectors/source/MySQL-CDC.md @@ -215,6 +215,8 @@ When an initial consistent snapshot is made for large databases, your establishe | exactly_once | Boolean | No | false | Enable exactly once semantic. | | format | Enum | No | DEFAULT | Optional output format for MySQL CDC, valid enumerations are `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`. | | schema-changes.enabled | Boolean | No | false | Schema evolution is disabled by default. Now we only support `add column`、`drop column`、`rename column` and `modify column`. | +| schema-changes.include | List | No | - | Only the listed schema change event types are sent downstream (when `schema-changes.enabled = true`). Empty means all are eligible. See [Schema change event filtering](#schema-change-event-filtering). | +| schema-changes.exclude | List | No | - | Schema change event types listed here are NOT sent downstream. Applied after `schema-changes.include`; exclude wins on conflict. See [Schema change event filtering](#schema-change-event-filtering). | | debezium | Config | No | - | Pass-through [Debezium's properties](https://github.com/debezium/debezium/blob/v1.9.8.Final/documentation/modules/ROOT/pages/connectors/mysql.adoc#connector-properties) to Debezium Embedded Engine which is used to capture data changes from MySQL server. | | int_type_narrowing | Boolean | No | true | Int type narrowing, if true, the tinyint(1) type will be narrowed to the boolean type if without loss of precision. Support for MySQL at now. Please refer to `int_type_narrowing` below | | common-options | | no | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details | @@ -340,6 +342,43 @@ sink { } ``` + +### Schema change event filtering + +When `schema-changes.enabled = true`, you can further control which schema change event types are +propagated downstream using `schema-changes.include` / `schema-changes.exclude`. + +Use these SeaTunnel-owned canonical names: + +| Canonical name | Operation | +|-----------------|---------------------------------------------------------------------------| +| `add.column` | add a column | +| `drop.column` | drop a column | +| `modify.column` | change a column's type/attributes, name unchanged | +| `change.column` | rename a column, optionally re-type | +| `update.columns`| group alias for all four column-level changes above | + +Precedence is deterministic: + +1. if `schema-changes.include` is set, only included event types are eligible; +2. `schema-changes.exclude` is then applied; +3. **exclude wins** when a type appears in both lists. + +```hocon +source { + MySQL-CDC { + # ... + schema-changes.enabled = true + schema-changes.include = ["add.column", "drop.column"] + schema-changes.exclude = ["change.column"] + } +} +``` + +**Data handling when `drop.column` is excluded. For a retained **NOT NULL** column the `NULL` write is rejected +by the sink, so excluding `drop.column` for a NOT NULL column that the source has stopped supplying +will fail at the sink. + ### Support table-pattern for multi-table reading > `table-pattern` and `table-names` are mutually exclusive diff --git a/docs/en/connectors/source/Oracle-CDC.md b/docs/en/connectors/source/Oracle-CDC.md index c67fec3a1a05..5393335b44da 100644 --- a/docs/en/connectors/source/Oracle-CDC.md +++ b/docs/en/connectors/source/Oracle-CDC.md @@ -254,6 +254,8 @@ exit; | skip_analyze | Boolean | No | false | Skip the analysis of table count in full stage.In this scenario, you schedule analysis table sql to update related table statistics periodically or your table data does not change frequently | | format | Enum | No | DEFAULT | Optional output format for Oracle CDC, valid enumerations are `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`. | | schema-changes.enabled | Boolean | No | false | Schema evolution is disabled by default. Now we only support `add column`、`drop column`、`rename column` and `modify column`. | +| schema-changes.include | List | No | - | Only the listed schema change event types are sent downstream (when `schema-changes.enabled = true`). Empty means all are eligible. See [Schema change event filtering](#schema-change-event-filtering). | +| schema-changes.exclude | List | No | - | Schema change event types listed here are NOT sent downstream. Applied after `schema-changes.include`; exclude wins on conflict. See [Schema change event filtering](#schema-change-event-filtering). | | debezium | Config | No | - | Pass-through [Debezium's properties](https://github.com/debezium/debezium/blob/v1.9.8.Final/documentation/modules/ROOT/pages/connectors/oracle.adoc#connector-properties) to Debezium Embedded Engine which is used to capture data changes from Oracle server. | | common-options | | no | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details | | decimal_type_narrowing | Boolean | No | true | Decimal type narrowing, if true, the decimal type will be narrowed to the int or long type if without loss of precision. Only support for Oracle at now. Please refer to `decimal_type_narrowing` below | @@ -363,6 +365,42 @@ source { } ``` +### Schema change event filtering + +When `schema-changes.enabled = true`, you can further control which schema change event types are +propagated downstream using `schema-changes.include` / `schema-changes.exclude`. + +Use these SeaTunnel-owned canonical names: + +| Canonical name | Operation | +|------------------|------------------------------------------------------| +| `add.column` | add a column | +| `drop.column` | drop a column | +| `modify.column` | change a column's type/attributes, name unchanged | +| `change.column` | rename a column, optionally re-type | +| `update.columns` | group alias for all four column-level changes above | + +Precedence is deterministic: + +1. if `schema-changes.include` is set, only included event types are eligible; +2. `schema-changes.exclude` is then applied; +3. **exclude wins** when a type appears in both lists. + +```hocon +source { + Oracle-CDC { + # ... + schema-changes.enabled = true + schema-changes.include = ["add.column", "drop.column"] + schema-changes.exclude = ["change.column"] + } +} +``` + +**Data handling when `drop.column` is excluded. For a retained **NOT NULL** column the `NULL` write is rejected +by the sink, so excluding `drop.column` for a NOT NULL column that the source has stopped supplying +will fail at the sink. + ### Support debezium-compatible format send to kafka > Must be used with kafka connector sink, see [compatible debezium format](../formats/cdc-compatible-debezium-json.md) for details diff --git a/docs/en/connectors/source/SqlServer-CDC.md b/docs/en/connectors/source/SqlServer-CDC.md index 5ca2cfb2ce37..56cb05c39036 100644 --- a/docs/en/connectors/source/SqlServer-CDC.md +++ b/docs/en/connectors/source/SqlServer-CDC.md @@ -104,6 +104,9 @@ case-sensitive databases, make sure the configured identifier case matches the d | exactly_once | Boolean | No | false | Enable exactly once semantic. | | debezium.* | config | No | - | Pass-through Debezium's properties to Debezium Embedded Engine which is used to capture data changes from SqlServer server.
See more about
the [Debezium's SqlServer Connector properties](https://github.com/debezium/debezium/blob/1.6/documentation/modules/ROOT/pages/connectors/sqlserver.adoc#connector-properties) | | format | Enum | No | DEFAULT | Optional output format for SqlServer CDC, valid enumerations are "DEFAULT"、"COMPATIBLE_DEBEZIUM_JSON". | +| schema-changes.enabled | Boolean | No | false | Schema evolution is disabled by default. Now we only support `add column`、`drop column`、`rename column` and `modify column`. | +| schema-changes.include | List | No | - | Only the listed schema change event types are sent downstream (when `schema-changes.enabled = true`). Empty means all are eligible. See [Schema change event filtering](#schema-change-event-filtering). | +| schema-changes.exclude | List | No | - | Schema change event types listed here are NOT sent downstream. Applied after `schema-changes.include`; exclude wins on conflict. See [Schema change event filtering](#schema-change-event-filtering). | | common-options | | no | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details. | ### Enable Sql Server CDC @@ -241,6 +244,42 @@ sink { } ``` +### Schema change event filtering + +When `schema-changes.enabled = true`, you can further control which schema change event types are +propagated downstream using `schema-changes.include` / `schema-changes.exclude`. + +Use these SeaTunnel-owned canonical names: + +| Canonical name | Operation | +|------------------|------------------------------------------------------| +| `add.column` | add a column | +| `drop.column` | drop a column | +| `modify.column` | change a column's type/attributes, name unchanged | +| `change.column` | rename a column, optionally re-type | +| `update.columns` | group alias for all four column-level changes above | + +Precedence is deterministic: + +1. if `schema-changes.include` is set, only included event types are eligible; +2. `schema-changes.exclude` is then applied; +3. **exclude wins** when a type appears in both lists. + +```hocon +source { + SqlServer-CDC { + # ... + schema-changes.enabled = true + schema-changes.include = ["add.column", "drop.column"] + schema-changes.exclude = ["change.column"] + } +} +``` + +**Data handling when `drop.column` is excluded. For a retained **NOT NULL** column the `NULL` write is rejected +by the sink, so excluding `drop.column` for a NOT NULL column that the source has stopped supplying +will fail at the sink. + ## Changelog diff --git a/docs/zh/connectors/source/MySQL-CDC.md b/docs/zh/connectors/source/MySQL-CDC.md index c08e6596ce61..b8c27ecc0fe8 100644 --- a/docs/zh/connectors/source/MySQL-CDC.md +++ b/docs/zh/connectors/source/MySQL-CDC.md @@ -214,6 +214,8 @@ show variables where variable_name in ('log_bin', 'binlog_format', 'binlog_row_i | exactly_once | Boolean | 否 | false | 启用精确一次语义. | | format | Enum | 否 | DEFAULT | MySQL CDC 的可选输出格式, 有效的枚举值为 `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`. | | schema-changes.enabled | Boolean | 否 | false | 模式演进默认是禁用的. 当前我们只支持 `add column`、`drop column`、`rename column` 和 `modify column`. | +| schema-changes.include | List | 否 | - | 仅向下游发送列出的 schema change 事件类型（需 `schema-changes.enabled = true`）。为空表示全部允许。详见 [Schema change 事件过滤](#schema-change-事件过滤)。 | +| schema-changes.exclude | List | 否 | - | 此处列出的 schema change 事件类型不会发送到下游。在 `schema-changes.include` 之后应用；冲突时 exclude 优先。详见 [Schema change 事件过滤](#schema-change-事件过滤)。 | | debezium | Config | 否 | - | 传递 [Debezium的属性](https://github.com/debezium/debezium/blob/v1.9.8.Final/documentation/modules/ROOT/pages/connectors/mysql.adoc#connector-properties) 给Debezium嵌入式引擎, 该引擎用于捕获 MySQL 服务的数据变更. | | int_type_narrowing | Boolean | 否 | true | Int类型收窄，如果为 true，则 tinyint(1) 类型将被收窄为 boolean 类型（如果没有精度损失）。目前仅支持 MySQL。 | | common-options | | 否 | - | Source插件通用参数, 详见 [Source Common Options](../common-options/source-common-options.md) | @@ -339,6 +341,42 @@ sink { } ``` + +### Schema change 事件过滤 + +当 `schema-changes.enabled = true` 时，可通过 `schema-changes.include` / `schema-changes.exclude` 进一步 +控制哪些 schema change 事件类型会被发送到下游。 + +使用以下 SeaTunnel 统一的规范名称 + +| 规范名称 | 操作 | +|-----------------|-------------------------------------------------------------| +| `add.column` | 新增列 | +| `drop.column` | 删除列 | +| `modify.column` | 修改列的类型/属性，列名不变 | +| `change.column` | 列重命名，可同时改类型 | +| `update.columns` | 上述四种列级变更的分组别名 | + +优先级规则（确定性）： + +1. 若设置了 `schema-changes.include`，则只有被包含的事件类型才有资格； +2. 然后应用 `schema-changes.exclude`； +3. 当某类型同时出现在两个列表中时，**exclude 优先**。 + +```hocon +source { + MySQL-CDC { + # ... + schema-changes.enabled = true + schema-changes.include = ["add.column", "drop.column"] + schema-changes.exclude = ["change.column"] + } +} +``` + +**排除 `drop.column` 时的数据处理方式。对于被保留的 **NOT NULL** 列，写入 `NULL` 会被 sink 拒绝，因此对一个源端已不再供数的 +NOT NULL 列排除 `drop.column` 会在 sink 端失败。 + ### 表名支持正则以读取多个表 > `table-pattern` 和 `table-names` 只能选择一个 diff --git a/docs/zh/connectors/source/Oracle-CDC.md b/docs/zh/connectors/source/Oracle-CDC.md index d542a5057f8f..534144ce3096 100644 --- a/docs/zh/connectors/source/Oracle-CDC.md +++ b/docs/zh/connectors/source/Oracle-CDC.md @@ -253,6 +253,8 @@ exit; | skip_analyze | Boolean | 否 | false | 在全量阶段跳过表行数的分析。在这种情况下，您需要定期调度分析表 SQL 以更新相关表统计信息，或者您的表数据更改不频繁。 | | format | Enum | 否 | DEFAULT | Oracle CDC 的可选输出格式，有效枚举值为 `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`。 | | schema-changes.enabled | Boolean | 否 | false | Schema 演进默认禁用。目前我们仅支持 `add column`、`drop column`、`rename column` 和 `modify column`。 | +| schema-changes.include | List | 否 | - | 仅向下游发送列出的 schema change 事件类型（需 `schema-changes.enabled = true`）。为空表示全部允许。详见 [Schema change 事件过滤](#schema-change-事件过滤)。 | +| schema-changes.exclude | List | 否 | - | 此处列出的 schema change 事件类型不会发送到下游。在 `schema-changes.include` 之后应用；冲突时 exclude 优先。详见 [Schema change 事件过滤](#schema-change-事件过滤)。 | | debezium | Config | 否 | - | 透传 [Debezium 属性](https://github.com/debezium/debezium/blob/v1.9.8.Final/documentation/modules/ROOT/pages/connectors/oracle.adoc#connector-properties) 给 Debezium Embedded Engine，该引擎用于捕获 Oracle 服务器的数据更改。 | | common-options | | 否 | - | 源端插件常用参数，详情请参阅 [源端常用选项](../common-options/source-common-options.md)。 | | decimal_type_narrowing | Boolean | 否 | true | 数值类型收缩，如果为 true，则在不损失精度的情况下，将 decimal 类型收缩为 int 或 long 类型。目前仅支持 Oracle。请参阅下文的 `decimal_type_narrowing`。 | @@ -362,6 +364,41 @@ source { } ``` +### Schema change 事件过滤 + +当 `schema-changes.enabled = true` 时，可通过 `schema-changes.include` / `schema-changes.exclude` 进一步 +控制哪些 schema change 事件类型会被发送到下游。过滤只影响“发往下游”的部分。 + +使用以下 SeaTunnel 统一的规范名称： + +| 规范名称 | 操作 | +|------------------|---------------------------------------------| +| `add.column` | 新增列 | +| `drop.column` | 删除列 | +| `modify.column` | 修改列的类型/属性，列名不变 | +| `change.column` | 列重命名，可同时改类型 | +| `update.columns` | 上述四种列级变更的分组别名 | + +优先级规则（确定性）： + +1. 若设置了 `schema-changes.include`，则只有被包含的事件类型才有资格； +2. 然后应用 `schema-changes.exclude`； +3. 当某类型同时出现在两个列表中时，**exclude 优先**。 + +```hocon +source { + Oracle-CDC { + # ... + schema-changes.enabled = true + schema-changes.include = ["add.column", "drop.column"] + schema-changes.exclude = ["change.column"] + } +} +``` + +**排除 `drop.column` 时的数据处理方式。对于被保留的 **NOT NULL** 列，写入 `NULL` 会被 sink 拒绝，因此对一个源端已不再供数的 +NOT NULL 列排除 `drop.column` 会在 sink 端失败。 + ### 支持以兼容 debezium 的格式发送到 kafka > 必须与 kafka 连接器 sink 配合使用，详情请参阅 [兼容 debezium 格式](../formats/cdc-compatible-debezium-json.md) diff --git a/docs/zh/connectors/source/SqlServer-CDC.md b/docs/zh/connectors/source/SqlServer-CDC.md index ee849ea7f43f..c7453e8d14d5 100644 --- a/docs/zh/connectors/source/SqlServer-CDC.md +++ b/docs/zh/connectors/source/SqlServer-CDC.md @@ -102,6 +102,9 @@ Sql Server CDC 连接器允许从 SqlServer 数据库读取快照数据和增量 | exactly_once | Boolean | 否 | false | 启用精确一次语义。 | | debezium.* | config | 否 | - | 将 Debezium 的属性传递给 Debezium Embedded Engine，用于捕获来自 SqlServer 服务器的数据变更。
了解更多关于
[Debezium 的 SqlServer 连接器属性](https://github.com/debezium/debezium/blob/1.6/documentation/modules/ROOT/pages/connectors/sqlserver.adoc#connector-properties) | | format | Enum | 否 | DEFAULT | SqlServer CDC 的可选输出格式，有效枚举为 "DEFAULT"、"COMPATIBLE_DEBEZIUM_JSON"。 | +| schema-changes.enabled | Boolean | 否 | false | 模式演进默认是禁用的。当前我们只支持 `add column`、`drop column`、`rename column` 和 `modify column`。 | +| schema-changes.include | List | 否 | - | 仅向下游发送列出的 schema change 事件类型（需 `schema-changes.enabled = true`）。为空表示全部允许。详见 [Schema change 事件过滤](#schema-change-事件过滤)。 | +| schema-changes.exclude | List | 否 | - | 此处列出的 schema change 事件类型不会发送到下游。在 `schema-changes.include` 之后应用；冲突时 exclude 优先。详见 [Schema change 事件过滤](#schema-change-事件过滤)。 | | common-options | | 否 | - | 源插件通用参数，请参考 [源通用选项](../common-options/source-common-options.md) 获取详细信息。 | ### 启用 Sql Server CDC @@ -239,6 +242,41 @@ sink { } ``` +### Schema change 事件过滤 + +当 `schema-changes.enabled = true` 时，可通过 `schema-changes.include` / `schema-changes.exclude` 进一步 +控制哪些 schema change 事件类型会被发送到下游。 + +使用以下 SeaTunnel 统一的规范名称： + +| 规范名称 | 操作 | +|------------------|---------------------------------------------| +| `add.column` | 新增列 | +| `drop.column` | 删除列 | +| `modify.column` | 修改列的类型/属性，列名不变 | +| `change.column` | 列重命名，可同时改类型 | +| `update.columns` | 上述四种列级变更的分组别名 | + +优先级规则（确定性）： + +1. 若设置了 `schema-changes.include`，则只有被包含的事件类型才有资格； +2. 然后应用 `schema-changes.exclude`； +3. 当某类型同时出现在两个列表中时，**exclude 优先**。 + +```hocon +source { + SqlServer-CDC { + # ... + schema-changes.enabled = true + schema-changes.include = ["add.column", "drop.column"] + schema-changes.exclude = ["change.column"] + } +} +``` + +**排除 `drop.column` 时的数据处理方式。对于被保留的 **NOT NULL** 列，写入 `NULL` 会被 sink 拒绝，因此对一个源端已不再供数的 +NOT NULL 列排除 `drop.column` 会在 sink 端失败。 + ## 变更日志 diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/option/SourceOptions.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/option/SourceOptions.java index 013d798642c0..97b229aedf4f 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/option/SourceOptions.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/option/SourceOptions.java @@ -20,8 +20,11 @@ import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.Options; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.connectors.cdc.base.schema.SchemaChangeEventType; import org.apache.seatunnel.connectors.cdc.debezium.DeserializeFormat; +import java.util.Collections; +import java.util.List; import java.util.Map; @SuppressWarnings("MagicNumber") @@ -121,6 +124,28 @@ public class SourceOptions { .withDescription( "Enable send schema change events, by default is false. If set to true, the schema changes will be sent to downstream."); + public static final Option> SCHEMA_CHANGES_INCLUDE = + Options.key("schema-changes.include") + .listType() + .defaultValue(Collections.emptyList()) + .withDescription( + "Only schema change event types listed here are sent downstream when schema-changes.enabled is true. " + + "Empty means all event types are eligible. " + + "Valid values: " + + SchemaChangeEventType.validNames() + + " (update.columns is a group alias for all column-level changes). "); + + public static final Option> SCHEMA_CHANGES_EXCLUDE = + Options.key("schema-changes.exclude") + .listType() + .defaultValue(Collections.emptyList()) + .withDescription( + "Schema change event types listed here are NOT sent downstream. Applied after schema-changes.include; " + + "exclude wins when a type appears in both lists. " + + "Valid values: " + + SchemaChangeEventType.validNames() + + " (update.columns is a group alias for all column-level changes). "); + public static OptionRule.Builder getBaseRule() { return OptionRule.builder() .optional(FORMAT) diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventFilter.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventFilter.java new file mode 100644 index 000000000000..3ebfb4b5519d --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventFilter.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.cdc.base.schema; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.event.EventType; +import org.apache.seatunnel.api.table.schema.event.AlterTableColumnEvent; +import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent; +import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent; +import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; + +import java.io.Serializable; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Include/exclude filter for CDC schema change event types. + * + *

Applied after schema change events are normalized to the SeaTunnel event model, and before + * they are emitted to downstream schema-change coordination. A filtered-out event is neither + * forwarded to downstream coordination nor applied to the produced schema, so the produced row + * shape stays in lockstep with the (filtered) sink schema — this is what keeps a column whose + * {@code drop.column} was suppressed by {@code exclude} present on both sides. + * + *

Precedence (deterministic): + * + *

    + *
  1. if {@code include} is non-empty, only included event types are eligible; + *
  2. {@code exclude} is then applied; + *
  3. {@code exclude} wins when the same type appears in both lists. + *
+ * + *

{@code update.columns} acts as a group alias for all column-level changes ({@code add.column}, + * {@code drop.column}, {@code modify.column}, {@code change.column}): including it admits the whole + * group, excluding it suppresses the whole group. + */ +public final class SchemaChangeEventFilter implements Serializable { + + private static final long serialVersionUID = 1L; + + private final Set includeTypes; + private final Set excludeTypes; + + public SchemaChangeEventFilter(Set includeTypes, Set excludeTypes) { + this.includeTypes = new HashSet<>(includeTypes); + this.excludeTypes = new HashSet<>(excludeTypes); + } + + /** Builds a filter from the {@code schema-changes.include} / {@code exclude} options. */ + public static SchemaChangeEventFilter fromConfig(ReadonlyConfig config) { + List include = config.get(SourceOptions.SCHEMA_CHANGES_INCLUDE); + List exclude = config.get(SourceOptions.SCHEMA_CHANGES_EXCLUDE); + return new SchemaChangeEventFilter( + SchemaChangeEventType.fromCanonicalNames(include), + SchemaChangeEventType.fromCanonicalNames(exclude)); + } + + /** + * Validates the {@code schema-changes.include} / {@code schema-changes.exclude} option values. + * + *

Invoked at job submission time (from the source factory) so an unknown canonical name — + * e.g. a typo such as {@code rename.tabble} — fails fast during submission with a message + * listing the valid names, instead of bypassing submission-time option validation and failing + * later during source initialization. + */ + public static void validateOptions(ReadonlyConfig config) { + validateNames( + SourceOptions.SCHEMA_CHANGES_INCLUDE.key(), + config.get(SourceOptions.SCHEMA_CHANGES_INCLUDE)); + validateNames( + SourceOptions.SCHEMA_CHANGES_EXCLUDE.key(), + config.get(SourceOptions.SCHEMA_CHANGES_EXCLUDE)); + } + + private static void validateNames(String optionKey, List names) { + try { + SchemaChangeEventType.fromCanonicalNames(names); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Invalid value for option '" + optionKey + "'. " + e.getMessage(), e); + } + } + + public boolean isNoOp() { + return includeTypes.isEmpty() && excludeTypes.isEmpty(); + } + + /** + * Applies the filter to a normalized schema change event. + * + * @return the original event when fully eligible, a reduced {@link AlterTableColumnsEvent} when + * only some of its column sub-events are eligible, or {@code null} when the whole event is + * filtered out. + */ + public SchemaChangeEvent filter(SchemaChangeEvent event) { + if (event == null || isNoOp()) { + return event; + } + + if (event instanceof AlterTableColumnsEvent) { + AlterTableColumnsEvent composite = (AlterTableColumnsEvent) event; + List survivors = + composite.getEvents().stream() + .filter(sub -> isColumnLevelEligible(sub.getEventType())) + .collect(Collectors.toList()); + if (survivors.isEmpty()) { + return null; + } + if (survivors.size() == composite.getEvents().size()) { + return composite; + } + return rebuildComposite(composite, survivors); + } + + if (event instanceof AlterTableColumnEvent) { + // A standalone column-level event (not wrapped in a composite). + return isColumnLevelEligible(event.getEventType()) ? event : null; + } + + // Table-level events. No canonical name currently maps to a table-level type (rename.table + // is not exposed yet), so this is defensive: such events are not produced today, but if one + // arrives it is filtered consistently with the include/exclude precedence. + return isEligible(event.getEventType()) ? event : null; + } + + /** Eligibility for table-level event types (no group alias). */ + private boolean isEligible(EventType type) { + boolean included = includeTypes.isEmpty() || includeTypes.contains(type); + return included && !excludeTypes.contains(type); + } + + /** + * Eligibility for column-level event types, honoring {@code update.columns} as the group alias + * for the whole column-change family. + */ + private boolean isColumnLevelEligible(EventType type) { + boolean included = + includeTypes.isEmpty() + || includeTypes.contains(type) + || includeTypes.contains(EventType.SCHEMA_CHANGE_UPDATE_COLUMNS); + boolean excluded = + excludeTypes.contains(type) + || excludeTypes.contains(EventType.SCHEMA_CHANGE_UPDATE_COLUMNS); + return included && !excluded; + } + + private static AlterTableColumnsEvent rebuildComposite( + AlterTableColumnsEvent original, List survivors) { + AlterTableColumnsEvent reduced = + new AlterTableColumnsEvent(original.tableIdentifier(), survivors); + reduced.setStatement(original.getStatement()); + reduced.setSourceDialectName(original.getSourceDialectName()); + reduced.setChangeAfter(original.getChangeAfter()); + reduced.setJobId(original.getJobId()); + return reduced; + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventType.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventType.java new file mode 100644 index 000000000000..dec081d9bc48 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventType.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.cdc.base.schema; + +import org.apache.seatunnel.api.event.EventType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Maps the user-facing canonical names used in {@code schema-changes.include} / {@code + * schema-changes.exclude} to the internal {@link EventType} enum, so users never need to know + * internal class or enum names. {@code update.columns} is a group alias for all column-level + * changes. + */ +public final class SchemaChangeEventType { + + private static final Map CANONICAL_NAME_TO_EVENT_TYPE; + + static { + Map map = new LinkedHashMap<>(); + map.put("add.column", EventType.SCHEMA_CHANGE_ADD_COLUMN); + map.put("drop.column", EventType.SCHEMA_CHANGE_DROP_COLUMN); + map.put("modify.column", EventType.SCHEMA_CHANGE_MODIFY_COLUMN); + map.put("change.column", EventType.SCHEMA_CHANGE_CHANGE_COLUMN); + map.put("update.columns", EventType.SCHEMA_CHANGE_UPDATE_COLUMNS); + // NOTE: rename.table (SCHEMA_CHANGE_RENAME_TABLE) is intentionally NOT exposed yet. CDC has + // no end-to-end handling for table renames: the DDL is never parsed into an + // AlterTableNameEvent, the schema handlers treat that event as a no-op, and no sink applies + // it. Exposing it as a filterable name would advertise a capability that does not exist. + // It should be added back only once table-rename is implemented end-to-end (see the + // rename-table design follow-up). + CANONICAL_NAME_TO_EVENT_TYPE = Collections.unmodifiableMap(map); + } + + private SchemaChangeEventType() {} + + public static String validNames() { + return String.join(", ", CANONICAL_NAME_TO_EVENT_TYPE.keySet()); + } + + public static EventType fromCanonicalName(String canonicalName) { + if (canonicalName == null) { + throw new IllegalArgumentException( + "Schema change event type name must not be null. Valid names are: " + + validNames()); + } + String normalized = canonicalName.trim().toLowerCase(); + EventType eventType = CANONICAL_NAME_TO_EVENT_TYPE.get(normalized); + if (eventType == null) { + throw new IllegalArgumentException( + "Unknown schema change event type '" + + canonicalName + + "'. Valid names are: " + + validNames()); + } + return eventType; + } + + public static Set fromCanonicalNames(Collection canonicalNames) { + if (canonicalNames == null || canonicalNames.isEmpty()) { + return Collections.emptySet(); + } + return canonicalNames.stream() + .map(SchemaChangeEventType::fromCanonicalName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + /** Visible for testing: the supported canonical names. */ + static List canonicalNames() { + return new ArrayList<>(CANONICAL_NAME_TO_EVENT_TYPE.keySet()); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/BaseChangeStreamTableSourceFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/BaseChangeStreamTableSourceFactory.java index a801a626e168..bc2c4961cd22 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/BaseChangeStreamTableSourceFactory.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/BaseChangeStreamTableSourceFactory.java @@ -25,6 +25,7 @@ import org.apache.seatunnel.api.table.factory.ChangeStreamTableSourceFactory; import org.apache.seatunnel.api.table.factory.ChangeStreamTableSourceState; import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; +import org.apache.seatunnel.connectors.cdc.base.schema.SchemaChangeEventFilter; import org.apache.seatunnel.connectors.cdc.base.source.split.IncrementalSplit; import org.apache.seatunnel.connectors.cdc.base.source.split.SourceSplitBase; @@ -47,6 +48,7 @@ public abstract class BaseChangeStreamTableSourceFactory implements ChangeStream @Override public TableSource createSource(TableSourceFactoryContext context) { + SchemaChangeEventFilter.validateOptions(context.getOptions()); return restoreSource(context, Collections.emptyList()); } @@ -55,6 +57,7 @@ TableSource createSource(TableSourceFactoryContext context) { TableSource restoreSource( TableSourceFactoryContext context, ChangeStreamTableSourceState state) { + SchemaChangeEventFilter.validateOptions(context.getOptions()); return restoreSource(context, getRestoreTableStruct(state)); } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializeSchema.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializeSchema.java index 4dcfb770782f..4c237b1bcdbd 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializeSchema.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializeSchema.java @@ -30,6 +30,7 @@ import org.apache.seatunnel.api.table.type.MetadataUtil; import org.apache.seatunnel.api.table.type.RowKind; import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.cdc.base.schema.SchemaChangeEventFilter; import org.apache.seatunnel.connectors.cdc.base.schema.SchemaChangeResolver; import org.apache.seatunnel.connectors.cdc.base.utils.SourceRecordUtils; import org.apache.seatunnel.connectors.cdc.debezium.AbstractDebeziumDeserializationSchema; @@ -72,6 +73,7 @@ public final class SeaTunnelRowDebeziumDeserializeSchema private final ZoneId serverTimeZone; private final DebeziumDeserializationConverterFactory userDefinedConverterFactory; private final SchemaChangeResolver schemaChangeResolver; + private final SchemaChangeEventFilter schemaChangeEventFilter; private final TableSchemaChangeEventHandler tableSchemaChangeHandler; private List tables; private Map tableRowConverters; @@ -82,6 +84,7 @@ public final class SeaTunnelRowDebeziumDeserializeSchema ZoneId serverTimeZone, DebeziumDeserializationConverterFactory userDefinedConverterFactory, SchemaChangeResolver schemaChangeResolver, + SchemaChangeEventFilter schemaChangeEventFilter, Map tableIdTableChangeMap) { super(tableIdTableChangeMap); this.metadataConverters = metadataConverters; @@ -89,6 +92,7 @@ public final class SeaTunnelRowDebeziumDeserializeSchema this.userDefinedConverterFactory = userDefinedConverterFactory; this.tables = checkNotNull(tables); this.schemaChangeResolver = schemaChangeResolver; + this.schemaChangeEventFilter = schemaChangeEventFilter; this.tableSchemaChangeHandler = new TableSchemaChangeEventDispatcher(); this.tableRowConverters = createTableRowConverters( @@ -136,6 +140,18 @@ private void deserializeSchemaChangeRecord( log.warn("Unsupported resolve schemaChangeEvent {}, just skip.", record); return; } + + // Filter before updating the produced schema, so the produced row shape stays in lockstep + // with the (filtered) sink schema. Only surviving events are applied below. + if (schemaChangeEventFilter != null) { + schemaChangeEvent = schemaChangeEventFilter.filter(schemaChangeEvent); + } + if (schemaChangeEvent == null) { + log.debug( + "Schema change event is fully filtered out by schema-changes.include/exclude, not applied to schema and not sent downstream."); + return; + } + boolean tableExist = false; for (int i = 0; i < tables.size(); i++) { CatalogTable changeBefore = tables.get(i); @@ -394,6 +410,7 @@ public static class Builder { DebeziumDeserializationConverterFactory.DEFAULT; private Map tableIdTableChangeMap = new HashMap<>(); private SchemaChangeResolver schemaChangeResolver; + private SchemaChangeEventFilter schemaChangeEventFilter; public SeaTunnelRowDebeziumDeserializeSchema build() { return new SeaTunnelRowDebeziumDeserializeSchema( @@ -402,6 +419,7 @@ public SeaTunnelRowDebeziumDeserializeSchema build() { serverTimeZone, userDefinedConverterFactory, schemaChangeResolver, + schemaChangeEventFilter, tableIdTableChangeMap); } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventFilterTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventFilterTest.java new file mode 100644 index 000000000000..edd42553f02e --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/schema/SchemaChangeEventFilterTest.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.cdc.base.schema; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.event.EventType; +import org.apache.seatunnel.api.table.catalog.Column; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent; +import org.apache.seatunnel.api.table.schema.event.AlterTableChangeColumnEvent; +import org.apache.seatunnel.api.table.schema.event.AlterTableColumnEvent; +import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent; +import org.apache.seatunnel.api.table.schema.event.AlterTableDropColumnEvent; +import org.apache.seatunnel.api.table.schema.event.AlterTableModifyColumnEvent; +import org.apache.seatunnel.api.table.schema.event.AlterTableNameEvent; +import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent; +import org.apache.seatunnel.api.table.type.BasicType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +class SchemaChangeEventFilterTest { + + private static final TableIdentifier TABLE = TableIdentifier.of("", "db", "tbl"); + + private static Column column(String name) { + return PhysicalColumn.of(name, BasicType.STRING_TYPE, 10L, true, null, ""); + } + + private static SchemaChangeEventFilter filter(List include, List exclude) { + Map map = new HashMap<>(); + map.put("schema-changes.include", include); + map.put("schema-changes.exclude", exclude); + return SchemaChangeEventFilter.fromConfig(ReadonlyConfig.fromMap(map)); + } + + private static AlterTableColumnsEvent composite(AlterTableColumnEvent... events) { + return new AlterTableColumnsEvent(TABLE, new ArrayList<>(Arrays.asList(events))); + } + + private static List subTypes(SchemaChangeEvent event) { + return ((AlterTableColumnsEvent) event) + .getEvents().stream() + .map(AlterTableColumnEvent::getEventType) + .collect(Collectors.toList()); + } + + @Test + void noConfigIsAllowAll() { + SchemaChangeEventFilter f = filter(Collections.emptyList(), Collections.emptyList()); + Assertions.assertTrue(f.isNoOp()); + AlterTableColumnsEvent event = + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + new AlterTableDropColumnEvent(TABLE, "b")); + Assertions.assertSame(event, f.filter(event)); + } + + @Test + void includeOnlyKeepsOnlyListedLeafTypes() { + SchemaChangeEventFilter f = filter(Arrays.asList("add.column"), Collections.emptyList()); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + new AlterTableDropColumnEvent(TABLE, "b"), + AlterTableModifyColumnEvent.modify(TABLE, column("c")))); + Assertions.assertEquals( + Collections.singletonList(EventType.SCHEMA_CHANGE_ADD_COLUMN), subTypes(result)); + } + + @Test + void excludeOnlyDropsListedLeafTypes() { + SchemaChangeEventFilter f = filter(Collections.emptyList(), Arrays.asList("drop.column")); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + new AlterTableDropColumnEvent(TABLE, "b"))); + Assertions.assertEquals( + Collections.singletonList(EventType.SCHEMA_CHANGE_ADD_COLUMN), subTypes(result)); + } + + @Test + void excludeWinsOverIncludeForSameType() { + SchemaChangeEventFilter f = + filter(Arrays.asList("add.column", "drop.column"), Arrays.asList("add.column")); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + new AlterTableDropColumnEvent(TABLE, "b"))); + Assertions.assertEquals( + Collections.singletonList(EventType.SCHEMA_CHANGE_DROP_COLUMN), subTypes(result)); + } + + @Test + void wholeEventDroppedWhenNoSubEventSurvives() { + SchemaChangeEventFilter f = + filter(Collections.emptyList(), Arrays.asList("add.column", "drop.column")); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + new AlterTableDropColumnEvent(TABLE, "b"))); + Assertions.assertNull(result); + } + + @Test + void changeColumnIsFilteredIndependentlyOfModifyColumn() { + // exclude modify.column must NOT drop a change.column (rename) sub-event + SchemaChangeEventFilter f = filter(Collections.emptyList(), Arrays.asList("modify.column")); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableModifyColumnEvent.modify(TABLE, column("c")), + AlterTableChangeColumnEvent.change(TABLE, "old", column("new")))); + Assertions.assertEquals( + Collections.singletonList(EventType.SCHEMA_CHANGE_CHANGE_COLUMN), subTypes(result)); + } + + @Test + void updateColumnsIncludeIsGroupAliasForAllColumnChanges() { + SchemaChangeEventFilter f = + filter(Arrays.asList("update.columns"), Collections.emptyList()); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + new AlterTableDropColumnEvent(TABLE, "b"), + AlterTableModifyColumnEvent.modify(TABLE, column("c")))); + Assertions.assertEquals(3, ((AlterTableColumnsEvent) result).getEvents().size()); + } + + @Test + void updateColumnsExcludeSuppressesAllColumnChanges() { + SchemaChangeEventFilter f = + filter(Collections.emptyList(), Arrays.asList("update.columns")); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + AlterTableModifyColumnEvent.modify(TABLE, column("c")))); + Assertions.assertNull(result); + } + + @Test + void renameTableIsNotAnExposedCanonicalName() { + // rename.table is intentionally not exposed: CDC has no end-to-end handling for table + // renames, so it must be rejected as an unknown name rather than advertised as filterable. + IllegalArgumentException ex = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> SchemaChangeEventType.fromCanonicalName("rename.table")); + Assertions.assertTrue(ex.getMessage().contains("rename.table")); + } + + @Test + void strayTableLevelEventIsHandledDefensively() { + // No canonical name maps to a table-level type, so such an event is not produced by CDC + // today. The filter still handles it defensively: an active column-level include list + // drops it, and a no-op filter passes it through unchanged. + AlterTableNameEvent rename = + new AlterTableNameEvent(TABLE, TableIdentifier.of("", "db", "tbl2")); + Assertions.assertNull( + filter(Arrays.asList("add.column"), Collections.emptyList()).filter(rename)); + Assertions.assertSame( + rename, filter(Collections.emptyList(), Collections.emptyList()).filter(rename)); + } + + @Test + void unknownNameFailsFast() { + IllegalArgumentException ex = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> filter(Arrays.asList("create.table"), Collections.emptyList())); + Assertions.assertTrue(ex.getMessage().contains("create.table")); + Assertions.assertTrue(ex.getMessage().contains("add.column")); + } + + @Test + void validateOptionsAcceptsValidNames() { + Map map = new HashMap<>(); + map.put("schema-changes.include", Arrays.asList("add.column")); + map.put("schema-changes.exclude", Arrays.asList("drop.column")); + Assertions.assertDoesNotThrow( + () -> SchemaChangeEventFilter.validateOptions(ReadonlyConfig.fromMap(map))); + } + + @Test + void validateOptionsAcceptsEmptyConfig() { + Assertions.assertDoesNotThrow( + () -> + SchemaChangeEventFilter.validateOptions( + ReadonlyConfig.fromMap(new HashMap<>()))); + } + + @Test + void validateOptionsFailsFastOnUnknownIncludeName() { + Map map = new HashMap<>(); + map.put("schema-changes.include", Arrays.asList("rename.tabble")); + IllegalArgumentException ex = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> SchemaChangeEventFilter.validateOptions(ReadonlyConfig.fromMap(map))); + Assertions.assertTrue(ex.getMessage().contains("schema-changes.include")); + Assertions.assertTrue(ex.getMessage().contains("rename.tabble")); + Assertions.assertTrue(ex.getMessage().contains("add.column")); + } + + @Test + void validateOptionsFailsFastOnUnknownExcludeName() { + Map map = new HashMap<>(); + map.put("schema-changes.exclude", Arrays.asList("drop.colum")); + IllegalArgumentException ex = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> SchemaChangeEventFilter.validateOptions(ReadonlyConfig.fromMap(map))); + Assertions.assertTrue(ex.getMessage().contains("schema-changes.exclude")); + Assertions.assertTrue(ex.getMessage().contains("drop.colum")); + } + + @Test + void namesAreNormalizedAndDeduplicated() { + SchemaChangeEventFilter f = + filter(Arrays.asList(" ADD.COLUMN ", "add.column"), Collections.emptyList()); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + new AlterTableDropColumnEvent(TABLE, "b"))); + Assertions.assertEquals( + Collections.singletonList(EventType.SCHEMA_CHANGE_ADD_COLUMN), subTypes(result)); + } + + @Test + void canonicalNameMappingIsExhaustiveAndStable() { + Assertions.assertEquals( + EventType.SCHEMA_CHANGE_ADD_COLUMN, + SchemaChangeEventType.fromCanonicalName("add.column")); + Assertions.assertEquals( + EventType.SCHEMA_CHANGE_DROP_COLUMN, + SchemaChangeEventType.fromCanonicalName("drop.column")); + Assertions.assertEquals( + EventType.SCHEMA_CHANGE_MODIFY_COLUMN, + SchemaChangeEventType.fromCanonicalName("modify.column")); + Assertions.assertEquals( + EventType.SCHEMA_CHANGE_CHANGE_COLUMN, + SchemaChangeEventType.fromCanonicalName("change.column")); + Assertions.assertEquals( + EventType.SCHEMA_CHANGE_UPDATE_COLUMNS, + SchemaChangeEventType.fromCanonicalName("update.columns")); + Assertions.assertEquals(5, SchemaChangeEventType.canonicalNames().size()); + } + + /** + * Per-type coverage: for each column-level canonical name, {@code include=[name]} must keep + * exactly that sub-event out of a composite carrying all four column-level changes. + */ + @Test + void includeEachColumnLevelTypeKeepsOnlyThatType() { + Map cases = new HashMap<>(); + cases.put("add.column", EventType.SCHEMA_CHANGE_ADD_COLUMN); + cases.put("drop.column", EventType.SCHEMA_CHANGE_DROP_COLUMN); + cases.put("modify.column", EventType.SCHEMA_CHANGE_MODIFY_COLUMN); + cases.put("change.column", EventType.SCHEMA_CHANGE_CHANGE_COLUMN); + + for (Map.Entry c : cases.entrySet()) { + SchemaChangeEventFilter f = + filter(Collections.singletonList(c.getKey()), Collections.emptyList()); + SchemaChangeEvent result = + f.filter( + composite( + AlterTableAddColumnEvent.add(TABLE, column("a")), + new AlterTableDropColumnEvent(TABLE, "b"), + AlterTableModifyColumnEvent.modify(TABLE, column("c")), + AlterTableChangeColumnEvent.change(TABLE, "old", column("d")))); + Assertions.assertEquals( + Collections.singletonList(c.getValue()), + subTypes(result), + "include=[" + c.getKey() + "] should keep only " + c.getValue()); + } + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSource.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSource.java index 4a7cb09c9bf1..abb39f5897f8 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSource.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSource.java @@ -31,6 +31,7 @@ import org.apache.seatunnel.connectors.cdc.base.option.JdbcSourceOptions; import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; import org.apache.seatunnel.connectors.cdc.base.option.StopMode; +import org.apache.seatunnel.connectors.cdc.base.schema.SchemaChangeEventFilter; import org.apache.seatunnel.connectors.cdc.base.source.IncrementalSource; import org.apache.seatunnel.connectors.cdc.base.source.offset.OffsetFactory; import org.apache.seatunnel.connectors.cdc.debezium.ConnectTableChangeSerializer; @@ -129,6 +130,7 @@ public DebeziumDeserializationSchema createDebeziumDeserializationSchema( .setTableIdTableChangeMap(tableIdTableChangeMap) .setSchemaChangeResolver( new MySqlSchemaChangeResolver(createSourceConfigFactory(config))) + .setSchemaChangeEventFilter(SchemaChangeEventFilter.fromConfig(config)) .build(); } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java index 667f7b065433..04858dcb2922 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java @@ -78,6 +78,8 @@ public OptionRule optionRule() { MySqlIncrementalSourceOptions.SPLIT_ALLOW_SAMPLING, MySqlIncrementalSourceOptions.TABLE_NAMES_CONFIG, MySqlIncrementalSourceOptions.SCHEMA_CHANGES_ENABLED, + MySqlIncrementalSourceOptions.SCHEMA_CHANGES_INCLUDE, + MySqlIncrementalSourceOptions.SCHEMA_CHANGES_EXCLUDE, MySqlIncrementalSourceOptions.INT_TYPE_NARROWING) .optional( MySqlIncrementalSourceOptions.STARTUP_MODE, diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSource.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSource.java index fb28bda574ba..c2c9e860fa32 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSource.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSource.java @@ -31,6 +31,7 @@ import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; import org.apache.seatunnel.connectors.cdc.base.option.StopMode; +import org.apache.seatunnel.connectors.cdc.base.schema.SchemaChangeEventFilter; import org.apache.seatunnel.connectors.cdc.base.source.IncrementalSource; import org.apache.seatunnel.connectors.cdc.base.source.offset.OffsetFactory; import org.apache.seatunnel.connectors.cdc.debezium.ConnectTableChangeSerializer; @@ -112,6 +113,7 @@ public DebeziumDeserializationSchema createDebeziumDeserializationSchema( .setServerTimeZone(ZoneId.of(zoneId)) .setSchemaChangeResolver( new OracleSchemaChangeResolver(createSourceConfigFactory(config))) + .setSchemaChangeEventFilter(SchemaChangeEventFilter.fromConfig(config)) .setTableIdTableChangeMap(tableIdStructMap) .build(); } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSourceFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSourceFactory.java index 7c0371478cac..93e2e7f5b4b5 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSourceFactory.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSourceFactory.java @@ -78,7 +78,9 @@ public OptionRule optionRule() { OracleIncrementalSourceOptions.INVERSE_SAMPLING_RATE, OracleIncrementalSourceOptions.SPLIT_ALLOW_SAMPLING, OracleIncrementalSourceOptions.TABLE_NAMES_CONFIG, - OracleIncrementalSourceOptions.SCHEMA_CHANGES_ENABLED) + OracleIncrementalSourceOptions.SCHEMA_CHANGES_ENABLED, + OracleIncrementalSourceOptions.SCHEMA_CHANGES_INCLUDE, + OracleIncrementalSourceOptions.SCHEMA_CHANGES_EXCLUDE) .optional( OracleIncrementalSourceOptions.STARTUP_MODE, OracleIncrementalSourceOptions.STOP_MODE) diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSource.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSource.java index c80340d99b3c..2945d26777f2 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSource.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSource.java @@ -31,6 +31,7 @@ import org.apache.seatunnel.connectors.cdc.base.option.JdbcSourceOptions; import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; import org.apache.seatunnel.connectors.cdc.base.option.StopMode; +import org.apache.seatunnel.connectors.cdc.base.schema.SchemaChangeEventFilter; import org.apache.seatunnel.connectors.cdc.base.source.IncrementalSource; import org.apache.seatunnel.connectors.cdc.base.source.offset.OffsetFactory; import org.apache.seatunnel.connectors.cdc.debezium.ConnectTableChangeSerializer; @@ -119,6 +120,7 @@ public DebeziumDeserializationSchema createDebeziumDeserializationSchema( .setTableIdTableChangeMap(tableIdTableChangeMap) .setSchemaChangeResolver( schemaChangesEnabled ? new SqlServerSchemaChangeResolver() : null) + .setSchemaChangeEventFilter(SchemaChangeEventFilter.fromConfig(config)) .build(); } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSourceFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSourceFactory.java index 9436bc885f4e..02e40917b7e2 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSourceFactory.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSourceFactory.java @@ -72,7 +72,9 @@ public OptionRule optionRule() { SqlServerIncrementalSourceOptions.INVERSE_SAMPLING_RATE, SqlServerIncrementalSourceOptions.SPLIT_ALLOW_SAMPLING, SqlServerIncrementalSourceOptions.TABLE_NAMES_CONFIG, - SqlServerIncrementalSourceOptions.SCHEMA_CHANGES_ENABLED) + SqlServerIncrementalSourceOptions.SCHEMA_CHANGES_ENABLED, + SqlServerIncrementalSourceOptions.SCHEMA_CHANGES_INCLUDE, + SqlServerIncrementalSourceOptions.SCHEMA_CHANGES_EXCLUDE) .optional( SqlServerIncrementalSourceOptions.STARTUP_MODE, SqlServerIncrementalSourceOptions.STOP_MODE) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithSchemaChangeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithSchemaChangeIT.java index b865fd88471f..d2150a0103fb 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithSchemaChangeIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithSchemaChangeIT.java @@ -51,6 +51,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -70,6 +71,9 @@ public class MysqlCDCWithSchemaChangeIT extends TestSuiteBase implements TestRes private static final String SINK_TABLE = "mysql_cdc_e2e_sink_table_with_schema_change"; private static final String SINK_TABLE2 = "mysql_cdc_e2e_sink_table_with_schema_change_exactly_once"; + private static final String SINK_TABLE_FILTER = "mysql_cdc_e2e_sink_table_schema_change_filter"; + private static final String STABLE_QUERY = + "select id,name,description,weight from %s.%s order by id"; private static final String MYSQL_HOST = "mysql_cdc_e2e"; private static final String MYSQL_USER_NAME = "mysqluser"; private static final String MYSQL_USER_PASSWORD = "mysqlpw"; @@ -198,6 +202,123 @@ public void testMysqlCdcWithSchemaEvolutionCaseExactlyOnce(TestContainer contain assertSchemaEvolution(MYSQL_DATABASE, SOURCE_TABLE, SINK_TABLE2); } + /** + * Regression for issue #11044. With {@code schema-changes.exclude = ["drop.column"]}: a dropped + * column must NOT propagate to the sink (it stays in the sink schema), and the data changes + * happening at the same time must still reach the sink. + * + *

The dropped column here is intentionally NULLABLE. #11044 is event-type filtering + * only and, per its non-goals, does not define a schema-change data-handling policy, so the + * source simply writes {@code null} for a retained-but-no-longer-supplied column — valid for a + * nullable column. Excluding {@code drop.column} for a NOT NULL column is a known limitation + * that fails at the sink and is deferred to a future behavior-policy feature; see MySQL-CDC.md. + * Using a nullable column is exactly why this test needs its own {@code *_filter} DDL templates + * instead of the shared {@code add_columns}/{@code drop_columns} (which add/drop NOT NULL + * columns). + */ + @Order(3) + @TestTemplate + public void testMysqlCdcSchemaChangeEventTypeFilter(TestContainer container) { + shopDatabase.setTemplateName("shop").createAndInitialize(); + CompletableFuture.runAsync( + () -> { + try { + container.executeJob("/mysqlcdc_to_mysql_with_schema_change_filter.conf"); + } catch (Exception e) { + log.error("Commit task exception :" + e.getMessage()); + throw new RuntimeException(e); + } + }); + + // initial snapshot synced + await().atMost(30000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> + Assertions.assertIterableEquals( + query( + String.format( + STABLE_QUERY, + MYSQL_DATABASE, + SOURCE_TABLE)), + query( + String.format( + STABLE_QUERY, + MYSQL_DATABASE, + SINK_TABLE_FILTER)))); + + // add.column is NOT excluded + shopDatabase.setTemplateName("add_columns_filter").createAndInitialize(); + await().atMost(30000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> + Assertions.assertTrue( + columnExists( + MYSQL_DATABASE, SINK_TABLE_FILTER, "add_column1"), + "add.column should propagate to the sink")); + + // drop.column IS excluded; this template also inserts/updates/deletes rows at the same time + shopDatabase.setTemplateName("drop_columns_filter").createAndInitialize(); + + // regression: the concurrent data changes must still reach the sink (job did not crash) + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> + Assertions.assertIterableEquals( + query( + String.format( + STABLE_QUERY, + MYSQL_DATABASE, + SOURCE_TABLE)), + query( + String.format( + STABLE_QUERY, + MYSQL_DATABASE, + SINK_TABLE_FILTER)))); + + // Row-level hardening: drop_columns_filter.sql performs INSERT + UPDATE + DELETE alongside + // the filtered drop.column DDL. Every one of those data changes must be reflected in the + // sink (the earlier iterable-equals already converged, so these reads are stable). + List> sourceRows = + query(String.format(STABLE_QUERY, MYSQL_DATABASE, SOURCE_TABLE)); + List> sinkRows = + query(String.format(STABLE_QUERY, MYSQL_DATABASE, SINK_TABLE_FILTER)); + Assertions.assertEquals( + sourceRows.size(), + sinkRows.size(), + "sink row count must match source after the concurrent INSERT/UPDATE/DELETE"); + // DELETE propagated: drop_columns_filter.sql runs `delete from products where id = 102`. + Assertions.assertTrue( + sinkRows.stream().noneMatch(row -> ((Number) row.get(0)).intValue() == 102), + "rows deleted at the source must also be deleted in the sink"); + // INSERT propagated: id 110 is inserted by drop_columns_filter.sql. + Assertions.assertTrue( + sinkRows.stream().anyMatch(row -> ((Number) row.get(0)).intValue() == 110), + "rows inserted at the source must appear in the sink"); + // UPDATE propagated: `set name='dailai' where id = 101`. + assertSinkNameEquals(sinkRows, 101, "dailai"); + + // the excluded drop.column must NOT have been applied to the sink schema + Assertions.assertTrue( + columnExists(MYSQL_DATABASE, SINK_TABLE_FILTER, "add_column1"), + "drop.column was excluded, so the sink must keep the column the source dropped"); + } + + /** Asserts the sink row with the given id exists and its {@code name} matches expectedName. */ + private void assertSinkNameEquals(List> sinkRows, int id, Object expectedName) { + Optional> row = + sinkRows.stream().filter(r -> ((Number) r.get(0)).intValue() == id).findFirst(); + Assertions.assertTrue(row.isPresent(), "expected sink row with id=" + id); + Assertions.assertEquals( + expectedName, + row.get().get(1), + "updated value must propagate to the sink for id=" + id); + } + + private boolean columnExists(String database, String table, String column) { + return query(String.format(DESC, database, table)).stream() + .anyMatch(row -> column.equalsIgnoreCase(String.valueOf(row.get(0)))); + } + private void assertSchemaEvolution(String database, String sourceTable, String sinkTable) { await().atMost(30000, TimeUnit.MILLISECONDS) .untilAsserted( diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/add_columns_filter.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/add_columns_filter.sql new file mode 100644 index 000000000000..814bcbf7afac --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/add_columns_filter.sql @@ -0,0 +1,24 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: shop +-- ---------------------------------------------------------------------------------------------------------------- +CREATE DATABASE IF NOT EXISTS `shop`; +use shop; + +alter table products add column add_column1 varchar(64) null; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/drop_columns_filter.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/drop_columns_filter.sql new file mode 100644 index 000000000000..1376bedd1fbb --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/drop_columns_filter.sql @@ -0,0 +1,29 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: shop +-- ---------------------------------------------------------------------------------------------------------------- +CREATE DATABASE IF NOT EXISTS `shop`; +use shop; + +alter table products drop column add_column1; +insert into products +values (110,"spare tire","24 inch spare tire",22.2), + (111,"new battery","12V battery",8.1); +update products set name = 'dailai' where id = 101; +delete from products where id = 102; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/shop.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/shop.sql index 9887b4e68772..bddc222fb5ca 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/shop.sql +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/shop.sql @@ -46,6 +46,14 @@ CREATE TABLE if not exists mysql_cdc_e2e_sink_table_with_schema_change_exactly_o weight FLOAT ); +drop table if exists mysql_cdc_e2e_sink_table_schema_change_filter; +CREATE TABLE if not exists mysql_cdc_e2e_sink_table_schema_change_filter ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL DEFAULT 'SeaTunnel', + description VARCHAR(512), + weight FLOAT +); + ALTER TABLE products AUTO_INCREMENT = 101; INSERT INTO products diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_schema_change_filter.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_schema_change_filter.conf new file mode 100644 index 000000000000..828ef81cc22e --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_schema_change_filter.conf @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Schema change event-type filtering (issue #11044): +# add.column propagates, drop.column is excluded. Regression guard: a filtered +# drop.column must NOT desync the produced row shape from the sink schema, so +# subsequent data changes keep flowing instead of crashing the sink. + +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + MySQL-CDC { + server-id = 5680-5685 + username = "st_user_source" + password = "mysqlpw" + table-names = ["shop.products"] + url = "jdbc:mysql://mysql_cdc_e2e:3306/shop" + + schema-changes.enabled = true + schema-changes.exclude = ["drop.column"] + } +} + +sink { + jdbc { + url = "jdbc:mysql://mysql_cdc_e2e:3306/shop" + driver = "com.mysql.cj.jdbc.Driver" + user = "st_user_sink" + password = "mysqlpw" + generate_sink_sql = true + database = shop + table = mysql_cdc_e2e_sink_table_schema_change_filter + primary_keys = ["id"] + } +} From b38601a074494c53006ca14cd7b353873e4052a9 Mon Sep 17 00:00:00 2001 From: Jast Date: Thu, 18 Jun 2026 08:43:08 +0800 Subject: [PATCH 028/375] [Fix][Connector-V2] Fix Parquet INT96 mixed-case field matching (#11067) --- .../sink/writer/ParquetWriteStrategy.java | 32 +++++++++----- .../file/writer/ParquetWriteStrategyTest.java | 44 +++++++++++++++++++ 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java index d8ed92d5fbb4..8537fc688bd8 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java @@ -65,9 +65,9 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Date; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.TimeUnit; @@ -99,12 +99,15 @@ public ParquetWriteStrategy(FileSinkConfig fileSinkConfig) { public void init(HadoopConf conf, String jobId, String uuidPrefix, int subTaskIndex) { super.init(conf, jobId, uuidPrefix, subTaskIndex); Configuration configuration = getConfiguration(hadoopConf); - writePathsAsInt96 = new HashSet<>(fileSinkConfig.getParquetAvroWriteFixedAsInt96()); + writePathsAsInt96 = + fileSinkConfig.getParquetAvroWriteFixedAsInt96().stream() + .map(this::normalizeFieldName) + .collect(Collectors.toSet()); if (fileSinkConfig.getParquetWriteTimestampAsInt96()) { List timestampFields = new ArrayList<>(); for (int i = 0; i < seaTunnelRowType.getTotalFields(); i++) { if (SqlType.TIMESTAMP.equals(seaTunnelRowType.getFieldType(i).getSqlType())) { - timestampFields.add(seaTunnelRowType.getFieldName(i)); + timestampFields.add(normalizeFieldName(seaTunnelRowType.getFieldName(i))); } } writePathsAsInt96.addAll(timestampFields); @@ -244,7 +247,7 @@ private Object resolveObject(String name, Object data, SeaTunnelDataType seaT case DATE: return data; case TIMESTAMP: - if (writePathsAsInt96.contains(name)) { + if (writePathsAsInt96.contains(normalizeFieldName(name))) { LocalDateTime localDateTime = (LocalDateTime) data; Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); calendar.setTime( @@ -264,7 +267,7 @@ private Object resolveObject(String name, Object data, SeaTunnelDataType seaT calendar.get(Calendar.MILLISECOND)); NanoTime nanoTime = new NanoTime(julianDays, timeOfDayNanos); return new GenericData.Fixed( - schema.getField(name.toLowerCase()).schema(), + schema.getField(normalizeFieldName(name)).schema(), nanoTime.toBinary().getBytes()); } return ((LocalDateTime) data) @@ -272,9 +275,9 @@ private Object resolveObject(String name, Object data, SeaTunnelDataType seaT .toInstant() .toEpochMilli(); case BYTES: - if (writePathsAsInt96.contains(name)) { + if (writePathsAsInt96.contains(normalizeFieldName(name))) { return new GenericData.Fixed( - schema.getField(name.toLowerCase()).schema(), (byte[]) data); + schema.getField(normalizeFieldName(name)).schema(), (byte[]) data); } return ByteBuffer.wrap((byte[]) data); case ROW: @@ -365,7 +368,7 @@ public Type seaTunnelDataType2ParquetDataType( PrimitiveType.PrimitiveTypeName.INT64, Type.Repetition.OPTIONAL) .named(fieldName); case TIMESTAMP: - if (writePathsAsInt96.contains(fieldName)) { + if (writePathsAsInt96.contains(normalizeFieldName(fieldName))) { return Types.primitive( PrimitiveType.PrimitiveTypeName.INT96, Type.Repetition.OPTIONAL) .named(fieldName); @@ -392,7 +395,7 @@ public Type seaTunnelDataType2ParquetDataType( .scale(scale) .named(fieldName); case BYTES: - if (writePathsAsInt96.contains(fieldName)) { + if (writePathsAsInt96.contains(normalizeFieldName(fieldName))) { return Types.primitive( PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, Type.Repetition.OPTIONAL) @@ -430,10 +433,13 @@ protected void onSchemaChanged() { // are included. Without this, new TIMESTAMP columns get INT64 encoding instead of INT96 // when parquetWriteTimestampAsInt96=true — silent data corruption. if (fileSinkConfig.getParquetWriteTimestampAsInt96()) { - writePathsAsInt96 = new HashSet<>(fileSinkConfig.getParquetAvroWriteFixedAsInt96()); + writePathsAsInt96 = + fileSinkConfig.getParquetAvroWriteFixedAsInt96().stream() + .map(this::normalizeFieldName) + .collect(Collectors.toSet()); for (int i = 0; i < seaTunnelRowType.getTotalFields(); i++) { if (SqlType.TIMESTAMP.equals(seaTunnelRowType.getFieldType(i).getSqlType())) { - writePathsAsInt96.add(seaTunnelRowType.getFieldName(i)); + writePathsAsInt96.add(normalizeFieldName(seaTunnelRowType.getFieldName(i))); } } } @@ -457,4 +463,8 @@ private Schema buildAvroSchemaWithRowType( Types.buildMessage().addFields(types.toArray(new Type[0])).named("SeaTunnelRecord"); return schemaConverter.convert(seaTunnelRow); } + + private String normalizeFieldName(String fieldName) { + return fieldName.toLowerCase(Locale.ROOT); + } } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java index f24def6288a4..eea84414c910 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java @@ -149,4 +149,48 @@ public Object getCheckpointLock() { Assertions.assertEquals(1, readRows.size()); readStrategy.close(); } + + @DisabledOnOs(OS.WINDOWS) + @Test + public void testParquetWriteInt96WithMixedCaseTimestampColumn() throws Exception { + Map writeConfig = new HashMap<>(); + writeConfig.put("tmp_path", TMP_PATH + "-mixed-case"); + writeConfig.put("path", "file:///tmp/seatunnel/parquet/int96-mixed-case"); + writeConfig.put("file_format_type", FileFormat.PARQUET.name()); + writeConfig.put("parquet_avro_write_timestamp_as_int96", "true"); + + SeaTunnelRowType writeRowType = + new SeaTunnelRowType( + new String[] {"createTime"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TIME_TYPE}); + FileSinkConfig writeSinkConfig = + new FileSinkConfig(ReadonlyConfig.fromMap(writeConfig), writeRowType); + ParquetWriteStrategy writeStrategy = new ParquetWriteStrategy(writeSinkConfig); + LocalFileSystemConf.LocalConf hadoopConf = + new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT); + writeStrategy.setCatalogTable( + CatalogTableUtil.getCatalogTable("test", null, null, "test", writeRowType)); + writeStrategy.init(hadoopConf, "test1", "test1", 0); + writeStrategy.beginTransaction(1L); + writeStrategy.write(new SeaTunnelRow(new Object[] {LocalDateTime.now()})); + writeStrategy.finishAndCloseFile(); + writeStrategy.close(); + + ParquetReadStrategy readStrategy = new ParquetReadStrategy(); + readStrategy.init(hadoopConf); + List readFiles = readStrategy.getFileNamesByPath(TMP_PATH + "-mixed-case"); + Assertions.assertEquals(1, readFiles.size()); + try (ParquetFileReader reader = + ParquetFileReader.open( + HadoopInputFile.fromPath( + new org.apache.hadoop.fs.Path(readFiles.get(0)), + new Configuration()))) { + FileMetaData metadata = reader.getFooter().getFileMetaData(); + Type createTimeType = metadata.getSchema().getType("createtime"); + Assertions.assertEquals( + PrimitiveType.PrimitiveTypeName.INT96, + createTimeType.asPrimitiveType().getPrimitiveTypeName()); + } + readStrategy.close(); + } } From 346fdfe3d58a47b18bf32c68c1e2a91c321b8d4a Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Fri, 19 Jun 2026 18:16:12 +0800 Subject: [PATCH 029/375] [Improve][Connector-V2] Migrate jdbc validation to declarative OptionRule (#11106) --- .github/workflows/backend.yml | 12 +- .../jdbc/catalog/dm/DamengCatalogFactory.java | 6 - .../catalog/highgo/HighGoCatalogFactory.java | 7 - .../jdbc/catalog/iris/IrisCatalogFactory.java | 6 - .../kingbase/KingbaseCatalogFactory.java | 6 - .../catalog/mysql/MySqlCatalogFactory.java | 6 - .../oceanbase/OceanBaseCatalogFactory.java | 23 +- .../opengauss/OpenGaussCatalogFactory.java | 7 - .../catalog/oracle/OracleCatalogFactory.java | 7 - .../catalog/psql/PostgresCatalogFactory.java | 7 - .../redshift/RedshiftCatalogFactory.java | 13 - .../saphana/SapHanaCatalogFactory.java | 7 - .../jdbc/catalog/tidb/TiDBCatalogFactory.java | 7 - .../jdbc/catalog/xugu/XuguCatalogFactory.java | 7 - .../jdbc/config/JdbcCommonOptions.java | 55 +++- .../jdbc/config/JdbcSourceConfig.java | 10 +- .../jdbc/config/JdbcSourceTableConfig.java | 5 - .../internal/JdbcOutputFormatBuilder.java | 20 -- .../seatunnel/jdbc/sink/JdbcSinkFactory.java | 91 ++++++- .../jdbc/source/JdbcSourceFactory.java | 51 +++- .../jdbc/catalog/JdbcCatalogFactoryTest.java | 113 ++++++++ .../internal/JdbcOutputFormatBuilderTest.java | 47 ---- .../jdbc/sink/JdbcSinkFactoryTest.java | 245 ++++++++++++++++++ .../jdbc/source/JdbcSourceFactoryTest.java | 185 +++++++++++++ 24 files changed, 739 insertions(+), 204 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/JdbcCatalogFactoryTest.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactoryTest.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactoryTest.java diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 4f34cc94cb79..20565bccea8f 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -460,7 +460,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-2: needs: [ changes, sanity-check ] @@ -522,7 +522,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-4: needs: [ changes, sanity-check ] @@ -583,7 +583,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-6: needs: [ changes, sanity-check ] if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' @@ -613,7 +613,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-7: needs: [ changes, sanity-check ] if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' @@ -643,7 +643,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-8: needs: [ changes, sanity-check ] @@ -674,7 +674,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m engine-v2-it: needs: [ changes, sanity-check ] diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalogFactory.java index 3e4b8058dd47..ead206a230ff 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalogFactory.java @@ -17,9 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.dm; -import org.apache.seatunnel.shade.com.google.common.base.Preconditions; -import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; - import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.catalog.Catalog; @@ -42,9 +39,6 @@ public String factoryIdentifier() { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); - Preconditions.checkArgument( - StringUtils.isNoneBlank(urlWithDatabase), - "Miss config ! Please check your config."); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); return new DamengCatalog( catalogName, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/highgo/HighGoCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/highgo/HighGoCatalogFactory.java index 59c5222056a7..e61254107588 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/highgo/HighGoCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/highgo/HighGoCatalogFactory.java @@ -18,25 +18,18 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.common.utils.JdbcUrlUtil; import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier; -import java.util.Optional; - public class HighGoCatalogFactory implements CatalogFactory { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } return new HighGoCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalogFactory.java index 820c3a01a1ed..33ed340f2790 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalogFactory.java @@ -17,9 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.iris; -import org.apache.seatunnel.shade.com.google.common.base.Preconditions; -import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; - import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.catalog.Catalog; @@ -42,9 +39,6 @@ public String factoryIdentifier() { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); - Preconditions.checkArgument( - StringUtils.isNoneBlank(urlWithDatabase), - "Miss config ! Please check your config."); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); return new IrisCatalog( catalogName, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/kingbase/KingbaseCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/kingbase/KingbaseCatalogFactory.java index 6003a9a3f81a..b535ce5f50f3 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/kingbase/KingbaseCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/kingbase/KingbaseCatalogFactory.java @@ -17,9 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.kingbase; -import org.apache.seatunnel.shade.com.google.common.base.Preconditions; -import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; - import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.catalog.Catalog; @@ -42,9 +39,6 @@ public String factoryIdentifier() { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); - Preconditions.checkArgument( - StringUtils.isNoneBlank(urlWithDatabase), - "Miss config ! Please check your config."); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); return new KingbaseCatalog( catalogName, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalogFactory.java index d31c7933b099..052f8763ab93 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalogFactory.java @@ -17,9 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.mysql; -import org.apache.seatunnel.shade.com.google.common.base.Preconditions; -import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; - import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.catalog.Catalog; @@ -42,9 +39,6 @@ public String factoryIdentifier() { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); - Preconditions.checkArgument( - StringUtils.isNoneBlank(urlWithDatabase), - "Miss config ! Please check your config."); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); return new MySqlCatalog( catalogName, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oceanbase/OceanBaseCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oceanbase/OceanBaseCatalogFactory.java index 4711a0f18810..ba48bf040b02 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oceanbase/OceanBaseCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oceanbase/OceanBaseCatalogFactory.java @@ -17,12 +17,8 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oceanbase; -import org.apache.seatunnel.shade.com.google.common.base.Preconditions; -import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; - import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -30,18 +26,11 @@ import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(Factory.class) public class OceanBaseCatalogFactory implements CatalogFactory { - private static final Logger log = LoggerFactory.getLogger(OceanBaseCatalogFactory.class); - @Override public String factoryIdentifier() { return DatabaseIdentifier.OCEANBASE; @@ -50,19 +39,9 @@ public String factoryIdentifier() { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); - Preconditions.checkArgument( - StringUtils.isNoneBlank(urlWithDatabase), - "Miss config ! Please check your config."); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } String compatibleMode = options.get(JdbcCommonOptions.COMPATIBLE_MODE); - Preconditions.checkArgument( - StringUtils.isNoneBlank(compatibleMode), - "Miss config ! Please check your config."); if ("oracle".equalsIgnoreCase(compatibleMode.trim())) { return new OceanBaseOracleCatalog( @@ -83,7 +62,7 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE + return JdbcCommonOptions.baseCatalogRule() .required(JdbcCommonOptions.COMPATIBLE_MODE) .build(); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/opengauss/OpenGaussCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/opengauss/OpenGaussCatalogFactory.java index 0b43cc002ffb..c80b163e740c 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/opengauss/OpenGaussCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/opengauss/OpenGaussCatalogFactory.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -29,8 +28,6 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(Factory.class) public class OpenGaussCatalogFactory implements CatalogFactory { @@ -43,10 +40,6 @@ public String factoryIdentifier() { public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } return new OpenGaussCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogFactory.java index 63e21ea5c9b5..782130bd9642 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogFactory.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -29,8 +28,6 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(Factory.class) public class OracleCatalogFactory implements CatalogFactory { @@ -43,10 +40,6 @@ public String factoryIdentifier() { public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); JdbcUrlUtil.UrlInfo urlInfo = OracleURLParser.parse(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } return new OracleCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogFactory.java index 6c7c061fbfad..7217432eb6d1 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogFactory.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -29,8 +28,6 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(Factory.class) public class PostgresCatalogFactory implements CatalogFactory { @@ -43,10 +40,6 @@ public String factoryIdentifier() { public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } return new PostgresCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalogFactory.java index 6f34355e8c49..7931ef382636 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalogFactory.java @@ -17,12 +17,8 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.redshift; -import org.apache.seatunnel.shade.com.google.common.base.Preconditions; -import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; - import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -32,8 +28,6 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(Factory.class) public class RedshiftCatalogFactory implements CatalogFactory { @@ -45,14 +39,7 @@ public String factoryIdentifier() { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); - Preconditions.checkArgument( - StringUtils.isNotBlank(urlWithDatabase), - "Miss config ! Please check your config."); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } return new RedshiftCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalogFactory.java index bd11d415fbca..ba7dd361b649 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalogFactory.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -29,8 +28,6 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(Factory.class) public class SapHanaCatalogFactory implements CatalogFactory { @@ -43,10 +40,6 @@ public String factoryIdentifier() { public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); JdbcUrlUtil.UrlInfo urlInfo = SapHanaURLParser.parse(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } return new SapHanaCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalogFactory.java index 76591107c93a..beb2945b8762 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalogFactory.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -29,8 +28,6 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(Factory.class) public class TiDBCatalogFactory implements CatalogFactory { @@ -43,10 +40,6 @@ public String factoryIdentifier() { public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } return new TiDBCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalogFactory.java index 28da20a47a83..63f35bfd358c 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalogFactory.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -30,8 +29,6 @@ import com.google.auto.service.AutoService; -import java.util.Optional; - @AutoService(Factory.class) public class XuguCatalogFactory implements CatalogFactory { @@ -44,10 +41,6 @@ public String factoryIdentifier() { public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); JdbcUrlUtil.UrlInfo urlInfo = OracleURLParser.parse(urlWithDatabase); - Optional defaultDatabase = urlInfo.getDefaultDatabase(); - if (!defaultDatabase.isPresent()) { - throw new OptionValidationException(JdbcCommonOptions.URL); - } return new XuguCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcCommonOptions.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcCommonOptions.java index f0ace0aa1481..2c636be5c4b6 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcCommonOptions.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcCommonOptions.java @@ -19,7 +19,11 @@ import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.common.utils.JdbcUrlUtil; import java.util.Map; @@ -158,9 +162,50 @@ public class JdbcCommonOptions { public static final Option REGION = Options.key("region").stringType().noDefaultValue().withDescription("region"); - public static final OptionRule.Builder BASE_CATALOG_RULE = - OptionRule.builder() - .required(URL) - .required(USERNAME, PASSWORD) - .optional(SCHEMA, DECIMAL_TYPE_NARROWING, HANDLE_BLOB_AS_STRING); + /** @deprecated Use {@link #baseCatalogRule()} instead to avoid shared mutable state. */ + @Deprecated public static final OptionRule.Builder BASE_CATALOG_RULE = baseCatalogRule(); + + /** + * Returns a fresh {@link OptionRule.Builder} with the base validation rules shared by all JDBC + * catalog factories (MySQL, PostgreSQL, Oracle, etc.). + * + *

These rules are evaluated at submission time via {@code + * ConfigValidator.validate(factory.optionRule())} in the {@code FactoryUtil} entry path. They + * enforce that the JDBC URL contains a database name, and that username/password are provided. + * + *

Individual catalog factories may append additional rules (e.g. OceanBase requires {@code + * compatible_mode}) before calling {@code .build()}. + */ + public static OptionRule.Builder baseCatalogRule() { + return OptionRule.builder() + .required(URL, Conditions.extension(URL, new UrlContainsDatabaseValidator())) + .required(USERNAME, PASSWORD) + .optional(SCHEMA, DECIMAL_TYPE_NARROWING, HANDLE_BLOB_AS_STRING); + } + + /** + * Submission-time validator that ensures the JDBC URL contains a database name. + * + *

This validator is attached to the {@code url} option via {@link + * Conditions#extension(Option, ConditionExtension)} and is evaluated by {@code ConfigValidator} + * before the catalog/source/sink factory creates its connector instance. + */ + public static class UrlContainsDatabaseValidator implements ConditionExtension { + @Override + public String description() { + return "JDBC URL must contain a database name"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String url) { + if (url == null || url.trim().isEmpty()) { + return false; + } + try { + return JdbcUrlUtil.getUrlInfo(url).getDefaultDatabase().isPresent(); + } catch (IllegalArgumentException e) { + return false; + } + } + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java index a890abf968c1..74f7b7b78f38 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceConfig.java @@ -88,15 +88,7 @@ public static JdbcSourceConfig of(ReadonlyConfig config) { config.get(MultiTableCommonOptions.MULTI_TABLE_FAILURE_POLICY)); config.getOptional(JdbcSourceOptions.WHERE_CONDITION) - .ifPresent( - whereConditionClause -> { - if (!whereConditionClause.toLowerCase().startsWith("where")) { - throw new IllegalArgumentException( - "The where condition clause must start with 'where'. value: " - + whereConditionClause); - } - builder.whereConditionClause(whereConditionClause); - }); + .ifPresent(builder::whereConditionClause); return builder.build(); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceTableConfig.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceTableConfig.java index 937e21558c5f..e08013561868 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceTableConfig.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSourceTableConfig.java @@ -72,11 +72,6 @@ public JdbcSourceTableConfig() {} public static List of(ReadonlyConfig connectorConfig) { List tableList; if (connectorConfig.getOptional(JdbcSourceOptions.TABLE_LIST).isPresent()) { - if (connectorConfig.getOptional(JdbcSourceOptions.QUERY).isPresent() - || connectorConfig.getOptional(JdbcSourceOptions.TABLE_PATH).isPresent()) { - throw new IllegalArgumentException( - "Please configure either `table_list` or `table_path`/`query`, not both"); - } tableList = connectorConfig.get(JdbcSourceOptions.TABLE_LIST); } else { JdbcSourceTableConfig tableProperty = diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatBuilder.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatBuilder.java index 84dcf0a6de75..c6e4514e7438 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatBuilder.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatBuilder.java @@ -380,30 +380,10 @@ private static void validateOracleInsertMode( throw new IllegalArgumentException( "oracle_insert_mode=APPEND_VALUES only supports Oracle JDBC sink."); } - if (jdbcSinkConfig.isUseCopyStatement()) { - throw new IllegalArgumentException( - "oracle_insert_mode=APPEND_VALUES does not support copy statement."); - } - if (StringUtils.isNotBlank(jdbcSinkConfig.getSimpleSql())) { - throw new IllegalArgumentException( - "oracle_insert_mode=APPEND_VALUES does not support custom query."); - } - if (jdbcSinkConfig.isExactlyOnce()) { - throw new IllegalArgumentException( - "oracle_insert_mode=APPEND_VALUES does not support exactly-once JDBC sink."); - } - if (!jdbcSinkConfig.getJdbcConnectionConfig().isAutoCommit()) { - throw new IllegalArgumentException( - "oracle_insert_mode=APPEND_VALUES requires auto_commit=true."); - } if (primaryKeys != null && !primaryKeys.isEmpty()) { throw new IllegalArgumentException( "oracle_insert_mode=APPEND_VALUES only supports insert-only writes without primary keys."); } - if (jdbcSinkConfig.isSupportUpsertByInsertOnly()) { - throw new IllegalArgumentException( - "oracle_insert_mode=APPEND_VALUES does not support insert-only upsert paths."); - } } private static boolean isOracleAppendValuesConfigured(JdbcSinkConfig jdbcSinkConfig) { diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java index 61bae380bf3b..bf035c2c0b72 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java @@ -20,7 +20,10 @@ import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.options.ConnectorCommonOptions; import org.apache.seatunnel.api.options.SinkConnectorCommonOptions; import org.apache.seatunnel.api.sink.DataSaveMode; @@ -213,6 +216,16 @@ public OptionRule optionRule() { JdbcSinkOptions.DRIVER, JdbcSinkOptions.SCHEMA_SAVE_MODE, JdbcSinkOptions.DATA_SAVE_MODE) + .optional( + JdbcSinkOptions.ORACLE_INSERT_MODE, + Conditions.extension( + JdbcSinkOptions.ORACLE_INSERT_MODE, + new OracleAppendValuesValidator())) + .optional( + JdbcSinkOptions.IS_EXACTLY_ONCE, + Conditions.extension( + JdbcSinkOptions.IS_EXACTLY_ONCE, + new ExactlyOnceMaxRetriesValidator())) .optional( JdbcSinkOptions.CREATE_INDEX, JdbcSinkOptions.USERNAME, @@ -220,14 +233,12 @@ public OptionRule optionRule() { JdbcSinkOptions.CONNECTION_CHECK_TIMEOUT_SEC, JdbcSinkOptions.BATCH_SIZE, JdbcSinkOptions.BATCH_INTERVAL_MS, - JdbcSinkOptions.IS_EXACTLY_ONCE, JdbcSinkOptions.GENERATE_SINK_SQL, JdbcSinkOptions.AUTO_COMMIT, JdbcSinkOptions.PRIMARY_KEYS, JdbcSinkOptions.IS_PRIMARY_KEY_UPDATED, JdbcSinkOptions.SUPPORT_UPSERT_BY_INSERT_ONLY, JdbcSinkOptions.USE_COPY_STATEMENT, - JdbcSinkOptions.ORACLE_INSERT_MODE, JdbcSinkOptions.COMPATIBLE_MODE, JdbcSinkOptions.ENABLE_UPSERT, JdbcSinkOptions.FIELD_IDE, @@ -250,4 +261,80 @@ public OptionRule optionRule() { JdbcSinkOptions.CUSTOM_SQL) .build(); } + + /** + * Submission-time validator for {@code oracle_insert_mode=APPEND_VALUES}. + * + *

Enforces config-level incompatibilities that can be detected from the user-supplied + * options alone: copy statement, exactly-once, auto_commit=false, custom query, and insert-only + * upsert. + * + *

Note: The {@code primary_keys} conflict is not checked here because + * primary keys may be derived from the upstream {@code CatalogTable} at factory time (inside + * {@link #createSink}), which happens after OptionRule validation. That case is guarded at + * runtime by {@code JdbcOutputFormatBuilder.validateOracleInsertMode}. + */ + static class OracleAppendValuesValidator + implements ConditionExtension { + @Override + public String description() { + return "oracle_insert_mode=APPEND_VALUES conflicts with certain options"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, JdbcSinkConfig.OracleInsertMode value) + throws OptionValidationException { + if (value != JdbcSinkConfig.OracleInsertMode.APPEND_VALUES) { + return true; + } + if (config.get(JdbcSinkOptions.USE_COPY_STATEMENT)) { + throw new OptionValidationException( + "oracle_insert_mode=APPEND_VALUES does not support copy statement."); + } + if (config.get(JdbcSinkOptions.IS_EXACTLY_ONCE)) { + throw new OptionValidationException( + "oracle_insert_mode=APPEND_VALUES does not support exactly-once."); + } + if (!config.get(JdbcSinkOptions.AUTO_COMMIT)) { + throw new OptionValidationException( + "oracle_insert_mode=APPEND_VALUES requires auto_commit=true."); + } + if (!config.get(JdbcSinkOptions.GENERATE_SINK_SQL)) { + throw new OptionValidationException( + "oracle_insert_mode=APPEND_VALUES does not support custom query."); + } + if (config.get(JdbcSinkOptions.SUPPORT_UPSERT_BY_INSERT_ONLY)) { + throw new OptionValidationException( + "oracle_insert_mode=APPEND_VALUES does not support insert-only upsert."); + } + return true; + } + } + + /** + * Submission-time validator for {@code is_exactly_once=true}. + * + *

JDBC XA sink does not support retries; {@code max_retries} must be 0 when exactly-once is + * enabled, otherwise duplicates may occur. + */ + static class ExactlyOnceMaxRetriesValidator implements ConditionExtension { + @Override + public String description() { + return "is_exactly_once=true requires max_retries=0"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Boolean value) + throws OptionValidationException { + if (Boolean.TRUE.equals(value)) { + int maxRetries = config.get(JdbcSinkOptions.MAX_RETRIES); + if (maxRetries != 0) { + throw new OptionValidationException( + "JDBC XA sink requires max_retries equal to 0 when is_exactly_once=true, " + + "otherwise it could cause duplicates."); + } + } + return true; + } + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java index de2da4aa4161..4197bdcef468 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactory.java @@ -17,6 +17,9 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.source; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.source.SourceSplit; @@ -26,6 +29,7 @@ import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcSourceConfig; import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcSourceTableConfig; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectLoader; @@ -33,6 +37,7 @@ import lombok.extern.slf4j.Slf4j; import java.io.Serializable; +import java.util.List; @Slf4j @AutoService(Factory.class) @@ -63,6 +68,15 @@ TableSource createSource(TableSourceFactoryContext context) { public OptionRule optionRule() { return OptionRule.builder() .required(JdbcSourceOptions.URL, JdbcSourceOptions.DRIVER) + .optional( + JdbcSourceOptions.TABLE_LIST, + Conditions.extension( + JdbcSourceOptions.TABLE_LIST, new TableListExclusiveValidator())) + .optional( + JdbcSourceOptions.WHERE_CONDITION, + Conditions.extension( + JdbcSourceOptions.WHERE_CONDITION, + new WhereConditionPrefixValidator())) .optional( JdbcSourceOptions.USERNAME, JdbcSourceOptions.PASSWORD, @@ -82,8 +96,6 @@ public OptionRule optionRule() { JdbcSourceOptions.SKIP_ANALYZE, JdbcSourceOptions.USE_REGEX, JdbcSourceOptions.TABLE_PATH, - JdbcSourceOptions.WHERE_CONDITION, - JdbcSourceOptions.TABLE_LIST, JdbcSourceOptions.SPLIT_SIZE, JdbcSourceOptions.SPLIT_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND, JdbcSourceOptions.SPLIT_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND, @@ -100,4 +112,39 @@ public OptionRule optionRule() { public Class getSourceClass() { return JdbcSource.class; } + + /** + * Submission-time validator that enforces mutual exclusion between {@code table_list} and the + * legacy {@code table_path}/{@code query} options. Users must choose one table selection mode, + * not both. + */ + static class TableListExclusiveValidator + implements ConditionExtension> { + @Override + public String description() { + return "'table_list' and 'table_path'/'query' are mutually exclusive"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List value) { + return !config.getOptional(JdbcSourceOptions.TABLE_PATH).isPresent() + && !config.getOptional(JdbcSourceOptions.QUERY).isPresent(); + } + } + + /** + * Submission-time validator that ensures {@code where_condition} starts with the keyword {@code + * "where"} to avoid malformed SQL at runtime. + */ + static class WhereConditionPrefixValidator implements ConditionExtension { + @Override + public String description() { + return "'where_condition' must start with 'where'"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String value) { + return value == null || value.toLowerCase().startsWith("where"); + } + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/JdbcCatalogFactoryTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/JdbcCatalogFactoryTest.java new file mode 100644 index 000000000000..5aadf30058d6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/JdbcCatalogFactoryTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.mysql.MySqlCatalogFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oceanbase.OceanBaseCatalogFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.psql.PostgresCatalogFactory; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class JdbcCatalogFactoryTest { + + private final OptionRule mysqlRule = new MySqlCatalogFactory().optionRule(); + private final OptionRule pgRule = new PostgresCatalogFactory().optionRule(); + + private void validate(OptionRule rule, Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + @Test + void testValidCatalogConfig() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:mysql://localhost:3306/mydb"); + cfg.put("username", "root"); + cfg.put("password", "secret"); + Assertions.assertDoesNotThrow(() -> validate(mysqlRule, cfg)); + } + + @Test + void testMySqlCatalogValidConfig() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:mysql://host:3306/mydb"); + cfg.put("username", "root"); + cfg.put("password", "pass"); + Assertions.assertDoesNotThrow(() -> validate(mysqlRule, cfg)); + } + + @Test + void testPostgresCatalogValidConfig() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:postgresql://host:5432/mydb"); + cfg.put("username", "postgres"); + cfg.put("password", "pass"); + Assertions.assertDoesNotThrow(() -> validate(pgRule, cfg)); + } + + @Test + void testUrlWithoutDatabaseFails() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:mysql://host:3306"); + cfg.put("username", "root"); + cfg.put("password", "pass"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(mysqlRule, cfg)); + } + + @Test + void testBlankUrlFails() { + Map cfg = new HashMap<>(); + cfg.put("url", ""); + cfg.put("username", "root"); + cfg.put("password", "pass"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(mysqlRule, cfg)); + } + + @Test + void testMissingUsernameFails() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:mysql://host:3306/mydb"); + cfg.put("password", "pass"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(mysqlRule, cfg)); + } + + @Test + void testMissingPasswordFails() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:mysql://host:3306/mydb"); + cfg.put("username", "root"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(mysqlRule, cfg)); + } + + @Test + void testOceanBaseWithoutCompatibleModeFails() { + OptionRule obRule = new OceanBaseCatalogFactory().optionRule(); + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:oceanbase://host:2881/mydb"); + cfg.put("username", "root"); + cfg.put("password", "pass"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(obRule, cfg)); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatBuilderTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatBuilderTest.java index 6f23eca84926..3b0ca748e325 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatBuilderTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatBuilderTest.java @@ -177,39 +177,6 @@ public void testOracleAppendValuesRejectsNonOracleDialect() { exception.getMessage()); } - @Test - public void testOracleAppendValuesRejectsCustomQuery() { - JdbcSinkConfig config = - appendValuesConfigBuilder().simpleSql("INSERT INTO TEST_TABLE VALUES (?)").build(); - - IllegalArgumentException exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> newBuilder(new OracleDialect(), config).build()); - - Assertions.assertEquals( - "oracle_insert_mode=APPEND_VALUES does not support custom query.", - exception.getMessage()); - } - - @Test - public void testOracleAppendValuesRejectsAutoCommitDisabled() { - JdbcSinkConfig config = - appendValuesConfigBuilder() - .jdbcConnectionConfig( - JdbcConnectionConfig.builder().autoCommit(false).build()) - .build(); - - IllegalArgumentException exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> newBuilder(new OracleDialect(), config).build()); - - Assertions.assertEquals( - "oracle_insert_mode=APPEND_VALUES requires auto_commit=true.", - exception.getMessage()); - } - @Test public void testOracleAppendValuesRejectsPrimaryKeys() { JdbcSinkConfig config = @@ -225,20 +192,6 @@ public void testOracleAppendValuesRejectsPrimaryKeys() { exception.getMessage()); } - @Test - public void testOracleAppendValuesRejectsExactlyOnce() { - JdbcSinkConfig config = appendValuesConfigBuilder().isExactlyOnce(true).build(); - - IllegalArgumentException exception = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> newBuilder(new OracleDialect(), config).build()); - - Assertions.assertEquals( - "oracle_insert_mode=APPEND_VALUES does not support exactly-once JDBC sink.", - exception.getMessage()); - } - private static JdbcSinkConfig.JdbcSinkConfigBuilder appendValuesConfigBuilder() { return JdbcSinkConfig.builder() .database("TEST_SCHEMA") diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactoryTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactoryTest.java new file mode 100644 index 000000000000..b2f55e410a68 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactoryTest.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.connector.TableSink; +import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext; +import org.apache.seatunnel.api.table.type.BasicType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class JdbcSinkFactoryTest { + + private final JdbcSinkFactory factory = new JdbcSinkFactory(); + private final OptionRule rule = factory.optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + private Map baseConfig() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:oracle:thin:@//localhost:1521/ORCL"); + cfg.put("driver", "oracle.jdbc.OracleDriver"); + cfg.put("schema_save_mode", "CREATE_SCHEMA_WHEN_NOT_EXIST"); + cfg.put("data_save_mode", "APPEND_DATA"); + cfg.put("generate_sink_sql", true); + cfg.put("database", "ORCL"); + return cfg; + } + + private CatalogTable createCatalogTable(boolean withPrimaryKey) { + TableSchema.Builder schemaBuilder = + TableSchema.builder() + .column(PhysicalColumn.of("id", BasicType.LONG_TYPE, 22, false, null, "id")) + .column( + PhysicalColumn.of( + "name", BasicType.STRING_TYPE, 128, false, null, "name")); + if (withPrimaryKey) { + schemaBuilder.primaryKey(PrimaryKey.of("pk_id", Collections.singletonList("id"))); + } + return CatalogTable.of( + TableIdentifier.of("catalog", "ORCL", null, "TEST_TABLE"), + schemaBuilder.build(), + new HashMap<>(), + new ArrayList<>(), + null, + "catalog"); + } + + /** + * Simulates the FactoryUtil.createAndPrepareSink entry path: OptionRule validation followed by + * factory.createSink(context). This covers the real submission-time path end-to-end. + */ + private TableSink createSinkViaFactoryContext(Map cfg, boolean withPrimaryKey) { + ReadonlyConfig config = ReadonlyConfig.fromMap(cfg); + ConfigValidator.of(config).validate(factory.optionRule()); + CatalogTable catalogTable = createCatalogTable(withPrimaryKey); + TableSinkFactoryContext context = + new TableSinkFactoryContext(catalogTable, config, getClass().getClassLoader()); + return factory.createSink(context); + } + + @Test + void testValidSinkConfig() { + Assertions.assertDoesNotThrow(() -> validate(baseConfig())); + } + + @Test + void testOracleAppendValuesValidConfig() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("generate_sink_sql", true); + cfg.put("auto_commit", true); + cfg.put("is_exactly_once", false); + cfg.put("use_copy_statement", false); + cfg.put("support_upsert_by_insert_only", false); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testExactlyOnceWithMaxRetriesZero() { + Map cfg = baseConfig(); + cfg.put("is_exactly_once", true); + cfg.put("max_retries", 0); + cfg.put("xa_data_source_class_name", "oracle.jdbc.xa.OracleXADataSource"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testOracleAppendValuesWithExactlyOnceFails() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("generate_sink_sql", true); + cfg.put("is_exactly_once", true); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testOracleAppendValuesWithCopyStatementFails() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("generate_sink_sql", true); + cfg.put("use_copy_statement", true); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testOracleAppendValuesWithAutoCommitFalseFails() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("generate_sink_sql", true); + cfg.put("auto_commit", false); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testOracleAppendValuesWithCustomQueryFails() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("generate_sink_sql", false); + cfg.remove("database"); + cfg.put("query", "INSERT INTO t VALUES(?,?)"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testOracleAppendValuesWithInsertOnlyUpsertFails() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("generate_sink_sql", true); + cfg.put("support_upsert_by_insert_only", true); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testExactlyOnceWithMaxRetriesNonZeroFails() { + Map cfg = baseConfig(); + cfg.put("is_exactly_once", true); + cfg.put("max_retries", 3); + cfg.put("xa_data_source_class_name", "oracle.jdbc.xa.OracleXADataSource"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingUrlFails() { + Map cfg = new HashMap<>(); + cfg.put("driver", "oracle.jdbc.OracleDriver"); + cfg.put("schema_save_mode", "CREATE_SCHEMA_WHEN_NOT_EXIST"); + cfg.put("data_save_mode", "APPEND_DATA"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingDriverFails() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:oracle:thin:@//localhost:1521/ORCL"); + cfg.put("schema_save_mode", "CREATE_SCHEMA_WHEN_NOT_EXIST"); + cfg.put("data_save_mode", "APPEND_DATA"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + // ---- Entry-level regression tests through factory-context path ---- + + @Test + void testFactoryContextPathValidConfig() { + Map cfg = baseConfig(); + Assertions.assertDoesNotThrow(() -> createSinkViaFactoryContext(cfg, false)); + } + + @Test + void testFactoryContextPathAppendValuesValidConfig() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("auto_commit", true); + cfg.put("is_exactly_once", false); + cfg.put("use_copy_statement", false); + cfg.put("support_upsert_by_insert_only", false); + Assertions.assertDoesNotThrow(() -> createSinkViaFactoryContext(cfg, false)); + } + + @Test + void testFactoryContextPathAppendValuesWithExactlyOnceFails() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("is_exactly_once", true); + Assertions.assertThrows( + OptionValidationException.class, () -> createSinkViaFactoryContext(cfg, false)); + } + + /** + * Verifies the CatalogTable-derived primary keys + APPEND_VALUES scenario. + * + *

When the upstream CatalogTable carries a primary key but the user does not set {@code + * primary_keys} in config, {@link JdbcSinkFactory#createSink} auto-populates primary_keys from + * the catalog schema. The OptionRule validation passes because the primary_keys conflict cannot + * be detected at config time (it depends on the catalog schema). The runtime guard in {@code + * JdbcOutputFormatBuilder.validateOracleInsertMode} catches this case and rejects APPEND_VALUES + * with non-empty primary keys. + */ + @Test + void testFactoryContextAppendValuesWithCatalogDerivedPrimaryKeys() { + Map cfg = baseConfig(); + cfg.put("oracle_insert_mode", "APPEND_VALUES"); + cfg.put("auto_commit", true); + cfg.put("is_exactly_once", false); + cfg.put("use_copy_statement", false); + cfg.put("support_upsert_by_insert_only", false); + + Assertions.assertDoesNotThrow( + () -> createSinkViaFactoryContext(cfg, true), + "Factory-level validation and createSink should succeed even when " + + "CatalogTable has primary keys; the PK + APPEND_VALUES conflict " + + "is guarded at runtime by JdbcOutputFormatBuilder.validateOracleInsertMode"); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactoryTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactoryTest.java new file mode 100644 index 000000000000..2a398704fa03 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/source/JdbcSourceFactoryTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.source; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class JdbcSourceFactoryTest { + + private final JdbcSourceFactory factory = new JdbcSourceFactory(); + private final OptionRule rule = factory.optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + private Map baseConfig() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:mysql://localhost:3306/test"); + cfg.put("driver", "com.mysql.cj.jdbc.Driver"); + return cfg; + } + + @Test + void testValidConfigWithTablePath() { + Map cfg = baseConfig(); + cfg.put("table_path", "test.users"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testValidConfigWithTableList() { + Map cfg = baseConfig(); + List> tableList = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("table_path", "test.users"); + tableList.add(entry); + cfg.put("table_list", tableList); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testValidConfigWithQuery() { + Map cfg = baseConfig(); + cfg.put("query", "SELECT * FROM users"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testValidConfigWithTablePathAndQuery() { + Map cfg = baseConfig(); + cfg.put("table_path", "test.users"); + cfg.put("query", "SELECT * FROM users WHERE active = 1"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testValidConfigWithWhereCondition() { + Map cfg = baseConfig(); + cfg.put("table_path", "test.users"); + cfg.put("where_condition", "where id > 100"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testTableListWithTablePathFails() { + Map cfg = baseConfig(); + List> tableList = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("table_path", "test.users"); + tableList.add(entry); + cfg.put("table_list", tableList); + cfg.put("table_path", "test.orders"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testTableListWithQueryFails() { + Map cfg = baseConfig(); + List> tableList = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("table_path", "test.users"); + tableList.add(entry); + cfg.put("table_list", tableList); + cfg.put("query", "SELECT * FROM orders"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testTableListWithBothTablePathAndQueryFails() { + Map cfg = baseConfig(); + List> tableList = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("table_path", "test.users"); + tableList.add(entry); + cfg.put("table_list", tableList); + cfg.put("table_path", "test.orders"); + cfg.put("query", "SELECT * FROM orders"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testWhereConditionWithoutPrefixFails() { + Map cfg = baseConfig(); + cfg.put("table_path", "test.users"); + cfg.put("where_condition", "id > 100"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingUrlFails() { + Map cfg = new HashMap<>(); + cfg.put("driver", "com.mysql.cj.jdbc.Driver"); + cfg.put("table_path", "test.users"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingDriverFails() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:mysql://localhost:3306/test"); + cfg.put("table_path", "test.users"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + // ---- Entry-level regression tests through factory-context path ---- + + /** + * Simulates the FactoryUtil.createAndPrepareSource entry path: OptionRule validation followed + * by factory.createSource(context). Verifies the real submission-time path end-to-end. + */ + @Test + void testFactoryContextPathValidConfig() { + Map cfg = baseConfig(); + cfg.put("table_path", "test.users"); + ReadonlyConfig config = ReadonlyConfig.fromMap(cfg); + ConfigValidator.of(config).validate(factory.optionRule()); + + TableSourceFactoryContext context = + new TableSourceFactoryContext(config, getClass().getClassLoader()); + Assertions.assertDoesNotThrow(() -> factory.createSource(context)); + } + + @Test + void testFactoryContextPathTableListExclusionFails() { + Map cfg = baseConfig(); + List> tableList = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("table_path", "test.users"); + tableList.add(entry); + cfg.put("table_list", tableList); + cfg.put("table_path", "test.orders"); + + ReadonlyConfig config = ReadonlyConfig.fromMap(cfg); + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(config).validate(factory.optionRule())); + } +} From 59ada4ec02f821d97491507c20ab3575be4aa86b Mon Sep 17 00:00:00 2001 From: yzeng1618 Date: Fri, 19 Jun 2026 18:25:52 +0800 Subject: [PATCH 030/375] [Improve][Core] Support transform and value constraint option rules in CLI (#11109) Co-authored-by: zengyi --- seatunnel-cli/README.md | 9 +- seatunnel-cli/README.zh-CN.md | 9 +- seatunnel-cli/seatunnel_cli/agents.py | 13 +- seatunnel-cli/seatunnel_cli/cli.py | 3 +- seatunnel-cli/seatunnel_cli/connectors.py | 99 +++++- seatunnel-cli/seatunnel_cli/skills.py | 32 +- .../tests/test_connector_metadata.py | 304 ++++++++++++++++++ 7 files changed, 446 insertions(+), 23 deletions(-) create mode 100644 seatunnel-cli/tests/test_connector_metadata.py diff --git a/seatunnel-cli/README.md b/seatunnel-cli/README.md index f268e3ff1293..17f81c02af8a 100644 --- a/seatunnel-cli/README.md +++ b/seatunnel-cli/README.md @@ -10,6 +10,7 @@ Describe your data synchronization task in English or Chinese, and the CLI gener - **Multi-Provider LLM** -- AWS Bedrock, Anthropic API, OpenAI (and compatible APIs like Azure OpenAI) - **Multi-Agent Pipeline** -- Planner -> Generator -> Validator -> Auto-fix, up to 3 correction rounds - **100+ Connectors** -- Full coverage of SeaTunnel's connector ecosystem with runtime metadata reflection +- **Transform Metadata** -- Source, sink, and transform plugins use full option rules and value constraints during generation - **Skill Framework** -- Three-layer generation: Skill SOP -> Golden Example -> Connector Metadata - **Auto-Save** -- Generated configs automatically saved to `.data/last_job.conf` (co-located with CLI) - **Auto-Fix** -- `/check` and `/run` failures trigger automatic LLM-powered diagnosis and config repair @@ -241,7 +242,7 @@ Options: | `/save ` | Save config to custom path (auto-saved to `.data/last_job.conf` on generation) | | `/check` | Dry-run validate last config; auto-diagnoses and fixes on failure | | `/run` | Execute last config via REST API or `seatunnel.sh`; auto-diagnoses on failure | -| `/connectors` | List all available sources, sinks, and transforms | +| `/connectors` | List all available sources, sinks, and transforms; transform option rules and constraints are supported during generation | | `/sessions` | List recent conversation sessions | | `/resume [id]` | Resume a previous session | | `/new` | Start a fresh session | @@ -371,7 +372,9 @@ User Input (natural language) Two-tier resolution with intelligent fallback: 1. **Runtime API** -- Live metadata from running SeaTunnel engine (`/option-rules` endpoint). Always accurate, zero maintenance. -2. **Bundled Metadata** -- `connector_metadata.json` ships with the CLI package. 150 connectors with full option rules, exported from SeaTunnel engine via reflection. Zero LLM token cost. +2. **Bundled Metadata** -- `connector_metadata.json` ships with the CLI package. Source, sink, and transform plugins include full option rules and value constraints exported from SeaTunnel engine via reflection. Zero LLM token cost. + +Transform metadata is resolved through the same path as source and sink metadata, so prompts can include transform-specific required options and constraints such as non-blank SQL queries. ### Memory System @@ -394,7 +397,7 @@ Memory is stored locally at `.data/memory.json` (co-located with the CLI package ## Connector Metadata -The CLI ships with `connector_metadata.json` (150 connectors), exported from the SeaTunnel engine via runtime reflection. No extra steps needed. +The CLI ships with `connector_metadata.json`, exported from the SeaTunnel engine via runtime reflection. It includes source, sink, and transform plugin option rules, conditional options, and value constraints. No extra steps needed. To re-export for a different SeaTunnel version (requires a running engine): diff --git a/seatunnel-cli/README.zh-CN.md b/seatunnel-cli/README.zh-CN.md index f59ea644ffdb..0b0f97f57bac 100644 --- a/seatunnel-cli/README.zh-CN.md +++ b/seatunnel-cli/README.zh-CN.md @@ -10,6 +10,7 @@ - **多 LLM 提供商** -- 支持 AWS Bedrock、Anthropic API、OpenAI（及兼容 API，如 Azure OpenAI） - **多智能体流水线** -- 规划器 -> 生成器 -> 校验器 -> 自动修复，最多 3 轮纠错 - **100+ 连接器** -- 全面覆盖 SeaTunnel 连接器生态，支持运行时元数据反射 +- **Transform 元数据** -- Source、Sink 和 Transform 插件在生成配置时都支持完整选项规则和值约束 - **技能框架** -- 三层生成：技能 SOP -> 黄金示例 -> 连接器元数据 - **自动保存** -- 生成的配置自动保存到 `.data/last_job.conf`（与 CLI 同目录） - **自动修复** -- `/check` 和 `/run` 失败时自动触发 LLM 诊断和配置修复 @@ -241,7 +242,7 @@ seatunnel [request] [options] | `/save ` | 将配置保存到自定义路径（生成时自动保存到 `.data/last_job.conf`） | | `/check` | 试运行校验最近的配置；失败时自动诊断并修复 | | `/run` | 通过 REST API 或 `seatunnel.sh` 执行最近的配置；失败时自动诊断 | -| `/connectors` | 列出所有可用的 Source、Sink 和 Transform | +| `/connectors` | 列出所有可用的 Source、Sink 和 Transform；生成配置时支持 Transform 选项规则和值约束 | | `/sessions` | 列出最近的对话会话 | | `/resume [id]` | 恢复之前的会话 | | `/new` | 开始新会话 | @@ -371,7 +372,9 @@ seatunnel [request] [options] 两级解析，智能回退： 1. **运行时 API** -- 从运行中的 SeaTunnel 引擎获取实时元数据（`/option-rules` 端点）。始终准确，零维护成本。 -2. **内置元数据** -- `connector_metadata.json` 随 CLI 包分发。包含 150 个连接器的完整选项规则，通过 SeaTunnel 引擎反射导出。零 LLM Token 消耗。 +2. **内置元数据** -- `connector_metadata.json` 随 CLI 包分发。Source、Sink 和 Transform 插件都包含通过 SeaTunnel 引擎反射导出的完整选项规则和值约束。零 LLM Token 消耗。 + +Transform 元数据与 Source、Sink 元数据走同一条解析路径，因此提示词可以包含 Transform 专属必填项和值约束，例如 SQL 查询不能为空。 ### 记忆系统 @@ -394,7 +397,7 @@ CLI 跨会话记忆信息，以提高配置准确性： ## 连接器元数据 -CLI 内置 `connector_metadata.json`（150 个连接器），通过运行时反射从 SeaTunnel 引擎导出，无需额外步骤。 +CLI 内置 `connector_metadata.json`，通过运行时反射从 SeaTunnel 引擎导出。它包含 Source、Sink 和 Transform 插件的选项规则、条件选项和值约束，无需额外步骤。 如需为不同 SeaTunnel 版本重新导出（需要运行中的引擎）： diff --git a/seatunnel-cli/seatunnel_cli/agents.py b/seatunnel-cli/seatunnel_cli/agents.py index 833f2d0f228f..89c5010a4c01 100644 --- a/seatunnel-cli/seatunnel_cli/agents.py +++ b/seatunnel-cli/seatunnel_cli/agents.py @@ -68,8 +68,10 @@ "toolSpec": { "name": "get_connector_info", "description": "Get detailed info about a specific connector including parameters and examples. " - "IMPORTANT: Always specify connector_type ('source' or 'sink') to get the correct " - "type-specific options. Source and sink connectors have different required/optional parameters.", + "IMPORTANT: Always specify connector_type " + "('source', 'sink', or 'transform') to get the correct " + "type-specific options. Source, sink, and transform plugins can " + "have different required/optional parameters.", "inputSchema": { "json": { "type": "object", @@ -80,9 +82,10 @@ }, "connector_type": { "type": "string", - "enum": ["source", "sink"], - "description": "Whether this connector is used as 'source' or 'sink'. " - "Source and sink have different options — always specify this.", + "enum": ["source", "sink", "transform"], + "description": "Whether this plugin is used as 'source', 'sink', " + "or 'transform'. These plugin types have different " + "options — always specify this.", }, }, "required": ["connector_name"], diff --git a/seatunnel-cli/seatunnel_cli/cli.py b/seatunnel-cli/seatunnel_cli/cli.py index e357d99783a2..6be82c7f70e2 100644 --- a/seatunnel-cli/seatunnel_cli/cli.py +++ b/seatunnel-cli/seatunnel_cli/cli.py @@ -70,7 +70,7 @@ [bold]/save [/bold] — Save config to custom path (auto-saved to .data/last_job.conf) [bold]/check[/bold] — Dry-run validate last config (auto-fixes on failure) [bold]/run[/bold] — Execute last config with SeaTunnel - [bold]/connectors[/bold] — List available connectors + [bold]/connectors[/bold] — List available sources, sinks, and transforms [bold]/config[/bold] — Show/change LLM provider settings [bold]/sessions[/bold] — List recent sessions [bold]/resume [id][/bold] — Resume a previous session @@ -1066,6 +1066,7 @@ def _handle_command(self, cmd: str): self.console.print(f" [bold]Sources:[/bold] {', '.join(names['sources'])}") self.console.print(f" [bold]Sinks:[/bold] {', '.join(names['sinks'])}") self.console.print(f" [bold]Transforms:[/bold] {', '.join(names['transforms'])}") + self.console.print(" Transform option rules and value constraints are supported during generation.") elif command == "/config": self._cmd_config(arg.strip()) diff --git a/seatunnel-cli/seatunnel_cli/connectors.py b/seatunnel-cli/seatunnel_cli/connectors.py index 614df5d91c73..1fa2e73769b7 100644 --- a/seatunnel-cli/seatunnel_cli/connectors.py +++ b/seatunnel-cli/seatunnel_cli/connectors.py @@ -180,6 +180,11 @@ def _api_response_to_detail(resp: dict) -> dict: "then_options": nested_opts, }) + value_constraints = [ + _value_constraint_to_dict(constraint) + for constraint in rule.get("valueConstraints", []) + ] + return { "name": resp.get("pluginName", ""), "types": [resp.get("pluginType", "unknown")], @@ -187,6 +192,7 @@ def _api_response_to_detail(resp: dict) -> dict: "optional": optional, "exclusive": [], "conditional": conditional, + "value_constraints": value_constraints, "examples": [], "source": "runtime_api", } @@ -242,6 +248,43 @@ def _api_opt_to_dict(opt: dict, rule_type: str = "") -> dict: return result +def _value_constraint_to_dict(constraint: dict) -> dict: + """Convert one value constraint to a compact prompt-friendly form.""" + result = {"expression": constraint.get("expression", "")} + tree = constraint.get("conditionTree") or {} + if not isinstance(tree, dict): + return result + + option = tree.get("option") or {} + key = tree.get("key") or option.get("key") + if key: + result["key"] = key + + expect_value = tree.get("expectValue") + if expect_value is not None: + result["expect"] = _constraint_value_to_text(expect_value) + + operator = tree.get("compareOperator") + if operator: + result["operator"] = str(operator) + + condition_operator = tree.get("conditionOperator") + if condition_operator: + result["condition_operator"] = str(condition_operator) + + return result + + +def _constraint_value_to_text(value) -> str: + """Return a stable, prompt-friendly string for a constraint expected value.""" + if isinstance(value, str): + return value + try: + return json.dumps(value, ensure_ascii=False) + except TypeError: + return str(value) + + # ─── Runtime JSON metadata (from Java exporter) ─── _RUNTIME_METADATA: dict | None = None # Parsed connector_metadata.json @@ -371,6 +414,11 @@ def _runtime_metadata_to_detail(entry: dict) -> dict: "then_options": [_runtime_opt_to_dict(opt)], }) + value_constraints = [ + _value_constraint_to_dict(constraint) + for constraint in entry.get("valueConstraints", []) + ] + detail = { "name": entry.get("name", ""), "types": [entry.get("type", "unknown")], @@ -378,6 +426,7 @@ def _runtime_metadata_to_detail(entry: dict) -> dict: "optional": optional, "exclusive": [], "conditional": conditional, + "value_constraints": value_constraints, "examples": [], "source": "runtime_json", } @@ -585,6 +634,11 @@ def format_metadata_for_prompt(metadata: dict, plugin_name: str, plugin_type: st opt_list = ", ".join(f"`{o['key']}`" for o in unique_opts) lines.append(f" - When {cond_label} → {opt_list}") + if metadata.get("value_constraints"): + lines.append("**Value Constraints (must satisfy):**") + for constraint in metadata["value_constraints"]: + lines.append(_format_value_constraint(constraint, prefix=" - ")) + # 3. Optional: show important ones with type, rest as key list important_keys = {"username", "password", "query", "table_path", "table_list", "user", "bucket", "path", "topic", "database", "table", @@ -609,6 +663,29 @@ def format_metadata_for_prompt(metadata: dict, plugin_name: str, plugin_type: st return "\n".join(lines) +def _format_value_constraint(constraint: dict, prefix: str = "- ") -> str: + """Format one value constraint without attempting to evaluate it locally.""" + key = constraint.get("key") + operator = constraint.get("operator") + expect = constraint.get("expect") + expression = constraint.get("expression", "") + + if key and operator and expect and str(operator).lower() == "extension": + line = f"{prefix}`{key}` {expect}" + elif key and operator and expect: + line = f"{prefix}`{key}` {operator} {expect}" + elif key and expect: + line = f"{prefix}`{key}` {expect}" + elif expression: + line = f"{prefix}{expression}" + else: + line = f"{prefix}" + + if expression and expression not in line.replace("`", ""): + line += f" ({expression})" + return line + + def _format_opt_detail(opt: dict) -> str: """Format one option with full detail for LLM prompt.""" line = f" - `{opt['key']}` ({opt.get('type', '?')})" @@ -904,7 +981,10 @@ def get_connector_catalog() -> str: count = sum(len(types) for types in connectors.values()) lines = [f"## Available Connectors ({len(connectors)} connectors, {count} endpoints) [engine: {engine_status}]\n"] - lines.append("Use `get_connector_info` tool with `connector_type` ('source' or 'sink') to get type-specific details.") + lines.append( + "Use `get_connector_info` tool with `connector_type` " + "('source', 'sink', or 'transform') to get type-specific details." + ) if engine_status == "connected": lines.append("(Details fetched live from running engine — always up-to-date)\n") elif runtime_meta: @@ -940,8 +1020,8 @@ def get_connector_detail(name: str, connector_type: str | None = None) -> str | Args: name: Connector name (e.g., 'Jdbc', 'Kafka', 'S3File'). - connector_type: 'source' or 'sink' for type-specific options. - None returns both types with separate sections. + connector_type: 'source', 'sink', or 'transform' for type-specific options. + None returns all available types with separate sections. Resolution order: 1. Runtime API (live /option-rules endpoint — always accurate) @@ -952,7 +1032,7 @@ def get_connector_detail(name: str, connector_type: str | None = None) -> str | Returns a formatted string with all options, defaults, conditions, and examples. """ # ── 1. Try runtime API ── - query_types = [connector_type] if connector_type else ["source", "sink"] + query_types = [connector_type] if connector_type else ["source", "sink", "transform"] api_details: dict[str, dict] = {} for ptype in query_types: api_resp = _fetch_option_rules(ptype, name) @@ -1044,6 +1124,11 @@ def _format_connector_detail_typed( deps = ", ".join(cond["then_require"]) lines.append(f"- When `{cond['when']}` = `{cond['equals']}` → also require: {deps}") + if detail.get("value_constraints"): + lines.append(f"\n## {type_label} — Value Constraints") + for constraint in detail["value_constraints"]: + lines.append(_format_value_constraint(constraint)) + if len(show_types) > 1: lines.append("") # separator between types @@ -1126,6 +1211,7 @@ def list_connector_names() -> dict: runtime_meta = _load_runtime_metadata() sources = set() sinks = set() + transforms = set(TRANSFORMS.keys()) if runtime_meta: for key in runtime_meta: parts = key.split(":", 1) @@ -1135,8 +1221,9 @@ def list_connector_names() -> dict: sources.add(name) elif ctype == "sink": sinks.add(name) - transforms = list(TRANSFORMS.keys()) - return {"sources": sorted(sources), "sinks": sorted(sinks), "transforms": transforms} + elif ctype == "transform": + transforms.add(name) + return {"sources": sorted(sources), "sinks": sorted(sinks), "transforms": sorted(transforms)} def validate_connector_options( diff --git a/seatunnel-cli/seatunnel_cli/skills.py b/seatunnel-cli/seatunnel_cli/skills.py index 60fcdf9d5841..21442e7e922a 100644 --- a/seatunnel-cli/seatunnel_cli/skills.py +++ b/seatunnel-cli/seatunnel_cli/skills.py @@ -273,6 +273,31 @@ def _collect_required_options( return result +def _transform_connector_name(transform: object) -> str: + """Return the transform plugin name from a structured or string plan field.""" + if not transform: + return "" + if isinstance(transform, dict): + for key in ("connector", "name", "type"): + value = transform.get(key) + if value: + return str(value) + return "" + return str(transform) + + +def _metadata_targets_for_pipeline(pipeline: PipelineSlot) -> list[tuple[str, str]]: + """Return source, sink, and transform metadata targets for a pipeline.""" + targets = [ + (pipeline.source_connector, "source"), + (pipeline.sink_connector, "sink"), + ] + transform_name = _transform_connector_name(pipeline.transform) + if transform_name: + targets.append((transform_name, "transform")) + return [(name, ctype) for name, ctype in targets if name] + + def llm_check_missing_info( client, user_request: str, @@ -516,10 +541,7 @@ def fill_and_check(self, user_request: str, all_required: list[dict] = [] seen_keys: set[str] = set() for p in self.plan.pipelines: - for connector, ctype in [ - (p.source_connector, "source"), - (p.sink_connector, "sink"), - ]: + for connector, ctype in _metadata_targets_for_pipeline(p): for opt in _collect_required_options(connector, ctype): if opt["key"] not in seen_keys: seen_keys.add(opt["key"]) @@ -539,7 +561,7 @@ def fetch_all_metadata(self, on_status: Callable) -> str: seen: set[tuple[str, str]] = set() sections: list[str] = [] for p in self.plan.pipelines: - for name, ctype in [(p.source_connector, "source"), (p.sink_connector, "sink")]: + for name, ctype in _metadata_targets_for_pipeline(p): if (name, ctype) in seen: continue seen.add((name, ctype)) diff --git a/seatunnel-cli/tests/test_connector_metadata.py b/seatunnel-cli/tests/test_connector_metadata.py new file mode 100644 index 000000000000..0be5be840524 --- /dev/null +++ b/seatunnel-cli/tests/test_connector_metadata.py @@ -0,0 +1,304 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import unittest +from unittest.mock import patch + +from seatunnel_cli import connectors +from seatunnel_cli.skills import PipelineSlot, SkillExecutor, StructuredPlan + + +def _runtime_transform_entry(): + return { + "name": "Sql", + "type": "transform", + "required": [ + { + "key": "query", + "type": "string", + "defaultValue": None, + "description": "SQL query", + "category": "absolutely_required", + } + ], + "optional": [], + "conditionRules": [], + "valueConstraints": [ + { + "expression": "query must not be blank", + "conditionTree": { + "key": "query", + "expectValue": "must not be blank", + "compareOperator": "extension", + "conditionOperator": "EXTENSION", + "conditionOperatorCategory": "EXTENSION", + }, + } + ], + } + + +def _api_transform_response(): + return { + "pluginName": "Sql", + "pluginType": "transform", + "optionRule": { + "requiredOptions": [ + { + "ruleType": "absolutely_required", + "options": [ + { + "key": "query", + "type": "java.lang.String", + "defaultValue": None, + "description": "SQL query", + } + ], + } + ], + "optionalOptions": [], + "conditionRules": [], + "valueConstraints": [ + { + "expression": "query must not be blank", + "conditionTree": { + "option": {"key": "query"}, + "expectValue": "must not be blank", + "compareOperator": "extension", + "conditionOperator": "EXTENSION", + "conditionOperatorCategory": "EXTENSION", + }, + } + ], + }, + } + + +class ConnectorMetadataSyncTest(unittest.TestCase): + def test_api_response_preserves_value_constraints(self): + detail = connectors._api_response_to_detail(_api_transform_response()) + + self.assertEqual( + detail["value_constraints"], + [ + { + "expression": "query must not be blank", + "key": "query", + "expect": "must not be blank", + "operator": "extension", + "condition_operator": "EXTENSION", + } + ], + ) + + def test_runtime_metadata_preserves_value_constraints(self): + detail = connectors._runtime_metadata_to_detail(_runtime_transform_entry()) + + self.assertEqual(detail["value_constraints"][0]["key"], "query") + self.assertEqual(detail["value_constraints"][0]["operator"], "extension") + self.assertEqual(detail["value_constraints"][0]["expect"], "must not be blank") + + def test_prompt_includes_value_constraints(self): + detail = connectors._runtime_metadata_to_detail(_runtime_transform_entry()) + + prompt = connectors.format_metadata_for_prompt(detail, "Sql", "transform") + + self.assertIn("Value Constraints", prompt) + self.assertIn("`query` must not be blank", prompt) + self.assertNotIn("`query` extension must not be blank", prompt) + + def test_value_constraint_tolerates_missing_or_malformed_condition_tree(self): + constraints = [ + {"expression": "query must not be blank"}, + {"expression": "query must not be blank", "conditionTree": None}, + { + "expression": "query must not be blank", + "conditionTree": ["not", "a", "dict"], + }, + ] + + for constraint in constraints: + with self.subTest(constraint=constraint): + self.assertEqual( + connectors._value_constraint_to_dict(constraint), + {"expression": "query must not be blank"}, + ) + + def test_value_constraint_formats_complex_expect_value_as_json(self): + constraint = connectors._value_constraint_to_dict( + { + "expression": "mode should be in the allowed list", + "conditionTree": { + "key": "mode", + "expectValue": ["batch", "streaming"], + "compareOperator": "in", + }, + } + ) + + self.assertEqual( + connectors._format_value_constraint(constraint), + '- `mode` in ["batch", "streaming"] (mode should be in the allowed list)', + ) + + def test_connector_detail_can_fetch_transform_metadata_from_runtime_api(self): + calls = [] + + def fake_fetch(plugin_type, plugin_name): + calls.append((plugin_type, plugin_name)) + if plugin_type == "transform" and plugin_name == "Sql": + return _api_transform_response() + return None + + with patch.object(connectors, "_fetch_option_rules", side_effect=fake_fetch): + with patch.object(connectors, "_load_runtime_metadata", return_value={}): + detail = connectors.get_connector_detail( + "Sql", connector_type="transform" + ) + + self.assertEqual(calls, [("transform", "Sql")]) + self.assertIn("[source: runtime API]", detail) + self.assertIn("Types: transform", detail) + self.assertIn("Value Constraints", detail) + + def test_connector_detail_queries_transform_when_type_is_not_specified(self): + calls = [] + + def fake_fetch(plugin_type, plugin_name): + calls.append((plugin_type, plugin_name)) + if plugin_type == "transform" and plugin_name == "Sql": + return _api_transform_response() + return None + + with patch.object(connectors, "_fetch_option_rules", side_effect=fake_fetch): + detail = connectors.get_connector_detail("Sql") + + self.assertEqual( + calls, + [("source", "Sql"), ("sink", "Sql"), ("transform", "Sql")], + ) + self.assertIn("Types: transform", detail) + + def test_list_connector_names_includes_runtime_transforms(self): + runtime_meta = { + "source:FakeSource": {"name": "FakeSource", "type": "source"}, + "sink:Console": {"name": "Console", "type": "sink"}, + "transform:DataValidator": {"name": "DataValidator", "type": "transform"}, + } + + with patch.object( + connectors, "_load_runtime_metadata", return_value=runtime_meta + ): + names = connectors.list_connector_names() + + self.assertIn("DataValidator", names["transforms"]) + self.assertIn("Sql", names["transforms"]) + + def test_skill_executor_fetches_transform_metadata_from_plan(self): + plan = StructuredPlan( + pipelines=[ + PipelineSlot( + "pipeline_1", + source_connector="FakeSource", + sink_connector="Console", + transform="Sql", + ) + ] + ) + executor = SkillExecutor(plan, skills=[]) + calls = [] + + def fake_fetch(name, connector_type): + calls.append((name, connector_type)) + return { + "name": name, + "required": [], + "optional": [], + "conditional": [], + "source": "test", + } + + with patch( + "seatunnel_cli.connectors.fetch_connector_metadata", side_effect=fake_fetch + ): + metadata = executor.fetch_all_metadata(lambda *_args: None) + + self.assertEqual( + calls, + [ + ("FakeSource", "source"), + ("Console", "sink"), + ("Sql", "transform"), + ], + ) + self.assertIn("### Sql (TRANSFORM)", metadata) + + def test_skill_executor_fill_and_check_includes_transform_required_options(self): + plan = StructuredPlan( + pipelines=[ + PipelineSlot( + "pipeline_1", + source_connector="FakeSource", + sink_connector="Console", + transform={"connector": "Sql"}, + ) + ] + ) + executor = SkillExecutor(plan, skills=[]) + collect_calls = [] + + def fake_collect(connector, connector_type): + collect_calls.append((connector, connector_type)) + if connector_type == "transform": + return [ + { + "key": "query", + "type": "string", + "description": "SQL query", + "connector": connector, + "connector_type": connector_type, + } + ] + return [] + + with patch( + "seatunnel_cli.skills._collect_required_options", side_effect=fake_collect + ): + with patch( + "seatunnel_cli.skills.llm_check_missing_info", + return_value=["Please provide query"], + ) as check_missing: + missing = executor.fill_and_check( + "Filter rows with a SQL transform", client=object() + ) + + self.assertEqual( + collect_calls, + [ + ("FakeSource", "source"), + ("Console", "sink"), + ("Sql", "transform"), + ], + ) + self.assertEqual(missing, ["Please provide query"]) + required_options = check_missing.call_args[0][2] + self.assertEqual(required_options[0]["key"], "query") + self.assertEqual(required_options[0]["connector_type"], "transform") + + +if __name__ == "__main__": + unittest.main() From 33ffd48a49b1f6074ff88818ce692c0c4db9b679 Mon Sep 17 00:00:00 2001 From: QuakeWang <45645138+QuakeWang@users.noreply.github.com> Date: Fri, 19 Jun 2026 18:28:02 +0800 Subject: [PATCH 031/375] [Fix][Connector-V2] Fix Paimon stream source state restore (#11132) Signed-off-by: QuakeWang --- .../seatunnel/paimon/source/PaimonSource.java | 1 + .../paimon/source/PaimonSourceState.java | 43 ++++- .../enumerator/AbstractSplitEnumerator.java | 34 +++- .../PaimonBatchSourceSplitEnumerator.java | 2 +- .../PaimonStreamSourceSplitEnumerator.java | 17 ++ ...PaimonStreamSourceSplitEnumeratorTest.java | 167 ++++++++++++++++++ 6 files changed, 255 insertions(+), 9 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonStreamSourceSplitEnumeratorTest.java diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSource.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSource.java index 501a1c46b928..aa16e457e13d 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSource.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSource.java @@ -176,6 +176,7 @@ public SourceSplitEnumerator restoreEnumer enumeratorContext, checkpointState.getAssignedSplits(), checkpointState.getCurrentSnapshotId(), + checkpointState.getCurrentSnapshotIds(), readBuilders, 1); } diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSourceState.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSourceState.java index db6392520c3b..73991cdd9a72 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSourceState.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/PaimonSourceState.java @@ -17,17 +17,15 @@ package org.apache.seatunnel.connectors.seatunnel.paimon.source; -import lombok.AllArgsConstructor; -import lombok.Getter; - import javax.annotation.Nullable; import java.io.Serializable; +import java.util.Collections; import java.util.Deque; +import java.util.HashMap; +import java.util.Map; /** Paimon connector source state, saves the splits has assigned to readers. */ -@Getter -@AllArgsConstructor public class PaimonSourceState implements Serializable { private static final long serialVersionUID = 1L; @@ -35,4 +33,39 @@ public class PaimonSourceState implements Serializable { private final Deque assignedSplits; private final @Nullable Long currentSnapshotId; + + private final Map currentSnapshotIds; + + public PaimonSourceState( + Deque assignedSplits, @Nullable Long currentSnapshotId) { + this.assignedSplits = assignedSplits; + this.currentSnapshotId = currentSnapshotId; + this.currentSnapshotIds = Collections.emptyMap(); + } + + public PaimonSourceState( + Deque assignedSplits, Map currentSnapshotIds) { + this.assignedSplits = assignedSplits; + this.currentSnapshotIds = new HashMap<>(currentSnapshotIds); + this.currentSnapshotId = getSingleSnapshotId(this.currentSnapshotIds); + } + + public Deque getAssignedSplits() { + return assignedSplits; + } + + public @Nullable Long getCurrentSnapshotId() { + return currentSnapshotId; + } + + public Map getCurrentSnapshotIds() { + return currentSnapshotIds == null ? Collections.emptyMap() : currentSnapshotIds; + } + + private static @Nullable Long getSingleSnapshotId(Map currentSnapshotIds) { + if (currentSnapshotIds.size() != 1) { + return null; + } + return currentSnapshotIds.values().iterator().next(); + } } diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/AbstractSplitEnumerator.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/AbstractSplitEnumerator.java index e18bda0e229f..ed1356656a8f 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/AbstractSplitEnumerator.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/AbstractSplitEnumerator.java @@ -42,6 +42,7 @@ import java.util.Deque; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -73,6 +74,8 @@ public abstract class AbstractSplitEnumerator @Nullable protected Long nextSnapshotId; + protected final Map nextSnapshotIds; + private ExecutorService executorService; public AbstractSplitEnumerator( @@ -82,9 +85,28 @@ public AbstractSplitEnumerator( Map readBuilders, int splitMaxPerTask, JobMode jobMode) { + this( + context, + pendingSplits, + nextSnapshotId, + new HashMap<>(), + readBuilders, + splitMaxPerTask, + jobMode); + } + + public AbstractSplitEnumerator( + Context context, + Deque pendingSplits, + @Nullable Long nextSnapshotId, + Map nextSnapshotIds, + Map readBuilders, + int splitMaxPerTask, + JobMode jobMode) { this.context = context; this.pendingSplits = new LinkedList<>(pendingSplits); this.nextSnapshotId = nextSnapshotId; + this.nextSnapshotIds = new LinkedHashMap<>(nextSnapshotIds); this.readersAwaitingSplit = new LinkedHashSet<>(); this.splitGenerator = new PaimonSourceSplitGenerator(); this.splitMaxNum = context.currentParallelism() * splitMaxPerTask; @@ -101,8 +123,12 @@ public AbstractSplitEnumerator( ? readBuilder.newScan() : readBuilder.newStreamScan(); tableScans.put(tableId, scan); - if (scan instanceof StreamTableScan && nextSnapshotId != null) { - ((StreamTableScan) scan).restore(nextSnapshotId); + Long tableSnapshotId = + this.nextSnapshotIds.isEmpty() + ? nextSnapshotId + : this.nextSnapshotIds.get(tableId); + if (scan instanceof StreamTableScan && tableSnapshotId != null) { + ((StreamTableScan) scan).restore(tableSnapshotId); } }); } @@ -146,7 +172,7 @@ public void registerReader(int subtaskId) { @Override public PaimonSourceState snapshotState(long checkpointId) throws Exception { synchronized (stateLock) { - return new PaimonSourceState(pendingSplits, nextSnapshotId); + return new PaimonSourceState(pendingSplits, nextSnapshotIds); } } @@ -245,6 +271,8 @@ protected void processDiscoveredSplits( for (PlanWithNextSnapshotId planWithNextSnapshotId : planWithNextSnapshotIds) { nextSnapshotId = planWithNextSnapshotId.nextSnapshotId; + nextSnapshotIds.put( + planWithNextSnapshotId.tableId, planWithNextSnapshotId.nextSnapshotId); TableScan.Plan plan = planWithNextSnapshotId.plan; if (plan.splits().isEmpty()) { continue; diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonBatchSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonBatchSourceSplitEnumerator.java index 43cf5c4e3e2e..6b777e7704f9 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonBatchSourceSplitEnumerator.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonBatchSourceSplitEnumerator.java @@ -58,7 +58,7 @@ public void run() throws Exception { @Override public PaimonSourceState snapshotState(long checkpointId) throws Exception { synchronized (stateLock) { - return new PaimonSourceState(pendingSplits, null); + return new PaimonSourceState(pendingSplits, (Long) null); } } diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonStreamSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonStreamSourceSplitEnumerator.java index 6852472f59c7..67706a8e5146 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonStreamSourceSplitEnumerator.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonStreamSourceSplitEnumerator.java @@ -39,10 +39,27 @@ public PaimonStreamSourceSplitEnumerator( @Nullable Long nextSnapshotId, Map readBuilders, int splitMaxPerTask) { + this( + context, + pendingSplits, + nextSnapshotId, + java.util.Collections.emptyMap(), + readBuilders, + splitMaxPerTask); + } + + public PaimonStreamSourceSplitEnumerator( + Context context, + Deque pendingSplits, + @Nullable Long nextSnapshotId, + Map nextSnapshotIds, + Map readBuilders, + int splitMaxPerTask) { super( context, pendingSplits, nextSnapshotId, + nextSnapshotIds, readBuilders, splitMaxPerTask, JobMode.STREAMING); diff --git a/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonStreamSourceSplitEnumeratorTest.java b/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonStreamSourceSplitEnumeratorTest.java new file mode 100644 index 000000000000..dce0b369192d --- /dev/null +++ b/seatunnel-connectors-v2/connector-paimon/src/test/java/org/apache/seatunnel/connectors/seatunnel/paimon/source/enumerator/PaimonStreamSourceSplitEnumeratorTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.paimon.source.enumerator; + +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.connectors.seatunnel.paimon.source.PaimonSourceSplit; +import org.apache.seatunnel.connectors.seatunnel.paimon.source.PaimonSourceState; + +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.StreamTableScan; +import org.apache.paimon.table.source.TableScan; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class PaimonStreamSourceSplitEnumeratorTest { + + @Test + void shouldKeepIndependentSnapshotIdForEachTableWhenRestore() throws Exception { + StreamTableScan firstScan = newStreamScan(10L); + StreamTableScan secondScan = newStreamScan(20L); + + PaimonStreamSourceSplitEnumerator enumerator = + new PaimonStreamSourceSplitEnumerator( + context(), + new LinkedList<>(), + null, + readBuilders(firstScan, secondScan), + 1); + + enumerator.processDiscoveredSplits(enumerator.scanNextSnapshot(), null); + PaimonSourceState state = enumerator.snapshotState(1L); + StreamTableScan restoredFirstScan = newStreamScan(11L); + StreamTableScan restoredSecondScan = newStreamScan(21L); + + PaimonStreamSourceSplitEnumerator restored = + new PaimonStreamSourceSplitEnumerator( + context(), + state.getAssignedSplits(), + state.getCurrentSnapshotId(), + state.getCurrentSnapshotIds(), + readBuilders(restoredFirstScan, restoredSecondScan), + 1); + restored.close(); + enumerator.close(); + + verify(firstScan).checkpoint(); + verify(secondScan).checkpoint(); + assertEquals(10L, state.getCurrentSnapshotIds().get("db.table_a")); + assertEquals(20L, state.getCurrentSnapshotIds().get("db.table_b")); + verify(restoredFirstScan).restore(10L); + verify(restoredSecondScan).restore(20L); + } + + @Test + void shouldNotFallbackToAnotherTableSnapshotWhenTableSnapshotIsNull() throws Exception { + StreamTableScan firstScan = newStreamScan(10L); + StreamTableScan secondScan = newStreamScan(null); + + PaimonStreamSourceSplitEnumerator enumerator = + new PaimonStreamSourceSplitEnumerator( + context(), + new LinkedList<>(), + null, + readBuilders(firstScan, secondScan), + 1); + + enumerator.processDiscoveredSplits(enumerator.scanNextSnapshot(), null); + PaimonSourceState state = enumerator.snapshotState(1L); + StreamTableScan restoredFirstScan = newStreamScan(11L); + StreamTableScan restoredSecondScan = newStreamScan(21L); + + PaimonStreamSourceSplitEnumerator restored = + new PaimonStreamSourceSplitEnumerator( + context(), + state.getAssignedSplits(), + state.getCurrentSnapshotId(), + state.getCurrentSnapshotIds(), + readBuilders(restoredFirstScan, restoredSecondScan), + 1); + restored.close(); + enumerator.close(); + + assertEquals(10L, state.getCurrentSnapshotIds().get("db.table_a")); + assertTrue(state.getCurrentSnapshotIds().containsKey("db.table_b")); + assertNull(state.getCurrentSnapshotIds().get("db.table_b")); + verify(restoredFirstScan).restore(10L); + verify(restoredSecondScan, never()).restore(10L); + } + + @Test + void shouldRestoreLegacySingleSnapshotId() throws Exception { + StreamTableScan restoredScan = newStreamScan(11L); + + PaimonStreamSourceSplitEnumerator restored = + new PaimonStreamSourceSplitEnumerator( + context(), new LinkedList<>(), 10L, readBuilders(restoredScan), 1); + restored.close(); + + verify(restoredScan).restore(10L); + } + + private static Map readBuilders( + StreamTableScan firstScan, StreamTableScan secondScan) { + Map readBuilders = new LinkedHashMap<>(); + readBuilders.put("db.table_a", readBuilder(firstScan)); + readBuilders.put("db.table_b", readBuilder(secondScan)); + return readBuilders; + } + + private static Map readBuilders(StreamTableScan scan) { + Map readBuilders = new LinkedHashMap<>(); + readBuilders.put("db.table_a", readBuilder(scan)); + return readBuilders; + } + + private static ReadBuilder readBuilder(StreamTableScan scan) { + ReadBuilder readBuilder = mock(ReadBuilder.class); + when(readBuilder.newStreamScan()).thenReturn(scan); + return readBuilder; + } + + private static StreamTableScan newStreamScan(Long nextSnapshotId) { + StreamTableScan scan = mock(StreamTableScan.class); + TableScan.Plan plan = mock(TableScan.Plan.class); + when(plan.splits()).thenReturn(Collections.emptyList()); + when(scan.plan()).thenReturn(plan); + when(scan.checkpoint()).thenReturn(nextSnapshotId); + return scan; + } + + private static SourceSplitEnumerator.Context context() { + SourceSplitEnumerator.Context context = + mock(SourceSplitEnumerator.Context.class); + when(context.currentParallelism()).thenReturn(1); + when(context.registeredReaders()).thenReturn(new HashSet<>()); + return context; + } +} From d1701ee6e085c7cd1de00dda7efe29ac481d1511 Mon Sep 17 00:00:00 2001 From: M MUKTHANANDA REDDDY <88380922+Muktha9491@users.noreply.github.com> Date: Fri, 19 Jun 2026 19:21:19 +0530 Subject: [PATCH 032/375] [Feature][Connector-V2] Support Pulsar sink multi-table writes (#10670) --- docs/en/connectors/sink/Pulsar.md | 25 +- .../seatunnel/pulsar/sink/PulsarSink.java | 13 +- .../pulsar/sink/PulsarSinkFactory.java | 20 +- .../pulsar/sink/PulsarSinkWriter.java | 240 ++++++++++++++---- .../pulsar/sink/PulsarSinkFactoryTest.java | 91 +++++++ .../pulsar/sink/PulsarSinkWriterTest.java | 240 ++++++++++++++++++ 6 files changed, 578 insertions(+), 51 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactoryTest.java create mode 100644 seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkWriterTest.java diff --git a/docs/en/connectors/sink/Pulsar.md b/docs/en/connectors/sink/Pulsar.md index e67086c693dc..924007edfefe 100644 --- a/docs/en/connectors/sink/Pulsar.md +++ b/docs/en/connectors/sink/Pulsar.md @@ -28,7 +28,7 @@ Sink connector for Apache Pulsar. | Name | Type | Required | Default | Description | |----------------------|--------|----------|---------------------|------------------------------------------------------------------------------------------------------------------| -| topic | String | Yes | - | sink pulsar topic | +| topic | String | No | - | Sink Pulsar topic. Required for single-table writes and optional when records provide `SeaTunnelRow.tableId`. | | client.service-url | String | Yes | - | Service URL provider for Pulsar service. | | admin.service-url | String | Yes | - | The Pulsar service HTTP URL for the admin endpoint. | | auth.plugin-class | String | No | - | Name of the authentication plugin. | @@ -77,6 +77,15 @@ If you customize the delimiter, add the "field_delimiter" option. Customize the field delimiter for data format.The default field_delimiter is ','. +### topic [String] + +The default Pulsar topic used by the sink. + +For single-table pipelines, this option is required. +For multi-table pipelines, the sink will use `SeaTunnelRow.getTableId()` as the target topic when it is present, and fall back to `topic` only when the row does not carry a table id. + +If neither `SeaTunnelRow.getTableId()` nor `topic` is available, the sink fails fast with a configuration error. + ### semantics [Enum] Consistency semantics for writing to pulsar. @@ -171,6 +180,20 @@ sink { } ``` +### Multi-table + +> This example routes each row to the Pulsar topic carried in `SeaTunnelRow.tableId`. In this mode, `topic` can be omitted. + +```hocon +sink { + Pulsar { + client.service-url = "pulsar://localhost:6650" + admin.service-url = "http://localhost:8080" + plugin_output = "test" + } +} +``` + ## Changelog diff --git a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSink.java b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSink.java index ef5aeb0c57d6..0826feefcabf 100644 --- a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSink.java +++ b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSink.java @@ -23,6 +23,7 @@ import org.apache.seatunnel.api.sink.SeaTunnelSink; import org.apache.seatunnel.api.sink.SinkCommitter; import org.apache.seatunnel.api.sink.SinkWriter; +import org.apache.seatunnel.api.sink.SupportMultiTableSink; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; @@ -42,7 +43,11 @@ */ public class PulsarSink implements SeaTunnelSink< - SeaTunnelRow, PulsarSinkState, PulsarCommitInfo, PulsarAggregatedCommitInfo> { + SeaTunnelRow, + PulsarSinkState, + PulsarCommitInfo, + PulsarAggregatedCommitInfo>, + SupportMultiTableSink { private final SeaTunnelRowType seaTunnelRowType; private final PulsarClientConfig clientConfig; @@ -68,7 +73,11 @@ public PulsarSink(ReadonlyConfig readonlyConfig, CatalogTable catalogTable) { public SinkWriter createWriter( SinkWriter.Context context) { return new PulsarSinkWriter( - context, clientConfig, seaTunnelRowType, readonlyConfig, Collections.emptyList()); + context, + clientConfig, + seaTunnelRowType, + readonlyConfig, + Collections.emptyList()); } @Override diff --git a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactory.java b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactory.java index 6317c55f760f..5db5575c69f5 100644 --- a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactory.java +++ b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactory.java @@ -17,12 +17,15 @@ package org.apache.seatunnel.connectors.seatunnel.pulsar.sink; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableSink; import org.apache.seatunnel.api.table.factory.Factory; import org.apache.seatunnel.api.table.factory.TableSinkFactory; import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext; +import org.apache.seatunnel.common.exception.CommonErrorCode; import org.apache.seatunnel.connectors.seatunnel.pulsar.config.PulsarSinkOptions; +import org.apache.seatunnel.connectors.seatunnel.pulsar.exception.PulsarConnectorException; import com.google.auto.service.AutoService; @@ -36,11 +39,9 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .required( - PulsarSinkOptions.CLIENT_SERVICE_URL, - PulsarSinkOptions.ADMIN_SERVICE_URL, - PulsarSinkOptions.TOPIC) + .required(PulsarSinkOptions.CLIENT_SERVICE_URL, PulsarSinkOptions.ADMIN_SERVICE_URL) .optional( + PulsarSinkOptions.TOPIC, PulsarSinkOptions.FORMAT, PulsarSinkOptions.FIELD_DELIMITER, PulsarSinkOptions.MESSAGE_ROUTING_MODE, @@ -58,6 +59,17 @@ public OptionRule optionRule() { @Override public TableSink createSink(TableSinkFactoryContext context) { + validateSingleTableTopic(context); return () -> new PulsarSink(context.getOptions(), context.getCatalogTable()); } + + private void validateSingleTableTopic(TableSinkFactoryContext context) { + ReadonlyConfig options = context.getOptions(); + if (context.getCatalogTable() != null + && !options.getOptional(PulsarSinkOptions.TOPIC).isPresent()) { + throw new PulsarConnectorException( + CommonErrorCode.ILLEGAL_ARGUMENT, + "Topic must be configured for single-table Pulsar sink."); + } + } } diff --git a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkWriter.java b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkWriter.java index 26f35fab9995..22e3c694482e 100644 --- a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkWriter.java +++ b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkWriter.java @@ -21,7 +21,9 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.serialization.SerializationSchema; +import org.apache.seatunnel.api.sink.MultiTableResourceManager; import org.apache.seatunnel.api.sink.SinkWriter; +import org.apache.seatunnel.api.sink.SupportMultiTableSinkWriter; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; @@ -47,26 +49,42 @@ import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.transaction.TransactionImpl; +import lombok.extern.slf4j.Slf4j; + import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +@Slf4j public class PulsarSinkWriter - implements SinkWriter { + implements SinkWriter, + SupportMultiTableSinkWriter { + + @FunctionalInterface + interface ProducerCreator { + Producer create(String topic) throws PulsarClientException; + } - private Producer producer; + private final Map> producerMap = new ConcurrentHashMap<>(); + private final AtomicLong pendingMessages; + private final AtomicReference sendMessageException; + private final ReadonlyConfig pluginConfig; + private final MessageRoutingMode messageRoutingMode; + private final ProducerCreator producerCreator; private PulsarClient pulsarClient; private SerializationSchema serializationSchema; private SerializationSchema keySerializationSchema; - private TransactionImpl transaction; + private volatile TransactionImpl transaction; private int transactionTimeout; private PulsarSemantics pulsarSemantics; - private final AtomicLong pendingMessages; public PulsarSinkWriter( Context context, @@ -74,51 +92,104 @@ public PulsarSinkWriter( SeaTunnelRowType seaTunnelRowType, ReadonlyConfig pluginConfig, List pulsarStates) { - String topic = pluginConfig.get(PulsarSinkOptions.TOPIC); + this( + seaTunnelRowType, + pluginConfig, + pulsarStates, + PulsarConfigUtil.createClient( + clientConfig, pluginConfig.get(PulsarSinkOptions.SEMANTICS)), + null); + } + + PulsarSinkWriter( + SeaTunnelRowType seaTunnelRowType, + ReadonlyConfig pluginConfig, + List pulsarStates, + PulsarClient pulsarClient, + ProducerCreator producerCreator) { String format = pluginConfig.get(PulsarSinkOptions.FORMAT); String delimiter = pluginConfig.get(PulsarSinkOptions.FIELD_DELIMITER); this.transactionTimeout = pluginConfig.get(PulsarSinkOptions.TRANSACTION_TIMEOUT); this.pulsarSemantics = pluginConfig.get(PulsarSinkOptions.SEMANTICS); - MessageRoutingMode messageRoutingMode = - pluginConfig.get(PulsarSinkOptions.MESSAGE_ROUTING_MODE); + this.messageRoutingMode = pluginConfig.get(PulsarSinkOptions.MESSAGE_ROUTING_MODE); this.serializationSchema = createSerializationSchema(seaTunnelRowType, format, delimiter); List partitionKeyList = getPartitionKeyFields(pluginConfig, seaTunnelRowType); this.keySerializationSchema = createKeySerializationSchema(partitionKeyList, seaTunnelRowType); - this.pulsarClient = PulsarConfigUtil.createClient(clientConfig, pulsarSemantics); + this.pulsarClient = pulsarClient; + this.pluginConfig = pluginConfig; + this.producerCreator = + producerCreator != null + ? producerCreator + : topic -> + PulsarConfigUtil.createProducer( + this.pulsarClient, + topic, + pulsarSemantics, + pluginConfig, + messageRoutingMode); + this.pendingMessages = new AtomicLong(0); + this.sendMessageException = new AtomicReference<>(); if (PulsarSemantics.EXACTLY_ONCE == pulsarSemantics) { - try { - this.transaction = - (TransactionImpl) - PulsarConfigUtil.getTransaction(pulsarClient, transactionTimeout); - } catch (Exception e) { - throw new PulsarConnectorException( - PulsarConnectorErrorCode.CREATE_TRANSACTION_FAILED, - "Pulsar transaction create fail."); - } + this.transaction = createTransaction(); + } + } + + String resolveTopic(SeaTunnelRow row) { + String tableId = row.getTableId(); + if (tableId != null && !tableId.isEmpty()) { + return tableId; } + + String topic = pluginConfig.get(PulsarSinkOptions.TOPIC); + if (topic == null || topic.isEmpty()) { + throw new PulsarConnectorException( + CommonErrorCode.ILLEGAL_ARGUMENT, + "Topic must be configured when SeaTunnelRow.getTableId() is null"); + } + + return topic; + } + + Producer getOrCreateProducer(String topic) { + Producer existing = producerMap.get(topic); + if (existing != null) { + return existing; + } + try { - this.producer = - PulsarConfigUtil.createProducer( - pulsarClient, topic, pulsarSemantics, pluginConfig, messageRoutingMode); + Producer producer = producerCreator.create(topic); + + producerMap.put(topic, producer); + return producer; + } catch (PulsarClientException e) { throw new PulsarConnectorException( PulsarConnectorErrorCode.CREATE_PRODUCER_FAILED, - "Pulsar Producer create fail."); + "Failed to create Pulsar producer for topic: " + topic, + e); } - this.pendingMessages = new AtomicLong(0); } @Override public void write(SeaTunnelRow element) throws IOException { + checkSendException(); + + String topic = resolveTopic(element); byte[] message = serializationSchema.serialize(element); byte[] key = null; if (keySerializationSchema != null) { key = keySerializationSchema.serialize(element); } + + Producer topicProducer = getOrCreateProducer(topic); + TypedMessageBuilder typedMessageBuilder = - PulsarConfigUtil.createTypedMessageBuilder(producer, transaction); + PulsarConfigUtil.createTypedMessageBuilder( + topicProducer, + PulsarSemantics.EXACTLY_ONCE == pulsarSemantics ? transaction : null); + if (key != null) { typedMessageBuilder.keyBytes(key); } @@ -132,9 +203,8 @@ public void write(SeaTunnelRow element) throws IOException { (id, ex) -> { pendingMessages.decrementAndGet(); if (ex != null) { - throw new PulsarConnectorException( - PulsarConnectorErrorCode.SEND_MESSAGE_FAILED, - "send message failed"); + log.error("Failed to send message to topic {}", topic, ex); + sendMessageException.compareAndSet(null, ex); } }); } @@ -142,7 +212,8 @@ public void write(SeaTunnelRow element) throws IOException { @Override public Optional prepareCommit() throws IOException { - if (PulsarSemantics.EXACTLY_ONCE == pulsarSemantics) { + checkSendException(); + if (PulsarSemantics.EXACTLY_ONCE == pulsarSemantics && transaction != null) { PulsarCommitInfo pulsarCommitInfo = new PulsarCommitInfo(this.transaction.getTxnID()); return Optional.of(pulsarCommitInfo); } else { @@ -153,40 +224,52 @@ public Optional prepareCommit() throws IOException { @Override public List snapshotState(long checkpointId) throws IOException { if (PulsarSemantics.NON != pulsarSemantics) { - /** flush pending messages */ - producer.flush(); - while (pendingMessages.longValue() > 0) { - producer.flush(); - } + flushPendingMessages(); + checkSendException(); } if (PulsarSemantics.EXACTLY_ONCE == pulsarSemantics) { List pulsarSinkStates = Lists.newArrayList(new PulsarSinkState(this.transaction.getTxnID())); - try { - this.transaction = - (TransactionImpl) - PulsarConfigUtil.getTransaction(pulsarClient, transactionTimeout); - } catch (Exception e) { - throw new PulsarConnectorException( - PulsarConnectorErrorCode.CREATE_TRANSACTION_FAILED, - "Pulsar transaction create fail."); - } + this.transaction = createTransaction(); return pulsarSinkStates; } + return Collections.emptyList(); } @Override public void abortPrepare() { - if (PulsarSemantics.EXACTLY_ONCE == pulsarSemantics) { + if (PulsarSemantics.EXACTLY_ONCE == pulsarSemantics && transaction != null) { transaction.abort(); } } @Override public void close() throws IOException { - producer.close(); - pulsarClient.close(); + Throwable closeFailure = null; + for (Producer producer : producerMap.values()) { + try { + producer.close(); + } catch (Throwable throwable) { + closeFailure = appendSuppressed(closeFailure, throwable); + } + } + if (pulsarClient != null) { + try { + pulsarClient.close(); + } catch (Throwable throwable) { + closeFailure = appendSuppressed(closeFailure, throwable); + } + } + + Throwable sendFailure = sendMessageException.get(); + if (sendFailure != null) { + closeFailure = appendSuppressed(closeFailure, buildSendFailureException(sendFailure)); + } + + if (closeFailure != null) { + rethrowCloseFailure(closeFailure); + } } private SerializationSchema createSerializationSchema( @@ -251,4 +334,73 @@ private List getPartitionKeyFields( } return Collections.emptyList(); } + + private TransactionImpl createTransaction() { + try { + return (TransactionImpl) + PulsarConfigUtil.getTransaction(pulsarClient, transactionTimeout); + } catch (Exception e) { + throw new PulsarConnectorException( + PulsarConnectorErrorCode.CREATE_TRANSACTION_FAILED, + "Pulsar transaction create fail.", + e); + } + } + + private void flushPendingMessages() throws IOException { + for (Producer producer : producerMap.values()) { + producer.flush(); + } + + while (pendingMessages.longValue() > 0) { + checkSendException(); + for (Producer producer : producerMap.values()) { + producer.flush(); + } + } + } + + private void checkSendException() { + Throwable throwable = sendMessageException.get(); + if (throwable != null) { + throw buildSendFailureException(throwable); + } + } + + private PulsarConnectorException buildSendFailureException(Throwable throwable) { + return new PulsarConnectorException( + PulsarConnectorErrorCode.SEND_MESSAGE_FAILED, + "Send message failed, please check previous error log for details.", + throwable); + } + + private Throwable appendSuppressed(Throwable existingFailure, Throwable newFailure) { + if (existingFailure == null) { + return newFailure; + } + existingFailure.addSuppressed(newFailure); + return existingFailure; + } + + private void rethrowCloseFailure(Throwable throwable) throws IOException { + if (throwable instanceof IOException) { + throw (IOException) throwable; + } + if (throwable instanceof RuntimeException) { + throw (RuntimeException) throwable; + } + throw new IOException("Failed to close Pulsar sink writer.", throwable); + } + + @Override + public MultiTableResourceManager initMultiTableResourceManager( + int tableSize, int queueSize) { + return null; + } + + @Override + public void setMultiTableResourceManager( + MultiTableResourceManager multiTableResourceManager, int queueIndex) { + // Pulsar sink does not require shared resources across tables + } } diff --git a/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactoryTest.java b/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactoryTest.java new file mode 100644 index 000000000000..69ba2888f2c1 --- /dev/null +++ b/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactoryTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.pulsar.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.Column; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext; +import org.apache.seatunnel.api.table.type.BasicType; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class PulsarSinkFactoryTest { + + @Test + public void testCreateSinkRequiresTopicForSingleTable() { + PulsarSinkFactory factory = new PulsarSinkFactory(); + + assertThrows( + IllegalArgumentException.class, + () -> + factory.createSink( + new TableSinkFactoryContext( + getCatalogTable(), config(), getClass().getClassLoader()))); + } + + @Test + public void testCreateSinkAllowsMissingTopicForMultiTable() { + PulsarSinkFactory factory = new PulsarSinkFactory(); + + assertDoesNotThrow( + () -> + factory.createSink( + new TableSinkFactoryContext( + null, config(), getClass().getClassLoader()))); + } + + private ReadonlyConfig config() { + Map options = new HashMap<>(); + options.put("client.service-url", "pulsar://localhost:6650"); + options.put("admin.service-url", "http://localhost:8080"); + options.put("format", "json"); + options.put("field_delimiter", ","); + options.put("semantics", "NON"); + options.put("message.routing.mode", "RoundRobinPartition"); + return ReadonlyConfig.fromMap(options); + } + + private CatalogTable getCatalogTable() { + List columns = new ArrayList<>(); + columns.add( + PhysicalColumn.builder() + .name("id") + .dataType(BasicType.INT_TYPE) + .nullable(true) + .build()); + TableSchema tableSchema = TableSchema.builder().columns(columns).build(); + return CatalogTable.of( + TableIdentifier.of("default", "default", "pulsar_table"), + tableSchema, + new HashMap<>(), + new ArrayList<>(), + "Pulsar table"); + } +} diff --git a/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkWriterTest.java b/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkWriterTest.java new file mode 100644 index 000000000000..e26dd9b6d063 --- /dev/null +++ b/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkWriterTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.pulsar.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.pulsar.exception.PulsarConnectorException; + +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PulsarSinkWriterTest { + + private PulsarSinkWriter createWriter(ReadonlyConfig config) { + return new PulsarSinkWriter( + new SeaTunnelRowType(new String[] {}, new SeaTunnelDataType[] {}), + config, + java.util.Collections.emptyList(), + null, + topic -> createProducerProxy()); + } + + @Test + public void testResolveTopicWithTableId() { + SeaTunnelRow row = new SeaTunnelRow(new Object[] {}); + row.setTableId("persistent://tenant/ns/topic1"); + + Map configMap = new HashMap<>(); + configMap.put("topic", "fallback-topic"); + configMap.put("format", "json"); + configMap.put("field_delimiter", ","); + configMap.put("transaction_timeout", 1000); + configMap.put("semantics", "NON"); + configMap.put("message.routing.mode", "RoundRobinPartition"); + + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + PulsarSinkWriter writer = createWriter(config); + + String topic = writer.resolveTopic(row); + + assertEquals("persistent://tenant/ns/topic1", topic); + } + + @Test + public void testResolveTopicWithoutTableId() { + SeaTunnelRow row = new SeaTunnelRow(new Object[] {}); + // no tableId set + + Map configMap = new HashMap<>(); + configMap.put("topic", "fallback-topic"); + configMap.put("format", "json"); + configMap.put("field_delimiter", ","); + configMap.put("transaction_timeout", 1000); + configMap.put("semantics", "NON"); + configMap.put("message.routing.mode", "RoundRobinPartition"); + + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + PulsarSinkWriter writer = createWriter(config); + + String topic = writer.resolveTopic(row); + + assertEquals("fallback-topic", topic); + } + + @Test + public void testResolveTopicWithoutTableIdAndWithoutTopic() { + SeaTunnelRow row = new SeaTunnelRow(new Object[] {}); + + Map configMap = new HashMap<>(); + configMap.put("format", "json"); + configMap.put("field_delimiter", ","); + configMap.put("transaction_timeout", 1000); + configMap.put("semantics", "NON"); + configMap.put("message.routing.mode", "RoundRobinPartition"); + + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + PulsarSinkWriter writer = createWriter(config); + + assertThrows(IllegalArgumentException.class, () -> writer.resolveTopic(row)); + } + + @Test + public void testGetOrCreateProducerCachesProducerByTopic() { + Map configMap = new HashMap<>(); + configMap.put("format", "json"); + configMap.put("field_delimiter", ","); + configMap.put("transaction_timeout", 1000); + configMap.put("semantics", "NON"); + configMap.put("message.routing.mode", "RoundRobinPartition"); + + AtomicInteger createCount = new AtomicInteger(); + PulsarSinkWriter writer = + new PulsarSinkWriter( + new SeaTunnelRowType(new String[] {}, new SeaTunnelDataType[] {}), + ReadonlyConfig.fromMap(configMap), + java.util.Collections.emptyList(), + null, + topic -> { + createCount.incrementAndGet(); + return createProducerProxy(); + }); + + Producer first = writer.getOrCreateProducer("topic-a"); + Producer second = writer.getOrCreateProducer("topic-a"); + Producer third = writer.getOrCreateProducer("topic-b"); + + assertSame(first, second); + assertTrue(first != third); + assertEquals(2, createCount.get()); + } + + @Test + public void testCloseReleasesResourcesWhenAsyncSendFailed() throws Exception { + Map configMap = new HashMap<>(); + configMap.put("format", "json"); + configMap.put("field_delimiter", ","); + configMap.put("transaction_timeout", 1000); + configMap.put("semantics", "NON"); + configMap.put("message.routing.mode", "RoundRobinPartition"); + + AtomicBoolean producerClosed = new AtomicBoolean(false); + AtomicBoolean clientClosed = new AtomicBoolean(false); + + PulsarSinkWriter writer = + new PulsarSinkWriter( + new SeaTunnelRowType(new String[] {}, new SeaTunnelDataType[] {}), + ReadonlyConfig.fromMap(configMap), + java.util.Collections.emptyList(), + createPulsarClientProxy(clientClosed), + topic -> createProducerProxy(producerClosed)); + writer.getOrCreateProducer("topic-a"); + + Field sendFailureField = PulsarSinkWriter.class.getDeclaredField("sendMessageException"); + sendFailureField.setAccessible(true); + @SuppressWarnings("unchecked") + AtomicReference sendFailure = + (AtomicReference) sendFailureField.get(writer); + sendFailure.set(new RuntimeException("send failed")); + + assertThrows(PulsarConnectorException.class, writer::close); + assertTrue(producerClosed.get()); + assertTrue(clientClosed.get()); + } + + @SuppressWarnings("unchecked") + private static Producer createProducerProxy() { + return createProducerProxy(new AtomicBoolean(false)); + } + + @SuppressWarnings("unchecked") + private static Producer createProducerProxy(AtomicBoolean closed) { + return (Producer) + Proxy.newProxyInstance( + Producer.class.getClassLoader(), + new Class[] {Producer.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closed.set(true); + return null; + } + Class returnType = method.getReturnType(); + if (returnType.equals(boolean.class)) { + return false; + } + if (returnType.equals(int.class)) { + return 0; + } + if (returnType.equals(long.class)) { + return 0L; + } + if (returnType.equals(float.class)) { + return 0F; + } + if (returnType.equals(double.class)) { + return 0D; + } + return null; + }); + } + + @SuppressWarnings("unchecked") + private static PulsarClient createPulsarClientProxy(AtomicBoolean closed) { + return (PulsarClient) + Proxy.newProxyInstance( + PulsarClient.class.getClassLoader(), + new Class[] {PulsarClient.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closed.set(true); + return null; + } + Class returnType = method.getReturnType(); + if (returnType.equals(boolean.class)) { + return false; + } + if (returnType.equals(int.class)) { + return 0; + } + if (returnType.equals(long.class)) { + return 0L; + } + return null; + }); + } +} From 77b3fd4d734919db76aff9abceaca264072126ac Mon Sep 17 00:00:00 2001 From: Jast Date: Fri, 19 Jun 2026 22:13:25 +0800 Subject: [PATCH 033/375] [Docs] Add zh DynamicCompile Scala example (#11130) --- docs/zh/transforms/dynamic-compile.md | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/zh/transforms/dynamic-compile.md b/docs/zh/transforms/dynamic-compile.md index 56ec41992821..7f5ca99d137a 100644 --- a/docs/zh/transforms/dynamic-compile.md +++ b/docs/zh/transforms/dynamic-compile.md @@ -33,6 +33,8 @@ DynamicCompile 转换插件提供一种可编程的方式来处理行，允许 Java中的某些语法可能不受支持，请参阅https://github.com/janino-compiler/janino GROOVY，JAVA，SCALA(目前支持 Zeta) +**注意**：SCALA 支持使用 Scala REPL 进行动态编译，需要编写符合 Scala 语法的代码。 + ### compile_pattern [Enum] SOURCE_CODE,ABSOLUTE_PATH @@ -220,6 +222,49 @@ transform { | Kin Dom | 30 | 123 | JAVA | | Joy Dom | 30 | 123 | JAVA | +- 使用 Scala + +下面的 Scala 示例演示如何新增 `compile_language` 字段。 + +```hacon +transform { + DynamicCompile { + plugin_input = "fake" + plugin_output = "scala_out" + compile_language="SCALA" + compile_pattern="SOURCE_CODE" + source_code=""" + import org.apache.seatunnel.api.table.catalog.Column + import org.apache.seatunnel.api.table.catalog.CatalogTable + import org.apache.seatunnel.api.table.catalog.PhysicalColumn + import org.apache.seatunnel.api.table.`type`.SeaTunnelRowAccessor + import org.apache.seatunnel.api.table.`type`.BasicType + import java.util.ArrayList + + class ScalaDemo { + def getInlineOutputColumns(inputCatalogTable: CatalogTable): Array[Column] = { + val columns = new ArrayList[Column]() + val destColumn = PhysicalColumn.of( + "compile_language", + BasicType.STRING_TYPE, + 10L, + true, + "", + "" + ) + columns.add(destColumn) + columns.toArray(new Array[Column](0)) + } + + def getInlineOutputFieldValues(inputRow: SeaTunnelRowAccessor): Array[Object] = { + Array[Object]("SCALA") + } + } + """ + } +} +``` + 更多复杂例子可以参考 https://github.com/apache/seatunnel/tree/dev/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/dynamic_compile/conf From d5408164da80cf7e4c1fd1a1740e78e63afd5394 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 19 Jun 2026 22:14:20 +0800 Subject: [PATCH 034/375] [Docs] Polish architecture overview diagrams for current docs (#11124) --- .../architecture/api-design/catalog-table.md | 38 +-- .../api-design/sink-architecture.md | 91 +++--- .../api-design/source-architecture.md | 59 ++-- .../api-design/translation-layer.md | 51 ++-- docs/en/architecture/engine/dag-execution.md | 230 ++++++-------- .../engine/engine-architecture.md | 248 ++++++--------- .../engine/resource-management.md | 94 ++---- .../fault-tolerance/checkpoint-mechanism.md | 112 ++----- .../fault-tolerance/exactly-once.md | 208 ++++++------- docs/en/architecture/features/multi-table.md | 86 ++---- docs/en/architecture/overview.md | 287 ++++++++---------- docs/en/engines/overview.md | 39 ++- docs/en/introduction/how-it-works.md | 89 +++--- ...multi-table-transform-and-join-boundary.md | 57 ++-- .../architecture/api-design/catalog-table.md | 38 +-- .../api-design/sink-architecture.md | 70 ++--- .../api-design/source-architecture.md | 59 ++-- .../api-design/translation-layer.md | 51 ++-- docs/zh/architecture/engine/dag-execution.md | 224 ++++++-------- .../engine/engine-architecture.md | 229 +++++--------- .../engine/resource-management.md | 68 ++--- .../fault-tolerance/checkpoint-mechanism.md | 98 ++---- .../fault-tolerance/exactly-once.md | 75 ++--- docs/zh/architecture/features/multi-table.md | 83 +++-- docs/zh/architecture/overview.md | 287 ++++++++---------- docs/zh/engines/overview.md | 39 ++- docs/zh/introduction/how-it-works.md | 89 +++--- ...multi-table-transform-and-join-boundary.md | 57 ++-- 28 files changed, 1289 insertions(+), 1867 deletions(-) diff --git a/docs/en/architecture/api-design/catalog-table.md b/docs/en/architecture/api-design/catalog-table.md index b9822865f8ee..1a15c2476133 100644 --- a/docs/en/architecture/api-design/catalog-table.md +++ b/docs/en/architecture/api-design/catalog-table.md @@ -200,29 +200,21 @@ TableSchema schema = TableSchema.builder() ### 4.1 Source → Transform → Sink Flow -``` -┌──────────────┐ -│ Source │ -│ │ -│ produces │ -│ CatalogTable │ -└──────┬───────┘ - │ - ▼ (Input Schema) -┌──────────────┐ -│ Transform │ -│ │ -│ modifies │ -│ CatalogTable │ -└──────┬───────┘ - │ - ▼ (Output Schema) -┌──────────────┐ -│ Sink │ -│ │ -│ validates │ -│ CatalogTable │ -└──────────────┘ +```mermaid +flowchart LR + source["Source
Produce CatalogTable"] + transform["Transform
Update CatalogTable"] + sink["Sink
Validate CatalogTable"] + + source -- "Input schema" --> transform + transform -- "Output schema" --> sink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source,sink layerBlue; + class transform layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 4.2 Source Schema Production diff --git a/docs/en/architecture/api-design/sink-architecture.md b/docs/en/architecture/api-design/sink-architecture.md index 5367ba8e1e50..7a49559b7547 100644 --- a/docs/en/architecture/api-design/sink-architecture.md +++ b/docs/en/architecture/api-design/sink-architecture.md @@ -39,52 +39,34 @@ SeaTunnel's Sink API aims to: ### 2.1 Overall Architecture -``` -┌────────────────────────────────────────────────────────────────┐ -│ TaskExecutionService (Worker Side) │ -│ │ -│ ┌──────────────────────────────────────────────────────┐ │ -│ │ SinkWriter │ │ -│ │ │ │ -│ │ • Receive records from upstream │ │ -│ │ • Buffer and write data │ │ -│ │ • Produce commitInfo at checkpoint boundary │ │ -│ │ • Snapshot writer state │ │ -│ │ • Cleanup/rollback on failure (engine-dependent) │ │ -│ └──────────────────────────────────────────────────────┘ │ -│ │ │ -└────────────────────────────┼─────────────────────────────────────┘ - │ (CommitInfo) - ▼ -┌────────────────────────────────────────────────────────────────┐ -│ Coordinator Side (control plane, engine-dependent) │ -│ │ -│ ┌──────────────────────────────────────────────────────┐ │ -│ │ SinkCommitter (Optional) │ │ -│ │ │ │ -│ │ • Receive commit infos from multiple writers │ │ -│ │ • Commit each writer's changes independently │ │ -│ │ • Retry failed commits │ │ -│ │ • Must be idempotent │ │ -│ └──────────────────────────────────────────────────────┘ │ -│ │ │ -│ │ (Optional: AggregatedCommitInfo) │ -│ ▼ │ -│ ┌──────────────────────────────────────────────────────┐ │ -│ │ SinkAggregatedCommitter │ │ -│ │ (Optional) │ │ -│ │ │ │ -│ │ • Aggregate commit infos from all writers │ │ -│ │ • Perform single global commit operation │ │ -│ │ • Single-threaded, global coordinator │ │ -│ └──────────────────────────────────────────────────────┘ │ -│ │ -└──────────────────────────────────────────────────────────────────┘ - │ - ▼ - External Data Sink - (Database / File / Message Queue) +```mermaid +flowchart LR + subgraph worker["TaskExecutionService (Worker Side)"] + writer["SinkWriter<IN, CommitInfoT, StateT>
Receive upstream records
Buffer and write data
Emit CommitInfo at checkpoint boundary
Snapshot writer state"] + committer["SinkCommitter<CommitInfoT> (Optional)
Created by createCommitter()
Triggered after checkpoint success
Commit each writer change independently"] + end + + subgraph coordinator["Coordinator Side (aggregated commit only)"] + aggregatedTask["SinkAggregatedCommitterTask (Optional)
Collect commit infos from writers
Run single coordinator-side commit"] + aggregated["SinkAggregatedCommitter<CommitInfoT, AggregatedCommitInfoT> (Optional)
Aggregate writer commit infos
Perform one global commit"] + end + + sink["External Data Sink
Database / File / Message Queue"] + + writer -- "worker-local commit path" --> committer + committer --> sink + writer -. "aggregated commit path" .-> aggregatedTask + aggregatedTask --> aggregated + aggregated --> sink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class worker,coordinator layerBlue; + class writer,committer layerCyan; + class aggregatedTask,aggregated,sink layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.2 Core Components @@ -158,9 +140,9 @@ public interface SeaTunnelSink ``` **Key Design Points**: -- Three-tier commit architecture: Writer → Committer → AggregatedCommitter -- Committer and AggregatedCommitter are optional (depends on sink requirements) - Writer is always required (performs actual data writing) +- SinkCommitter and SinkAggregatedCommitter are optional commit strategies +- In SeaTunnel Engine, SinkCommitter runs on the worker side, while aggregated commit uses a coordinator-side `SinkAggregatedCommitterTask` ### 2.3 Interaction Flow @@ -643,15 +625,15 @@ public class HiveAggregatedCommitter - Sink doesn't support transactions - Ultra-low latency required -#### Three-Tier vs Two-Tier Commit +#### Per-Writer Commit vs Aggregated Commit -**Two-Tier (Writer → Committer)**: +**Per-Writer Commit (Writer + SinkCommitter)**: - Each writer's commit handled independently -- Parallel commit operations +- Commit callback runs on the writer side in SeaTunnel Engine - Suitable for most sinks -**Three-Tier (Writer → Committer → AggregatedCommitter)**: -- All writers' commits aggregated into single operation +**Aggregated Commit (Writer + SinkAggregatedCommitterTask)**: +- Writers forward commit infos for one coordinator-side commit - Single global commit point - Required for table-level transactions (Hive, Iceberg) @@ -810,10 +792,9 @@ public class TransactionalSink implements SeaTunnelSink<...> { Optional createCommitter() { return Optional.of(new Committer()); } } -// Table sink: Writer + Committer + AggregatedCommitter +// Table sink: Writer + AggregatedCommitter (global commit) public class TableSink implements SeaTunnelSink<...> { SinkWriter createWriter(...) { return new TableWriter(); } - Optional createCommitter() { return Optional.of(new Committer()); } Optional createAggregatedCommitter() { return Optional.of(new AggregatedCommitter()); } diff --git a/docs/en/architecture/api-design/source-architecture.md b/docs/en/architecture/api-design/source-architecture.md index 91d7971e2b96..7ae73b1ec5bb 100644 --- a/docs/en/architecture/api-design/source-architecture.md +++ b/docs/en/architecture/api-design/source-architecture.md @@ -39,44 +39,27 @@ SeaTunnel's Source API aims to: ### 2.1 Overall Architecture -``` -┌──────────────────────────────────────────────────────────────┐ -│ Coordinator (master/coordinator side) │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ SourceSplitEnumerator │ │ -│ │ │ │ -│ │ • Discover/generate splits in run() (impl-defined) │ │ -│ │ • Assign splits to readers │ │ -│ │ • Handle reader registration │ │ -│ │ • Handle split requests │ │ -│ │ • Reclaim splits from failed readers │ │ -│ │ • Snapshot enumerator state │ │ -│ │ • Send/receive custom events │ │ -│ └────────────────────────────────────────────────────┘ │ -│ │ │ -└────────────────────────────┼───────────────────────────────────┘ - │ (Split Assignment) - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ TaskExecutionService (Worker Side) │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ SourceReader │ │ -│ │ │ │ -│ │ • Receive assigned splits │ │ -│ │ • Read data from splits │ │ -│ │ • Emit records downstream │ │ -│ │ • Snapshot reader state (split progress) │ │ -│ │ • Handle split completion │ │ -│ │ • Send/receive custom events │ │ -│ └────────────────────────────────────────────────────┘ │ -│ │ │ -└────────────────────────────┼───────────────────────────────────┘ - │ - ▼ - SeaTunnelRow - (to Transform/Sink) +```mermaid +flowchart TD + subgraph coordinator["Coordinator (master / coordinator side)"] + enumerator["SourceSplitEnumerator<SplitT, StateT>
Discover or generate splits
Assign splits to readers
Handle reader registration and split requests
Snapshot enumerator state"] + end + + subgraph worker["TaskExecutionService (Worker Side)"] + reader["SourceReader<T, SplitT>
Receive assigned splits
Read data from splits
Emit records downstream
Snapshot reader progress"] + end + + row["SeaTunnelRow
To Transform / Sink"] + + enumerator -- "Split assignment" --> reader + reader --> row + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + + class coordinator,worker layerBlue; + class enumerator,reader,row layerCyan; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.2 Core Components diff --git a/docs/en/architecture/api-design/translation-layer.md b/docs/en/architecture/api-design/translation-layer.md index 603516d65df6..02a45b979a44 100644 --- a/docs/en/architecture/api-design/translation-layer.md +++ b/docs/en/architecture/api-design/translation-layer.md @@ -29,29 +29,34 @@ SeaTunnel's translation layer aims to: ### 1.3 Architecture Overview -``` -┌──────────────────────────────────────────────────────────────┐ -│ SeaTunnel API Layer │ -│ (Engine-Independent Connector Interface) │ -│ │ -│ SeaTunnelSource SeaTunnelSink SeaTunnelTransform │ -└──────────────────────────────────────────────────────────────┘ - │ - │ Translation Layer - ┌─────────────┼─────────────┐ - ▼ ▼ ▼ -┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ -│ Flink Adapter │ │ Spark Adapter │ │ Zeta (Native) │ -│ │ │ │ │ │ -│ FlinkSource │ │ SparkSource │ │ Direct │ -│ FlinkSink │ │ SparkSink │ │ Execution │ -└──────────────────┘ └──────────────────┘ └──────────────────┘ - │ │ │ - ▼ ▼ ▼ -┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ -│ Apache Flink │ │ Apache Spark │ │ SeaTunnel Engine │ -│ Runtime │ │ Runtime │ │ (Zeta) │ -└──────────────────┘ └──────────────────┘ └──────────────────┘ +```mermaid +flowchart TB + api["SeaTunnel API Layer
Engine-independent connector interfaces
SeaTunnelSource / SeaTunnelSink / SeaTunnelTransform"] + + flinkAdapter["Flink Adapter
FlinkSource / FlinkSink"] + sparkAdapter["Spark Adapter
SparkSource / SparkSink"] + zetaAdapter["Zeta (Native)
Direct execution"] + + flinkRuntime["Apache Flink Runtime"] + sparkRuntime["Apache Spark Runtime"] + zetaRuntime["SeaTunnel Engine (Zeta)"] + + api --> flinkAdapter + api --> sparkAdapter + api --> zetaAdapter + + flinkAdapter --> flinkRuntime + sparkAdapter --> sparkRuntime + zetaAdapter --> zetaRuntime + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class api layerBlue; + class flinkAdapter,sparkAdapter,zetaAdapter layerCyan; + class flinkRuntime,sparkRuntime,zetaRuntime layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 1.4 Recommended Reading Path diff --git a/docs/en/architecture/engine/dag-execution.md b/docs/en/architecture/engine/dag-execution.md index cfd74e42b660..ef7754556845 100644 --- a/docs/en/architecture/engine/dag-execution.md +++ b/docs/en/architecture/engine/dag-execution.md @@ -29,47 +29,25 @@ SeaTunnel's DAG execution model aims to: ### 1.3 Execution Model Overview -``` -User Config (HOCON) - │ - ▼ -┌─────────────────────┐ -│ LogicalDag │ Logical Plan (What to do) -│ • LogicalVertex │ - Source/Transform/Sink actions -│ • LogicalEdge │ - Data dependencies -│ • Parallelism │ - Logical parallelism -└─────────────────────┘ - │ (Plan Generation) - ▼ -┌─────────────────────┐ -│ PhysicalPlan │ Physical Plan (How to execute) -│ • SubPlan[] │ - Multiple pipelines -│ • Resources │ - Resource requirements -│ • Scheduling │ - Deployment strategy -└─────────────────────┘ - │ (Pipeline Split) - ▼ -┌─────────────────────┐ -│ SubPlan (Pipeline) │ Independent Execution Unit -│ • PhysicalVertex[] │ - Parallel task instances -│ • CheckpointCoord │ - Independent checkpointing -│ • PipelineLocation │ - Unique identifier -└─────────────────────┘ - │ (Task Deployment) - ▼ -┌─────────────────────┐ -│ PhysicalVertex │ Deployed Task Group -│ • TaskGroup │ - Co-located tasks (fusion) -│ • SlotProfile │ - Assigned resource slot -│ • ExecutionState │ - Running state -└─────────────────────┘ - │ (Execution) - ▼ -┌─────────────────────┐ -│ SeaTunnelTask │ Actual Execution -│ • Source/Transform │ - Data processing -│ • /Sink Logic │ - State management -└─────────────────────┘ +```mermaid +flowchart TD + config["User Config
HOCON"] + logical["LogicalDag
LogicalVertex / LogicalEdge
Logical parallelism"] + physical["PhysicalPlan
SubPlan list
Resource requirements
Scheduling strategy"] + pipeline["SubPlan (Pipeline)
Independent execution unit
PhysicalVertex list
CheckpointCoordinator"] + vertex["PhysicalVertex
TaskGroup
SlotProfile
ExecutionState"] + task["SeaTunnelTask
Actual Source / Transform / Sink execution"] + + config --> logical --> physical --> pipeline --> vertex --> task + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class config,logical layerBlue; + class physical,pipeline layerCyan; + class vertex,task layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## 2. LogicalDag: User Intent @@ -204,14 +182,19 @@ sink { ``` Generated LogicalDag: -``` -Vertex 1 (JDBC Source, parallelism=4) - │ - ▼ -Vertex 2 (SQL Transform, parallelism=4) - │ - ▼ -Vertex 3 (Elasticsearch Sink, parallelism=4) +```mermaid +flowchart TD + v1["Vertex 1
JDBC Source
parallelism = 4"] + v2["Vertex 2
SQL Transform
parallelism = 4"] + v3["Vertex 3
Elasticsearch Sink
parallelism = 4"] + v1 --> v2 --> v3 + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class v1,v3 layerBlue; + class v2 layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## 3. PhysicalPlan: Execution Strategy @@ -254,9 +237,9 @@ sink { Elasticsearch { } } ``` Generated: **1 Pipeline** -``` -Pipeline 1: [JDBC Source] → [SQL Transform] → [Elasticsearch Sink] -``` +Generated result: + +- `Pipeline 1`: `JDBC Source → SQL Transform → Elasticsearch Sink` **Example 2: Multiple Sources**: ```hocon @@ -275,10 +258,10 @@ sink { ``` Generated: **2 Pipelines** -``` -Pipeline 1: [JDBC Source] → [SQL Transform] → [Elasticsearch Sink] -Pipeline 2: [Kafka Source] → [SQL Transform] → [Elasticsearch Sink] -``` +Generated result: + +- `Pipeline 1`: `JDBC Source → SQL Transform → Elasticsearch Sink` +- `Pipeline 2`: `Kafka Source → SQL Transform → Elasticsearch Sink` **Example 3: Multiple Sinks**: ```hocon @@ -293,8 +276,19 @@ sink { ``` Generated: **1 Pipeline** -``` -Pipeline 1: [MySQL-CDC Source] → ([Elasticsearch Sink], [JDBC Sink]) +Generated result: + +```mermaid +flowchart LR + source["MySQL-CDC Source"] --> elastic["

Elasticsearch
"] + source --> jdbc["
JDBC
"] + + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source layerCyan; + class elastic,jdbc layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 3.3 PhysicalPlan Generation @@ -343,36 +337,24 @@ public class SubPlan { Each LogicalVertex with parallelism N generates N PhysicalVertices. **Example**: -``` -LogicalVertex: JDBC Source (parallelism = 4) - ↓ -PhysicalVertices: - - PhysicalVertex (subtask 0, slot 1) - - PhysicalVertex (subtask 1, slot 2) - - PhysicalVertex (subtask 2, slot 3) - - PhysicalVertex (subtask 3, slot 4) -``` +| Logical vertex | Generated physical vertices | +|----------------|-----------------------------| +| `JDBC Source (parallelism = 4)` | `PhysicalVertex` subtask `0` on slot `1`, subtask `1` on slot `2`, subtask `2` on slot `3`, subtask `3` on slot `4` | ### 4.3 Coordinator Vertices Special vertices for coordination tasks: -- **SourceSplitEnumerator**: Runs on master, assigns splits to readers -- **SinkCommitter**: Runs on master, coordinates commits -- **SinkAggregatedCommitter**: Runs on master, global commit coordination +- **SourceSplitEnumerator**: Usually runs as a single coordination instance that assigns splits to readers (deployment is engine-specific) +- **SinkAggregatedCommitter**: When a sink provides an aggregated committer, it usually runs as a single coordination instance for global commit orchestration (deployment is engine-specific) + +Note: `SinkCommitter` depends on the execution engine and does not necessarily appear as an independent coordinator vertex. In SeaTunnel Engine, for example, the committer can be triggered inside the sink task's checkpoint callback. **Example**: -``` -SubPlan for JDBC → Transform → Elasticsearch: - physicalVertexList: - - JdbcSourceTask (4 instances) - - TransformTask (4 instances) - - ElasticsearchSinkTask (4 instances) - - coordinatorVertexList: - - JdbcSourceSplitEnumerator (1 instance, master) - - ElasticsearchSinkCommitter (1 instance, master) -``` +| Runtime scope | Instances | +|---------------|-----------| +| `physicalVertexList` | `JdbcSourceTask × 4`, `TransformTask × 4`, `ElasticsearchSinkTask × 4` | +| `coordinatorVertexList` | `JdbcSourceSplitEnumerator × 1`, plus `ElasticsearchSinkAggregatedCommitter × 1` (optional) | ### 4.4 Independent Checkpointing @@ -385,15 +367,10 @@ Each pipeline has its own `CheckpointCoordinator`: - Simpler barrier alignment **Example**: -``` -Pipeline 1 (JDBC → ES): - CheckpointCoordinator triggers every 60s - Manages checkpoints for JDBC and ES tasks only - -Pipeline 2 (Kafka → JDBC): - CheckpointCoordinator triggers every 30s (different interval) - Manages checkpoints for Kafka and JDBC tasks only -``` +| Pipeline | Checkpoint behavior | +|----------|---------------------| +| `Pipeline 1 (JDBC → ES)` | A `CheckpointCoordinator` triggers every `60s` and manages only JDBC and Elasticsearch tasks | +| `Pipeline 2 (Kafka → JDBC)` | A separate coordinator can trigger every `30s` and manages only Kafka and JDBC tasks | ## 5. PhysicalVertex: Deployed Task @@ -442,18 +419,11 @@ public class TaskGroupDefaultImpl implements TaskGroup { 3. No data shuffle required **Example (with fusion)**: -``` -LogicalDag: - Source (parallelism=4) → Transform (parallelism=4) → Sink (parallelism=4) - -Without Fusion: - 12 separate tasks (4 + 4 + 4) - Network overhead for Source → Transform and Transform → Sink - -With Fusion: - 4 TaskGroups, each containing: - [SourceTask → TransformTask → SinkTask] (single thread, shared memory) -``` +| Mode | Execution shape | Impact | +|------|-----------------|--------| +| Logical DAG | `Source (4) → Transform (4) → Sink (4)` | Same business topology in both modes | +| Without fusion | `12` separate tasks with network hops between stages | Higher serialization and network overhead | +| With fusion | `4` task groups, each containing `SourceTask → TransformTask → SinkTask` | Lower network cost and better locality | **Benefits**: - Reduced network serialization/deserialization @@ -600,17 +570,11 @@ sink { ### 7.3 Resource Allocation **Slot Calculation**: -``` -Required Slots = Sum of all task parallelism - -Example: - Source (parallelism=4) + Transform (parallelism=4) + Sink (parallelism=2) - = 10 slots required +Slot sizing rule: -With Fusion: - TaskGroup (parallelism=4, fusion[Source+Transform]) + Sink (parallelism=2) - = 6 slots required -``` +- `Required slots = sum of all task parallelism` +- Example without fusion: `Source (4) + Transform (4) + Sink (2) = 10 slots` +- Example with fusion: `TaskGroup (4, Source+Transform fused) + Sink (2) = 6 slots` **Resource Profile**: ```java @@ -641,15 +605,12 @@ ResourceProfile profile = **Key Insight**: Pipeline failures are isolated. **Example**: -``` -Job with 2 pipelines: - Pipeline 1: JDBC → ES (RUNNING) - Pipeline 2: Kafka → JDBC (FAILED) +| Pipeline | State | Recovery outcome | +|----------|-------|------------------| +| `Pipeline 1` | `RUNNING` | Keeps running | +| `Pipeline 2` | `FAILED` | Restarts from the latest checkpoint | -Result: - Pipeline 2 restarts from checkpoint - Pipeline 1 continues unaffected -``` +The key point is isolation: one failed pipeline does not automatically stop unrelated healthy pipelines. **Benefits**: - Reduced blast radius @@ -672,32 +633,21 @@ Result: ### 9.2 Visualization -``` -Job: mysql-to-es -│ -├── Pipeline 1 (mysql-cdc → elasticsearch) -│ ├── PhysicalVertex 0 [RUNNING] @ worker-1:slot-1 -│ ├── PhysicalVertex 1 [RUNNING] @ worker-2:slot-1 -│ ├── PhysicalVertex 2 [RUNNING] @ worker-3:slot-1 -│ └── PhysicalVertex 3 [RUNNING] @ worker-4:slot-1 -│ -└── Pipeline 2 (mysql-cdc → jdbc) - ├── PhysicalVertex 0 [RUNNING] @ worker-1:slot-2 - └── PhysicalVertex 1 [RUNNING] @ worker-2:slot-2 -``` +| Pipeline | Vertex placement | +|----------|------------------| +| `Pipeline 1 (mysql-cdc → elasticsearch)` | `PhysicalVertex 0 @ worker-1:slot-1`, `1 @ worker-2:slot-1`, `2 @ worker-3:slot-1`, `3 @ worker-4:slot-1` | +| `Pipeline 2 (mysql-cdc → jdbc)` | `PhysicalVertex 0 @ worker-1:slot-2`, `1 @ worker-2:slot-2` | ## 10. Best Practices ### 10.1 Parallelism Configuration **Rule of Thumb**: -``` -Parallelism = min( - data partitions, - available slots, - target throughput / single-task throughput -) -``` +Choose parallelism from the smallest practical bound among: + +- data partitions +- available slots +- `target throughput / single-task throughput` **Examples**: - **JDBC Source**: Set to number of DB partitions (e.g., 8 partitions → parallelism=8) diff --git a/docs/en/architecture/engine/engine-architecture.md b/docs/en/architecture/engine/engine-architecture.md index e52300808165..d0c54b3822d9 100644 --- a/docs/en/architecture/engine/engine-architecture.md +++ b/docs/en/architecture/engine/engine-architecture.md @@ -41,73 +41,39 @@ SeaTunnel Engine (Zeta) is designed as a native execution engine with: ### 2.1 Master-Worker Architecture -``` -┌─────────────────────────────────────────────────────────────────┐ -│ Master Node │ -│ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ CoordinatorService │ │ -│ │ • Manages all running jobs │ │ -│ │ • Job submission and lifecycle management │ │ -│ │ • Maintains job state (IMap) │ │ -│ │ • Resource manager factory │ │ -│ └───────────────────────────────────────────────────────┘ │ -│ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ JobMaster (one per job) │ │ -│ │ • Generates physical execution plan │ │ -│ │ • Requests resources from ResourceManager │ │ -│ │ • Deploys tasks to workers │ │ -│ │ • Coordinates checkpoints │ │ -│ │ • Handles failover and recovery │ │ -│ └───────────────────────────────────────────────────────┘ │ -│ │ │ │ -│ │ (Task Deploy) │ (Resource Request) │ -│ ▼ ▼ │ -│ ┌─────────────────┐ ┌────────────────────────────┐ │ -│ │ CheckpointManager│ │ ResourceManager │ │ -│ │ (per pipeline) │ │ • Slot allocation │ │ -│ └─────────────────┘ │ • Worker registration │ │ -│ │ • Load balancing │ │ -│ └────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ - │ - │ (Hazelcast Cluster) - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ Worker Nodes │ -│ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ TaskExecutionService │ │ -│ │ • Deploys and executes tasks │ │ -│ │ • Manages task lifecycle │ │ -│ │ • Reports heartbeat │ │ -│ │ • Slot resource management │ │ -│ └───────────────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ SeaTunnelTask (multiple per worker) │ │ -│ │ │ │ -│ │ ┌─────────────────────────────────────────────┐ │ │ -│ │ │ SourceFlowLifeCycle │ │ │ -│ │ │ • SourceReader │ │ │ -│ │ │ • SeaTunnelSourceCollector │ │ │ -│ │ └─────────────────────────────────────────────┘ │ │ -│ │ │ │ │ -│ │ ▼ │ │ -│ │ ┌─────────────────────────────────────────────┐ │ │ -│ │ │ TransformFlowLifeCycle │ │ │ -│ │ │ • Transform chain │ │ │ -│ │ └─────────────────────────────────────────────┘ │ │ -│ │ │ │ │ -│ │ ▼ │ │ -│ │ ┌─────────────────────────────────────────────┐ │ │ -│ │ │ SinkFlowLifeCycle │ │ │ -│ │ │ • SinkWriter │ │ │ -│ │ └─────────────────────────────────────────────┘ │ │ -│ └───────────────────────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ +```mermaid +flowchart TD + subgraph master["Master Node"] + coordinator["CoordinatorService
Manage running jobs
Handle submission and lifecycle
Maintain distributed job state"] + jobMaster["JobMaster (one per job)
Generate physical execution plan
Request resources
Deploy tasks and coordinate checkpoints"] + checkpoint["CheckpointManager
Per pipeline"] + resource["ResourceManager
Slot allocation
Worker registration
Load balancing"] + + coordinator --> jobMaster + jobMaster --> checkpoint + jobMaster --> resource + end + + subgraph workers["Worker Nodes"] + tes["TaskExecutionService
Deploy and execute tasks
Manage task lifecycle
Report heartbeat"] + sourceFlow["SourceFlowLifeCycle
SourceReader / Collector"] + transformFlow["TransformFlowLifeCycle
Transform chain"] + sinkFlow["SinkFlowLifeCycle
SinkWriter"] + + tes --> sourceFlow --> transformFlow --> sinkFlow + end + + resource -- "Hazelcast cluster / slot assignment" --> tes + jobMaster -- "Task deployment" --> tes + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class master,workers layerBlue; + class coordinator,jobMaster,checkpoint,resource layerCyan; + class tes,sourceFlow,transformFlow,sinkFlow layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.2 Core Components @@ -150,9 +116,7 @@ Manages single job execution lifecycle. - Handle task failures and reschedule **Lifecycle**: -``` -Created → Initialized → Scheduled → Running → Finished/Failed/Canceled -``` +`Created → Initialized → Scheduled → Running → Finished / Failed / Canceled` **Key Operations**: 1. `init()`: Generate physical plan, create checkpoint coordinators @@ -188,50 +152,26 @@ Manages worker resources and slot allocation. ### 3.1 Execution Plan Transformation -``` -User Config (HOCON) - │ - ▼ -┌───────────────┐ -│ LogicalDag │ • Logical vertices (Source/Transform/Sink) -│ │ • Logical edges (data flow) -│ │ • Parallelism (per vertex) -└───────────────┘ - │ (JobMaster.generatePhysicalPlan()) - ▼ -┌───────────────┐ -│ PhysicalPlan │ • List of SubPlan (pipelines) -│ │ • JobImmutableInformation -│ │ • Resource requirements -└───────────────┘ - │ - ▼ -┌───────────────┐ -│ SubPlan │ • Pipeline (independent execution unit) -│ (Pipeline) │ • List of PhysicalVertex -│ │ • CheckpointCoordinator -└───────────────┘ - │ - ▼ -┌───────────────┐ -│PhysicalVertex │ • TaskGroup (co-located tasks) -│ │ • Assigned SlotProfile -│ │ • ExecutionState -└───────────────┘ - │ - ▼ -┌───────────────┐ -│ TaskGroup │ • Multiple SeaTunnelTask instances -│ │ • Shared network buffer -│ │ • Thread pool -└───────────────┘ - │ - ▼ -┌───────────────┐ -│ SeaTunnelTask │ • Single task execution -│ │ • Source/Transform/Sink lifecycle -│ │ • Task state machine -└───────────────┘ +```mermaid +flowchart TD + config["User Config
HOCON"] + logical["LogicalDag
Logical vertices
Logical edges
Parallelism hints"] + physical["PhysicalPlan
SubPlan list
Immutable job info
Resource requirements"] + subplan["SubPlan (Pipeline)
Independent execution unit
PhysicalVertex list
CheckpointCoordinator"] + vertex["PhysicalVertex
TaskGroup
Assigned SlotProfile
ExecutionState"] + taskGroup["TaskGroup
Multiple SeaTunnelTask instances
Shared buffers / thread pool"] + task["SeaTunnelTask
Single task execution
Source / Transform / Sink lifecycle"] + + config --> logical --> physical --> subplan --> vertex --> taskGroup --> task + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class config,logical,physical layerBlue; + class subplan,vertex layerCyan; + class taskGroup,task layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 3.2 LogicalDag @@ -323,10 +263,10 @@ sink { ``` **Generated Pipelines**: -``` -Pipeline 1: MySQL-CDC → Transform → Elasticsearch -Pipeline 2: Kafka → Transform → JDBC -``` +Generated pipelines: + +- `Pipeline 1`: `MySQL-CDC → Transform → Elasticsearch` +- `Pipeline 2`: `Kafka → Transform → JDBC` **Benefits**: - Independent checkpoint coordination @@ -337,13 +277,10 @@ Pipeline 2: Kafka → Transform → JDBC Multiple actions can be fused into single TaskGroup for efficiency: -``` -Without Fusion: -[Source Task] → Network → [Transform Task] → Network → [Sink Task] - -With Fusion: -[TaskGroup: Source → Transform → Sink] (single thread, no network) -``` +| Mode | Runtime shape | Trade-off | +|------|---------------|-----------| +| Without fusion | `Source Task → Network → Transform Task → Network → Sink Task` | Clear stage separation, but more network serialization overhead | +| With fusion | `TaskGroup: Source → Transform → Sink` in one thread | Lower network cost and better locality, but less scheduling flexibility | **Fusion Conditions**: - Same parallelism @@ -354,43 +291,36 @@ With Fusion: ### 4.1 Task State Machine -``` - [Created] - │ - ▼ - [INIT] ────────────────────────────────────┐ - │ │ - ▼ │ -[WAITING_RESTORE] (if recovering) │ - │ │ - ▼ │ - [READY_START] │ - │ │ - ▼ │ - [STARTING] ──────────────┐ │ - │ │ │ - ▼ ▼ ▼ - [RUNNING] ──────────> [FAILED] ─────> (Restart) - │ - ▼ -[PREPARE_CLOSE] - │ - ▼ - [CLOSED] - │ - ▼ - [CANCELED] (if job canceled) +```mermaid +stateDiagram-v2 + [*] --> CREATED + CREATED --> INIT + INIT --> WAITING_RESTORE: restore path + INIT --> READY_START: fresh start + WAITING_RESTORE --> READY_START + READY_START --> STARTING + STARTING --> RUNNING + RUNNING --> PREPARE_CLOSE: normal completion + PREPARE_CLOSE --> CLOSED + INIT --> CANCELLING: external cancel + WAITING_RESTORE --> CANCELLING + READY_START --> CANCELLING + STARTING --> CANCELLING + RUNNING --> CANCELLING + PREPARE_CLOSE --> CANCELLING + CANCELLING --> CANCELED ``` **State Transitions**: -1. **CREATED → INIT**: Task created, initializing resources -2. **INIT → WAITING_RESTORE**: Recovering from checkpoint -3. **WAITING_RESTORE → READY_START**: State restored -4. **READY_START → STARTING**: Opening Source/Transform/Sink -5. **STARTING → RUNNING**: Data processing started -6. **RUNNING → PREPARE_CLOSE**: Normal completion -7. **PREPARE_CLOSE → CLOSED**: Resources cleaned up -8. **RUNNING → FAILED**: Exception occurred +1. **CREATED → INIT**: Task created and runtime resources initialized +2. **INIT → WAITING_RESTORE / READY_START**: Decide between restore path and fresh start +3. **WAITING_RESTORE → READY_START**: Restore is complete and flows are ready to open +4. **READY_START → STARTING → RUNNING**: The task receives the start signal and enters the main processing loop +5. **RUNNING → PREPARE_CLOSE → CLOSED**: Normal completion path after barriers and cleanup +6. **Active state → CANCELLING → CANCELED**: External cancellation path handled outside the normal completion flow + +**Failure Note**: +- `FAILED` exists as a runtime result, but task-level restart is handled by higher-level recovery logic rather than by a direct `FAILED → ...` edge in this state machine. ### 4.2 SeaTunnelTask Execution diff --git a/docs/en/architecture/engine/resource-management.md b/docs/en/architecture/engine/resource-management.md index 6d89cc5d89ed..f0290e3f3813 100644 --- a/docs/en/architecture/engine/resource-management.md +++ b/docs/en/architecture/engine/resource-management.md @@ -29,55 +29,22 @@ SeaTunnel's resource management system aims to: ### 1.3 Architecture Overview -``` -┌──────────────────────────────────────────────────────────────┐ -│ JobMaster │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ Request Resources │ │ -│ │ • Calculate required slots │ │ -│ │ • Specify resource profiles (CPU, memory) │ │ -│ │ • Apply tag filters (optional) │ │ -│ └────────────────────────────────────────────────────┘ │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ ResourceManager │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ Worker Registry │ │ -│ │ • WorkerProfile (per worker) │ │ -│ │ - Total resources │ │ -│ │ - Available resources │ │ -│ │ - Assigned slots │ │ -│ │ - Unassigned slots │ │ -│ └────────────────────────────────────────────────────┘ │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ Allocation Strategies │ │ -│ │ • RandomStrategy / SlotRatioStrategy / SystemLoadStrategy │ -│ └────────────────────────────────────────────────────┘ │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ Slot Management │ │ -│ │ • Allocate slots │ │ -│ │ • Release slots │ │ -│ │ • Track slot usage │ │ -│ └────────────────────────────────────────────────────┘ │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ Worker Nodes │ -│ │ -│ Worker 1 Worker 2 Worker N │ -│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ -│ │ Slot 1 │ │ Slot 1 │ │ Slot 1 │ │ -│ │ Slot 2 │ │ Slot 2 │ │ Slot 2 │ │ -│ │ ... │ │ ... │ │ ... │ │ -│ └──────────┘ └──────────┘ └──────────┘ │ -└──────────────────────────────────────────────────────────────┘ +```mermaid +flowchart TD + jobMaster["JobMaster
Request resources
Calculate required slots
Specify resource profiles
Apply optional tag filters"] + resource["ResourceManager
Maintain worker registry
Apply allocation strategies
Allocate and release slots
Track slot usage"] + workers["Worker Nodes
Worker 1 / Worker 2 / Worker N
Each worker exposes slot inventory"] + + jobMaster --> resource --> workers + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class jobMaster layerBlue; + class resource layerCyan; + class workers layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## 2. Core Concepts @@ -448,24 +415,17 @@ seatunnel: ### 9.2 Observability **Resource Dashboard Example**: -``` -Cluster Resources: - Workers: 10 (all healthy) - Total Slots: 20 - Available Slots: 8 - Utilization: 60% - -Top Resource Consumers: - job-123: 6 slots (mysql-cdc → elasticsearch) - job-456: 4 slots (kafka → jdbc) - job-789: 2 slots (file → s3) - -Worker Distribution: - worker-1: 2/2 slots (100%) - worker-2: 1/2 slots (50%) - worker-3: 2/2 slots (100%) - ... -``` +**Resource Dashboard Example**: + +| Scope | Metric | Example | +|-------|--------|---------| +| Cluster | Workers | `10` healthy workers | +| Cluster | Total slots | `20` | +| Cluster | Available slots | `8` | +| Cluster | Utilization | `60%` | +| Job | Top consumer | `job-123` using `6` slots for `mysql-cdc → elasticsearch` | +| Job | Next consumer | `job-456` using `4` slots for `kafka → jdbc` | +| Worker | Slot distribution | `worker-1: 2/2`, `worker-2: 1/2`, `worker-3: 2/2` | ## 10. Best Practices diff --git a/docs/en/architecture/fault-tolerance/checkpoint-mechanism.md b/docs/en/architecture/fault-tolerance/checkpoint-mechanism.md index d31a6b5a2269..31f55a179bd5 100644 --- a/docs/en/architecture/fault-tolerance/checkpoint-mechanism.md +++ b/docs/en/architecture/fault-tolerance/checkpoint-mechanism.md @@ -44,82 +44,29 @@ Result: Globally consistent snapshot without pausing entire system. ### 2.1 Checkpoint Architecture -``` -┌─────────────────────────────────────────────────────────────────┐ -│ JobMaster (per job) │ -│ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ CheckpointCoordinator (per pipeline) │ │ -│ │ │ │ -│ │ • Trigger checkpoint (periodic/manual) │ │ -│ │ • Generate checkpoint ID │ │ -│ │ • Track pending checkpoints │ │ -│ │ • Collect task acknowledgements │ │ -│ │ • Persist completed checkpoints │ │ -│ │ • Cleanup old checkpoints │ │ -│ └───────────────────────────────────────────────────────┘ │ -│ │ │ -│ │ (Trigger Barrier) │ -│ ▼ │ -└─────────────────────────────────────────────────────────────────┘ - │ - │ (CheckpointBarrier) - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ Worker Nodes │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ SourceTask 1 │ │ SourceTask 2 │ │ SourceTask N │ │ -│ │ │ │ │ │ │ │ -│ │ 1. Receive │ │ 1. Receive │ │ 1. Receive │ │ -│ │ Barrier │ │ Barrier │ │ Barrier │ │ -│ │ 2. Snapshot │ │ 2. Snapshot │ │ 2. Snapshot │ │ -│ │ State │ │ State │ │ State │ │ -│ │ 3. ACK │ │ 3. ACK │ │ 3. ACK │ │ -│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ -│ │ │ │ │ -│ │ (Barrier Propagation) │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ Transform 1 │ │ Transform 2 │ │ Transform N │ │ -│ │ │ │ │ │ │ │ -│ │ 1. Receive │ │ 1. Receive │ │ 1. Receive │ │ -│ │ Barrier │ │ Barrier │ │ Barrier │ │ -│ │ 2. Snapshot │ │ 2. Snapshot │ │ 2. Snapshot │ │ -│ │ State │ │ State │ │ State │ │ -│ │ 3. ACK │ │ 3. ACK │ │ 3. ACK │ │ -│ │ 4. Forward │ │ 4. Forward │ │ 4. Forward │ │ -│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ SinkTask 1 │ │ SinkTask 2 │ │ SinkTask N │ │ -│ │ │ │ │ │ │ │ -│ │ 1. Receive │ │ 1. Receive │ │ 1. Receive │ │ -│ │ Barrier │ │ Barrier │ │ Barrier │ │ -│ │ 2. Prepare │ │ 2. Prepare │ │ 2. Prepare │ │ -│ │ Commit │ │ Commit │ │ Commit │ │ -│ │ 3. Snapshot │ │ 3. Snapshot │ │ 3. Snapshot │ │ -│ │ State │ │ State │ │ State │ │ -│ │ 4. ACK │ │ 4. ACK │ │ 4. ACK │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ - │ - │ (All ACKs received) - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ CheckpointStorage │ -│ (HDFS / S3 / Local / OSS) │ -│ │ -│ CompletedCheckpoint { │ -│ checkpointId: 123 │ -│ taskStates: { │ -│ SourceTask-1: { splits: [...], offsets: [...] } │ -│ SinkTask-1: { commitInfo: XidInfo(...) } │ -│ ... │ -│ } │ -│ } │ -└─────────────────────────────────────────────────────────────────┘ +```mermaid +flowchart TD + jobMaster["JobMaster
One per job"] + coordinator["CheckpointCoordinator
Trigger checkpoints
Generate checkpoint IDs
Track pending checkpoints
Collect acknowledgements
Persist completed checkpoints"] + source["Source tasks
Receive barrier
Snapshot state
ACK"] + transform["Transform tasks
Receive barrier
Snapshot state
Forward barrier
ACK"] + sink["Sink tasks
Receive barrier
Prepare commit
Snapshot state
ACK"] + storage["CheckpointStorage
HDFS / S3 / Local / OSS
CompletedCheckpoint with task states"] + + jobMaster --> coordinator + coordinator -- "Trigger barrier" --> source + source -- "Barrier propagation" --> transform + transform -- "Barrier propagation" --> sink + sink -- "All ACKs received" --> storage + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class jobMaster,coordinator layerBlue; + class source,transform,sink layerCyan; + class storage layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.2 Key Data Structures @@ -331,15 +278,10 @@ sequenceDiagram 3. **Sink Tasks**: End of pipeline, receive from upstream, snapshot, no forward **Barrier Alignment** (for tasks with multiple inputs): -```java -// Task with 2 inputs -Input 1: ──data──data──[barrier-123]──data──data── - │ Wait! -Input 2: ──data──data──data──data──[barrier-123]── - │ - ▼ - Both barriers received, snapshot state -``` +| Input | Arrival pattern | Alignment behavior | +|-------|-----------------|--------------------| +| `Input 1` | `barrier-123` arrives first | This input waits at the alignment point | +| `Input 2` | `barrier-123` arrives later | Once the same barrier arrives here, the task snapshots state and resumes downstream emission | ### 3.3 State Snapshot diff --git a/docs/en/architecture/fault-tolerance/exactly-once.md b/docs/en/architecture/fault-tolerance/exactly-once.md index f89fc051ba38..ed4911fdf185 100644 --- a/docs/en/architecture/fault-tolerance/exactly-once.md +++ b/docs/en/architecture/fault-tolerance/exactly-once.md @@ -16,15 +16,10 @@ Distributed data processing faces fundamental delivery guarantees challenges: - **Exactly-Once**: Each record processed exactly once (ideal but complex) **Real-World Impact**: -``` -Scenario: Financial transaction processing - -At-Least-Once: - Transaction $100 processed twice → User charged $200 ❌ - -Exactly-Once: - Transaction $100 processed once → User charged $100 ✅ -``` +| Scenario | Outcome | Result | +|----------|---------|--------| +| At-least-once | `Transaction $100` processed twice | User is charged `$200` | +| Exactly-once | `Transaction $100` processed once | User is charged `$100` | ### 1.2 Design Goals @@ -75,48 +70,28 @@ SeaTunnel's exactly-once semantics aims to: ### 3.1 End-to-End Pipeline -``` -┌──────────────────────────────────────────────────────────────┐ -│ Source │ -│ • Read from external system │ -│ • Track offsets/positions │ -│ • Snapshot offsets in checkpoint │ -└──────────────────────────┬───────────────────────────────────┘ - │ - ▼ Checkpoint Barrier -┌──────────────────────────────────────────────────────────────┐ -│ Transform │ -│ • Process records │ -│ • Snapshot transform state (if any) │ -└──────────────────────────┬───────────────────────────────────┘ - │ - ▼ Checkpoint Barrier -┌──────────────────────────────────────────────────────────────┐ -│ Sink Writer │ -│ • Buffer writes │ -│ • prepareCommit(checkpointId) → Generate CommitInfo (PHASE 1)│ -│ • Snapshot writer state │ -└──────────────────────────┬───────────────────────────────────┘ - │ - │ CommitInfo - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ CheckpointCoordinator │ -│ • Collect all CommitInfos │ -│ • Persist CompletedCheckpoint │ -│ • Trigger commit phase │ -└──────────────────────────┬───────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ Sink Committer │ -│ • commit(CommitInfos) → Apply changes (PHASE 2) │ -│ • Must be idempotent │ -└──────────────────────────┬───────────────────────────────────┘ - │ - ▼ - External Sink - (Changes visible) +```mermaid +flowchart TD + source["Source
Read external data
Track offsets or positions
Snapshot offsets in checkpoint"] + transform["Transform
Process records
Snapshot transform state if needed"] + writer["Sink Writer
Buffer writes
prepareCommit(checkpointId)
Snapshot writer state"] + coordinator["CheckpointCoordinator
Collect CommitInfos
Persist CompletedCheckpoint
Trigger commit phase"] + committer["Sink Committer
commit(CommitInfos)
Must be idempotent"] + external["External Sink
Changes become visible"] + + source -- "Checkpoint barrier" --> transform + transform -- "Checkpoint barrier" --> writer + writer -- "CommitInfo" --> coordinator + coordinator --> committer --> external + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source,transform layerBlue; + class writer,coordinator layerCyan; + class committer,external layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 3.2 Key Components @@ -393,69 +368,69 @@ public class FileSinkCommitter { ### 5.1 Task Failure Before Checkpoint -``` -Timeline: - t0: Checkpoint N completed - t1: Process records [1000-2000] - t2: Task fails ❌ - t3: Restore from Checkpoint N - t4: Reprocess records [1000-2000] +| Time | Event | +|------|-------| +| `t0` | Checkpoint `N` completes | +| `t1` | Records `[1000-2000]` are processed | +| `t2` | Task fails before the next successful checkpoint | +| `t3` | Job restores from checkpoint `N` | +| `t4` | Records `[1000-2000]` are reprocessed | Result: - ✅ No data loss (records reprocessed) - ✅ No duplication (nothing committed before failure) -``` + +- No data loss because the records are replayed after recovery. +- No duplication because nothing was committed before the failure. ### 5.2 Task Failure After prepareCommit -``` -Timeline: - t0: Checkpoint N in progress - t1: SinkWriter.prepareCommit(checkpointId) → XID-123 prepared - t2: Task fails ❌ (before commit) - t3: Restore from Checkpoint N-1 - t4: Reprocess records - t5: New prepareCommit(checkpointId) → XID-124 prepared - t6: Committer commits XID-124 +| Time | Event | +|------|-------| +| `t0` | Checkpoint `N` is in progress | +| `t1` | `SinkWriter.prepareCommit(...)` prepares `XID-123` | +| `t2` | Task fails before the commit step | +| `t3` | Job restores from checkpoint `N-1` | +| `t4` | Records are reprocessed | +| `t5` | A new `prepareCommit(...)` prepares `XID-124` | +| `t6` | The committer commits `XID-124` | Result: - ✅ XID-123 never committed (automatically rolled back after timeout) - ✅ XID-124 committed (correct data) -``` + +- `XID-123` never becomes externally visible. +- `XID-124` is the only committed transaction for this checkpoint boundary. ### 5.3 Committer Failure During Commit -``` -Timeline: - t0: Checkpoint N completed - t1: Committer starts committing [XID-100, XID-101, XID-102] - t2: Commits XID-100 ✅ - t3: Committer fails ❌ (XID-101, XID-102 not committed) - t4: New committer retries [XID-100, XID-101, XID-102] - t5: Commits XID-100 (already committed, idempotent) ✅ - t6: Commits XID-101 ✅ - t7: Commits XID-102 ✅ +| Time | Event | +|------|-------| +| `t0` | Checkpoint `N` completes | +| `t1` | The committer starts committing `XID-100`, `XID-101`, `XID-102` | +| `t2` | `XID-100` commits successfully | +| `t3` | The committer fails before `XID-101` and `XID-102` finish | +| `t4` | A new committer retries the whole batch | +| `t5` | `XID-100` is seen as already committed and treated idempotently | +| `t6` | `XID-101` commits successfully | +| `t7` | `XID-102` commits successfully | Result: - ✅ All XIDs eventually committed - ✅ No duplication (idempotent commit) -``` + +- Every prepared transaction is eventually committed. +- Idempotent commit logic prevents duplicates during retry. ### 5.4 Network Partition -``` -Timeline: - t0: SinkWriter prepares XID-200 - t1: Checkpoint completes - t2: Committer sends commit(XID-200) - t3: Network partition ⚠️ (commit success, but ACK lost) - t4: Committer retries commit(XID-200) - t5: XID-200 already committed (idempotent) +| Time | Event | +|------|-------| +| `t0` | `SinkWriter` prepares `XID-200` | +| `t1` | The checkpoint completes | +| `t2` | The committer sends `commit(XID-200)` | +| `t3` | A network partition causes the ACK to be lost | +| `t4` | The committer retries `commit(XID-200)` | +| `t5` | The sink reports that `XID-200` is already committed | Result: - ✅ Data committed exactly once - ✅ Idempotency prevents duplication -``` + +- The external system still sees exactly one committed transaction. +- Idempotency absorbs the duplicate commit attempt safely. ## 6. Idempotency Requirements @@ -529,17 +504,10 @@ try { ### 7.1 Checkpoint Interval Trade-offs -``` -Short Interval (10-30s): - ✅ Fast recovery (less reprocessing) - ❌ Higher overhead (frequent snapshots) - ❌ More commit operations - -Long Interval (5-10min): - ✅ Lower overhead (less frequent snapshots) - ❌ Slower recovery (more reprocessing) - ✅ Fewer commit operations -``` +| Checkpoint interval | Strengths | Trade-offs | +|---------------------|-----------|------------| +| Short (`10-30s`) | Faster recovery and less reprocessing | Higher snapshot overhead and more commit operations | +| Long (`5-10min`) | Lower steady-state overhead and fewer commits | Slower recovery and more reprocessing after failure | **Recommendation**: 60-120 seconds for most workloads @@ -700,15 +668,13 @@ public void testExactlyOnceUnderChaos() { ### 9.3 Monitoring Verification -``` -Metrics to Track: - -source.records_read = 1,000,000 -sink.records_written = 1,000,000 -sink.records_committed = 1,000,000 +| Metric | Example value | Expectation | +|--------|---------------|-------------| +| `source.records_read` | `1,000,000` | Should match committed sink records | +| `sink.records_written` | `1,000,000` | Should match source reads after retries settle | +| `sink.records_committed` | `1,000,000` | Should match source reads for exactly-once verification | -✅ All counts match → Exactly-once verified -``` +When all three counters converge, the end-to-end path is behaving as expected. ## 10. Best Practices @@ -752,11 +718,11 @@ public void write(SeaTunnelRow element) { - `checkpoint.size`: Monitor growth over time **Alerts**: -``` -Alert if checkpoint.duration > 300s -Alert if checkpoint.failure_rate > 5% -Alert if no checkpoint in 2x interval -``` +Recommended alerts: + +- Alert when `checkpoint.duration > 300s` +- Alert when `checkpoint.failure_rate > 5%` +- Alert when no checkpoint completes within `2x` the configured interval ## 11. Related Resources diff --git a/docs/en/architecture/features/multi-table.md b/docs/en/architecture/features/multi-table.md index ff25365feb94..43d115a83296 100644 --- a/docs/en/architecture/features/multi-table.md +++ b/docs/en/architecture/features/multi-table.md @@ -373,16 +373,10 @@ public class MultiTableSinkCommitter **Solution**: Multiple replica writers per table for parallel writing. -``` -Without Replicas: - orders table (1000 writes/sec) → [Single Writer] → Bottleneck - -With Replicas (replicaNum=4): - orders table (1000 writes/sec) → [Writer 0] (250 writes/sec) - → [Writer 1] (250 writes/sec) - → [Writer 2] (250 writes/sec) - → [Writer 3] (250 writes/sec) -``` +| Mode | Write layout | Effect | +|------|--------------|--------| +| Without replicas | `orders` table → single writer | A single writer becomes the bottleneck | +| `replicaNum = 4` | `orders` table → writer `0/1/2/3`, each handling about `250 writes/sec` | Throughput is spread across replicas | ### 5.2 Replica Configuration @@ -454,43 +448,31 @@ public class MultiTableSinkWriter { ### 7.1 Full Pipeline -``` -┌──────────────────────────────────────────────────────────────┐ -│ MySQL CDC Source │ -│ • Captures changes from 100 tables │ -│ • Tags each row with TablePath │ -└──────────────────────────┬───────────────────────────────────┘ - │ - ▼ - ┌─────────────────────────────────────┐ - │ SeaTunnelRow (with TablePath) │ - │ tableId: "my_db.public.orders" │ - │ fields: [1, "order-001", 99.99] │ - └─────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ MultiTableSinkWriter │ -│ • Extracts TablePath from row │ -│ • Selects replica (hash or random) │ -│ • Routes to correct writer │ -└──────────────────────────┬───────────────────────────────────┘ - │ - ┌──────────────────┼──────────────────┐ - ▼ ▼ ▼ -┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ orders │ │ users │ │ products │ -│ Writer 0 │ │ Writer 0 │ │ Writer 0 │ -│ Writer 1 │ │ Writer 1 │ │ Writer 1 │ -│ Writer 2 │ │ │ │ │ -│ Writer 3 │ │ │ │ │ -└──────────────┘ └──────────────┘ └──────────────┘ - │ │ │ - ▼ ▼ ▼ -┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ PostgreSQL │ │ PostgreSQL │ │ PostgreSQL │ -│ orders │ │ users │ │ products │ -└──────────────┘ └──────────────┘ └──────────────┘ +```mermaid +flowchart TD + source["MySQL CDC Source
Capture changes from many tables
Tag each row with TablePath"] + row["SeaTunnelRow
tableId = my_db.public.orders
fields = [1, order-001, 99.99]"] + writer["MultiTableSinkWriter
Extract TablePath
Select replica by hash or random
Route to the correct writer"] + orders["orders writers
Writer 0 / 1 / 2 / 3"] + users["users writers
Writer 0 / 1"] + products["products writers
Writer 0 / 1"] + pgOrders["PostgreSQL orders"] + pgUsers["PostgreSQL users"] + pgProducts["PostgreSQL products"] + + source --> row --> writer + writer --> orders --> pgOrders + writer --> users --> pgUsers + writer --> products --> pgProducts + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source,row layerBlue; + class writer,orders,users,products layerCyan; + class pgOrders,pgUsers,pgProducts layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 7.2 Write Flow @@ -545,14 +527,10 @@ sequenceDiagram ### 8.1 Replica Sizing **Rule of Thumb**: -``` -replicaNum = ceil(Table Write Rate / Single Writer Throughput) +Use this sizing heuristic: -Example: - orders: 10,000 writes/sec - Single writer: 2,500 writes/sec - replicaNum = ceil(10,000 / 2,500) = 4 -``` +- `replicaNum = ceil(table write rate / single writer throughput)` +- Example: if `orders` writes at `10,000 writes/sec` and one writer sustains `2,500 writes/sec`, choose `replicaNum = 4` ### 8.2 Table-Specific Replicas diff --git a/docs/en/architecture/overview.md b/docs/en/architecture/overview.md index c45f6476c6a2..595c6a32a553 100644 --- a/docs/en/architecture/overview.md +++ b/docs/en/architecture/overview.md @@ -42,49 +42,26 @@ If you are using this section to build architectural understanding, read in this SeaTunnel adopts a layered architecture that separates concerns and enables flexibility: -``` -┌─────────────────────────────────────────────────────────────────┐ -│ User Configuration Layer │ -│ (HOCON Config / SQL / Web UI) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ SeaTunnel API Layer │ -│ (Source API / Sink API / Transform API / Table API) │ -│ │ -│ • SeaTunnelSource • CatalogTable │ -│ • SeaTunnelSink • TableSchema │ -│ • SeaTunnelTransform • SchemaChangeEvent │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ Connector Ecosystem │ -│ │ -│ [Jdbc] [Kafka] [MySQL-CDC] [Elasticsearch] [Iceberg] ... │ -│ (Connector Ecosystem) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ Translation Layer │ -│ (Adapts SeaTunnel API to Engine-Specific API) │ -│ │ -│ • FlinkSource/FlinkSink • SparkSource/SparkSink │ -│ • Context Adapters • Serialization Adapters │ -└─────────────────────────────────────────────────────────────────┘ - │ - ┌─────────────────────┼─────────────────────┐ - ▼ ▼ ▼ -┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ SeaTunnel │ │ Apache │ │ Apache │ -│ Engine (Zeta)│ │ Flink │ │ Spark │ -│ │ │ │ │ │ -│ • Master │ │ • JobManager │ │ • Driver │ -│ • Worker │ │ • TaskManager│ │ • Executor │ -│ • Checkpoint │ │ • State │ │ • RDD/DS │ -└──────────────┘ └──────────────┘ └──────────────┘ +```mermaid +flowchart TD + config["User Configuration Layer
HOCON Config / SQL / Web UI"] + api["SeaTunnel API Layer
Source API / Sink API / Transform API / Table API"] + connectors["Connector Ecosystem
JDBC / Kafka / MySQL-CDC / Elasticsearch / Iceberg / ..."] + translation["Translation Layer
Flink adapters / Spark adapters / Context wrappers / Serialization adapters"] + + config --> api --> connectors --> translation + translation --> zeta["SeaTunnel Engine (Zeta)
Master / Worker / Checkpoint"] + translation --> flink["Apache Flink
JobManager / TaskManager / State"] + translation --> spark["Apache Spark
Driver / Executor / RDD / Dataset"] + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class config,api layerBlue; + class connectors,translation layerCyan; + class zeta,flink,spark layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.1 Layer Responsibilities @@ -116,10 +93,10 @@ The API layer provides engine-independent abstractions: - [seatunnel-api/.../SourceSplitEnumerator.java](../../../seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceSplitEnumerator.java) #### Sink API -- **SeaTunnelSink**: Factory interface for creating writers and committers +- **SeaTunnelSink**: Factory interface for creating writers and optional commit strategies - **SinkWriter**: Worker-side component for writing data -- **SinkCommitter**: Coordinator for commit operations from multiple writers -- **SinkAggregatedCommitter**: Global coordinator for aggregated commits +- **SinkCommitter**: Optional worker-side committer for per-writer commit operations +- **SinkAggregatedCommitter**: Optional global committer for coordinator-side aggregated commits **Key Design**: Two-phase commit protocol (prepareCommit → commit) ensures exactly-once semantics. @@ -159,8 +136,16 @@ The native execution engine provides: - **FlowLifeCycle**: Manages lifecycle of Source/Transform/Sink components #### Execution Model -``` -LogicalDag → PhysicalPlan → SubPlan (Pipeline) → PhysicalVertex → TaskGroup → SeaTunnelTask +```mermaid +flowchart LR + logical["LogicalDag"] --> plan["PhysicalPlan"] --> pipeline["SubPlan
(Pipeline)"] --> vertex["PhysicalVertex"] --> taskGroup["TaskGroup"] --> task["SeaTunnelTask"] + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class logical,plan,pipeline layerBlue; + class vertex,taskGroup,task layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` **Code Reference**: @@ -183,20 +168,12 @@ Enables engine portability through adapter pattern: All connectors follow a standardized structure: -``` -connector-[name]/ -├── src/main/java/.../ -│ ├── [Name]Source.java # Implements SeaTunnelSource -│ ├── [Name]SourceReader.java # Implements SourceReader -│ ├── [Name]SourceSplitEnumerator.java -│ ├── [Name]SourceSplit.java -│ ├── [Name]Sink.java # Implements SeaTunnelSink -│ ├── [Name]SinkWriter.java # Implements SinkWriter -│ └── config/[Name]Config.java -└── src/main/resources/META-INF/services/ - ├── org.apache.seatunnel.api.table.factory.TableSourceFactory - └── org.apache.seatunnel.api.table.factory.TableSinkFactory -``` +| Area | Typical Files | Responsibility | +|------|---------------|----------------| +| Source entry | `[Name]Source.java`, `[Name]SourceReader.java`, `[Name]SourceSplitEnumerator.java`, `[Name]SourceSplit.java` | Read data, split work, and expose a unified Source contract | +| Sink entry | `[Name]Sink.java`, `[Name]SinkWriter.java` | Buffer, write, and commit data to the target system | +| Configuration | `config/[Name]Config.java` | Define connector options, validation rules, and defaults | +| SPI registration | `META-INF/services/TableSourceFactory`, `META-INF/services/TableSinkFactory` | Register factories for discovery and runtime loading | **Discovery Mechanism**: Java SPI (Service Provider Interface) for dynamic connector loading. @@ -204,47 +181,28 @@ connector-[name]/ ### 4.1 Source Data Flow -``` -Data Source - │ - ▼ -┌─────────────────────┐ -│ SourceSplitEnumerator│ (Master Side) -│ • Generate Splits │ -│ • Assign to Readers │ -└─────────────────────┘ - │ (Split Assignment) - ▼ -┌─────────────────────┐ -│ SourceReader │ (Worker Side) -│ • Read from Split │ -│ • Emit Records │ -└─────────────────────┘ - │ - ▼ - SeaTunnelRow - │ - ▼ - Transform Chain (Optional) - │ - ▼ - SeaTunnelRow - │ - ▼ -┌─────────────────────┐ -│ SinkWriter │ (Worker Side) -│ • Buffer Records │ -│ • Prepare Commit │ -└─────────────────────┘ - │ (CommitInfo) - ▼ -┌─────────────────────┐ -│ SinkCommitter │ (Coordinator) -│ • Commit Changes │ -└─────────────────────┘ - │ - ▼ -Data Sink +```mermaid +flowchart TD + source["Data Source"] --> enumerator["SourceSplitEnumerator
Master side
Generate splits / Assign readers"] + enumerator -->|Split assignment| reader["SourceReader
Worker side
Read split / Emit records"] + reader --> rowIn["SeaTunnelRow"] + rowIn --> transform["Transform Chain
(Optional)"] + transform --> rowOut["SeaTunnelRow"] + rowOut --> writer["SinkWriter
Worker side
Buffer records / Prepare commit"] + writer -->|"optional worker-local commit"| committer["SinkCommitter
Worker side
Commit each writer change independently"] + writer -. "optional aggregated commit path" .-> aggregatedTask["SinkAggregatedCommitterTask
Coordinator side
Collect commit infos from writers"] + aggregatedTask --> aggregated["SinkAggregatedCommitter
Coordinator side
Perform one global commit"] + committer --> sink["Data Sink"] + aggregated --> sink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source,sink,rowIn,rowOut layerBlue; + class enumerator,reader,transform layerCyan; + class writer,committer,aggregatedTask,aggregated layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 4.2 Split-based Parallelism @@ -258,10 +216,28 @@ Data Sink Jobs are divided into **Pipelines** (SubPlans): -``` -Pipeline 1: [Source A] → [Transform 1] → [Sink A] - ↓ -Pipeline 2: [Source B] ───────→ [Transform 2] → [Sink B] +The example below shows two independent subplans inside the same job. They do not directly exchange records with each other. + +```mermaid +flowchart TB + subgraph pipeline1["Pipeline 1"] + direction LR + sourceA["Source A"] --> transformA["Transform 1"] --> sinkA["Sink A"] + end + + subgraph pipeline2["Pipeline 2"] + direction LR + sourceB["Source B"] --> transformB["Transform 2"] --> sinkB["Sink B"] + end + + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class sourceA,sourceB,transformA,transformB layerCyan; + class sinkA,sinkB layerPurple; + style pipeline1 fill:#081425,stroke:#5db8e2,stroke-width:1.5px,color:#f8fbff; + style pipeline2 fill:#081425,stroke:#5db8e2,stroke-width:1.5px,color:#f8fbff; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` Each pipeline: @@ -309,25 +285,48 @@ sequenceDiagram 4. **Commit Phase** - SinkWriter prepares commit information - - SinkCommitter coordinates commits + - A worker-local `SinkCommitter` commits each writer change independently, or a coordinator-side aggregated commit runs when configured - State persisted to checkpoint storage ### 5.3 State Machine **Task State Transitions**: -``` -CREATED → INIT → WAITING_RESTORE → READY_START → STARTING → RUNNING - ↓ - FAILED ← ─────────────────────── → PREPARE_CLOSE → CLOSED - ↓ - CANCELED +```mermaid +stateDiagram-v2 + direction LR + [*] --> CREATED + CREATED --> INIT + INIT --> WAITING_RESTORE: restore path + INIT --> READY_START: fresh start + WAITING_RESTORE --> READY_START + READY_START --> STARTING + STARTING --> RUNNING + RUNNING --> PREPARE_CLOSE: normal completion + PREPARE_CLOSE --> CLOSED + INIT --> CANCELLING: external cancel + WAITING_RESTORE --> CANCELLING + READY_START --> CANCELLING + STARTING --> CANCELLING + RUNNING --> CANCELLING + PREPARE_CLOSE --> CANCELLING + CANCELLING --> CANCELED ``` +**Failure Note**: +- `FAILED` exists as a runtime result, but task-level restart is handled by higher-level recovery logic rather than by a direct `FAILED → ...` edge in this state machine. + **Job State Transitions**: -``` -CREATED → SCHEDULED → RUNNING → FINISHED - ↓ ↓ - FAILED CANCELING → CANCELED +```mermaid +stateDiagram-v2 + direction LR + [*] --> CREATED + CREATED --> SCHEDULED + SCHEDULED --> RUNNING + RUNNING --> FINISHED + SCHEDULED --> FAILED + RUNNING --> FAILED + RUNNING --> CANCELING + CANCELING --> CANCELED ``` ## 6. Key Features @@ -349,10 +348,10 @@ CREATED → SCHEDULED → RUNNING → FINISHED **Two-Phase Commit Protocol**: 1. **Prepare Phase**: SinkWriter prepares commit info during checkpoint -2. **Commit Phase**: SinkCommitter commits after checkpoint completes +2. **Commit Phase**: A worker-local `SinkCommitter` commits each writer change independently, or a coordinator-side aggregated commit performs one global commit after checkpoint success 3. **Abort Handling**: Roll back on failure before commit -**Idempotency**: SinkCommitter operations must be idempotent to handle retries +**Idempotency**: `SinkCommitter` and `SinkAggregatedCommitter` operations must be idempotent to handle retries ### 6.3 Dynamic Resource Management @@ -377,49 +376,23 @@ CREATED → SCHEDULED → RUNNING → FINISHED ## 7. Module Structure -``` -seatunnel/ -├── seatunnel-api/ # Core API definitions -│ ├── source/ # Source API -│ ├── sink/ # Sink API -│ ├── transform/ # Transform API -│ └── table/ # Table and Schema API -│ -├── seatunnel-connectors-v2/ # Connector implementations -│ ├── connector-jdbc/ # JDBC connector -│ ├── connector-kafka/ # Kafka connector -│ ├── connector-cdc-mysql/ # MySQL CDC connector -│ └── ... # connectors -│ -├── seatunnel-transforms-v2/ # Transform implementations -│ ├── transform-sql/ # SQL transform -│ ├── transform-filter/ # Filter transform -│ └── ... -│ -├── seatunnel-engine/ # SeaTunnel Engine (Zeta) -│ ├── seatunnel-engine-core/ # Core execution logic -│ ├── seatunnel-engine-server/ # Server components (Master/Worker) -│ └── seatunnel-engine-storage/ # Checkpoint storage -│ -├── seatunnel-translation/ # Engine translation layers -│ ├── seatunnel-translation-flink/ -│ └── seatunnel-translation-spark/ -│ -├── seatunnel-formats/ # Data format handlers -│ ├── seatunnel-format-json/ -│ ├── seatunnel-format-avro/ -│ └── ... -│ -├── seatunnel-core/ # Job submission and CLI -└── seatunnel-e2e/ # End-to-end tests -``` +| Module | Representative subdirectories | Responsibility | +|--------|-------------------------------|----------------| +| `seatunnel-api` | `source`, `sink`, `transform`, `table` | Defines the core APIs, table model, and engine-neutral abstractions | +| `seatunnel-connectors-v2` | `connector-jdbc`, `connector-kafka`, `connector-cdc-mysql` | Implements source and sink connectors | +| `seatunnel-transforms-v2` | `transform-sql`, `transform-filter` | Provides reusable transform implementations | +| `seatunnel-engine` | `seatunnel-engine-core`, `seatunnel-engine-server`, `seatunnel-engine-storage` | Hosts Zeta execution, scheduling, and checkpoint storage | +| `seatunnel-translation` | `seatunnel-translation-flink`, `seatunnel-translation-spark` | Adapts SeaTunnel APIs to different execution engines | +| `seatunnel-formats` | `seatunnel-format-json`, `seatunnel-format-avro` | Handles data format serialization and parsing | +| `seatunnel-core` | CLI and submission entrypoints | Owns job submission and command-line capabilities | +| `seatunnel-e2e` | End-to-end test suites | Covers critical regression scenarios | ## 8. Design Principles ### 8.1 Separation of Concerns - **API vs Implementation**: Clean API boundaries enable multiple implementations -- **Coordination vs Execution**: Enumerator/Committer (master) separate from Reader/Writer (worker) +- **Coordination vs Execution**: Enumerator and aggregated-commit orchestration handle coordination, while Reader/Writer execute on workers - **Logical vs Physical**: LogicalDag (user intent) separate from PhysicalPlan (execution details) ### 8.2 Plugin Architecture diff --git a/docs/en/engines/overview.md b/docs/en/engines/overview.md index 424ec54b5162..04f91f5c4fc3 100644 --- a/docs/en/engines/overview.md +++ b/docs/en/engines/overview.md @@ -127,21 +127,30 @@ For most new deployments, SeaTunnel Engine is the recommended default because it ## Decision Flowchart -``` -Start - │ - ▼ -Do you have existing Flink/Spark infrastructure? - │ - ├─ Yes ──► Do you want to reuse it? - │ │ - │ ├─ Yes (Flink) ──► Use Flink Engine - │ │ - │ ├─ Yes (Spark) ──► Use Spark Engine - │ │ - │ └─ No ──► Use SeaTunnel Engine - │ - └─ No ──► Use SeaTunnel Engine (Recommended) +```mermaid +flowchart TD + start["Start"] + infra{"Do you already have
Flink or Spark infrastructure?"} + reuse{"Do you want to reuse it?"} + flink["Use Flink Engine"] + spark["Use Spark Engine"] + zeta["Use SeaTunnel Engine
(Recommended by default)"] + + start --> infra + infra -- "Yes" --> reuse + infra -- "No" --> zeta + reuse -- "Yes, Flink" --> flink + reuse -- "Yes, Spark" --> spark + reuse -- "No" --> zeta + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class start,infra,reuse layerBlue; + class flink,spark layerCyan; + class zeta layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## Configuration Examples diff --git a/docs/en/introduction/how-it-works.md b/docs/en/introduction/how-it-works.md index cbb57aa8b2e5..e7d19575022f 100644 --- a/docs/en/introduction/how-it-works.md +++ b/docs/en/introduction/how-it-works.md @@ -10,30 +10,29 @@ SeaTunnel is a distributed multimodal data integration tool with a pluggable arc This page is the shortest bridge between first-run docs and deeper architecture docs. Read it when you already know SeaTunnel at a high level but still need a practical mental model of how job config, plugins, and engines connect. -``` -┌─────────────────────────────────────────────────────────────┐ -│ Job Configuration │ -│ (HOCON / SQL / Web UI) │ -└─────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ SeaTunnel Core │ -│ (Job Parser, Coordinator, Scheduler) │ -└─────────────────────────────────────────────────────────────┘ - │ - ┌─────────────────────┼─────────────────────┐ - ▼ ▼ ▼ -┌───────────────┐ ┌───────────────┐ ┌───────────────┐ -│ Source │────▶│ Transform │────▶│ Sink │ -│ Connectors │ │ (Optional) │ │ Connectors │ -└───────────────┘ └───────────────┘ └───────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Execution Engine │ -│ SeaTunnel Engine (Zeta) / Flink / Spark │ -└─────────────────────────────────────────────────────────────┘ +```mermaid +flowchart TD + config["Job Configuration
HOCON / SQL / Web UI"] + core["SeaTunnel Core
Job parser / coordinator / scheduler"] + source["Source Connectors"] + transform["Transform (Optional)"] + sink["Sink Connectors"] + engine["Execution Engine
SeaTunnel Engine (Zeta) / Flink / Spark"] + + config --> core + core --> source + source --> transform + transform --> sink + sink --> engine + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class config,core layerBlue; + class source,transform,sink layerCyan; + class engine layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## Recommended Reading Path @@ -73,14 +72,19 @@ Translates SeaTunnel's unified API to engine-specific implementations, enabling ## Data Flow -``` -Source ──▶ [Split] ──▶ Reader ──▶ Transform ──▶ Writer ──▶ Sink - │ │ │ - │ ▼ │ - │ Checkpoint/State │ - │ │ │ - └───────────────────────┴────────────────────────┘ - Fault Tolerance +```mermaid +flowchart LR + source["Source"] --> split["Split"] --> reader["Reader"] --> transform["Transform"] --> writer["Writer"] --> sink["Sink"] + reader -. "Checkpoint / state" .-> recovery["Fault tolerance"] + writer -. "Checkpoint / commit" .-> recovery + source -. "Replay / re-read" .-> recovery + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + + class source,split,reader,transform,writer,sink layerBlue; + class recovery layerCyan; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` **Key Features:** @@ -90,17 +94,16 @@ Source ──▶ [Split] ──▶ Reader ──▶ Transform ──▶ Writer ## Module Structure -``` -seatunnel/ -├── seatunnel-api/ # Core API definitions -├── seatunnel-connectors-v2/ # Source & Sink connectors -├── seatunnel-transforms-v2/ # Transform plugins -├── seatunnel-engine/ # SeaTunnel Engine (Zeta) -├── seatunnel-translation/ # Engine adapters (Flink/Spark) -├── seatunnel-core/ # Job submission & CLI -├── seatunnel-formats/ # Data format handlers -└── seatunnel-e2e/ # End-to-end tests -``` +| Module | Responsibility | +|--------|----------------| +| `seatunnel-api` | Core API definitions | +| `seatunnel-connectors-v2` | Source and sink connectors | +| `seatunnel-transforms-v2` | Transform plugins | +| `seatunnel-engine` | SeaTunnel Engine (Zeta) | +| `seatunnel-translation` | Engine adapters for Flink and Spark | +| `seatunnel-core` | Job submission and CLI | +| `seatunnel-formats` | Data format handlers | +| `seatunnel-e2e` | End-to-end tests | ## Job Execution Flow diff --git a/docs/en/transforms/multi-table-transform-and-join-boundary.md b/docs/en/transforms/multi-table-transform-and-join-boundary.md index f10d52b8ebeb..58af2b78a5f1 100644 --- a/docs/en/transforms/multi-table-transform-and-join-boundary.md +++ b/docs/en/transforms/multi-table-transform-and-join-boundary.md @@ -19,12 +19,28 @@ In a standard single-table pipeline, one Source feeds one Transform chain feeds In a multi-table pipeline, a single Source (e.g., MySQL-CDC) emits records from **many tables** simultaneously, and each downstream Transform or Sink must declare which table(s) it applies to. -``` -MySQL-CDC ──► FieldMapper (orders table) ──► Kafka Sink (orders topic) - │ - ├──► FieldMapper (users table) ──► Kafka Sink (users topic) - │ - └──► (unmatched tables pass through) ──► Elasticsearch Sink +```mermaid +flowchart LR + source["MySQL-CDC"] + ordersMap["FieldMapper
orders table"] + ordersSink["Kafka Sink
orders topic"] + usersMap["FieldMapper
users table"] + usersSink["Kafka Sink
users topic"] + passthrough["Unmatched tables
pass through"] + esSink["Elasticsearch Sink"] + + source --> ordersMap --> ordersSink + source --> usersMap --> usersSink + source --> passthrough --> esSink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source layerBlue; + class ordersMap,usersMap,passthrough layerCyan; + class ordersSink,usersSink,esSink layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` --- @@ -202,18 +218,23 @@ This works only when all three tables (`orders`, `payments`, `refunds`) share `c **EtLT** (Extract, light-transform, Load, then Transform in the warehouse) is the recommended pattern when SeaTunnel's transform layer cannot fulfil the full transformation requirement: -``` -CDC Source - │ - ▼ -Light transforms (field rename, type cast, row filter) - │ - ▼ -Data Lake / Warehouse (Hudi / Iceberg / ClickHouse) - │ - ▼ -Heavy transforms (JOINs, aggregations, complex SQL) -in dbt / Flink SQL / Spark SQL +```mermaid +flowchart TD + cdc["CDC Source"] + light["Light transforms
Field rename / type cast / row filter"] + lake["Data Lake / Warehouse
Hudi / Iceberg / ClickHouse"] + heavy["Heavy transforms
JOINs / aggregations / complex SQL
dbt / Flink SQL / Spark SQL"] + + cdc --> light --> lake --> heavy + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class cdc layerBlue; + class light layerCyan; + class lake,heavy layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` Use SeaTunnel's transform layer for: diff --git a/docs/zh/architecture/api-design/catalog-table.md b/docs/zh/architecture/api-design/catalog-table.md index 0e6333a6b673..5319040e3404 100644 --- a/docs/zh/architecture/api-design/catalog-table.md +++ b/docs/zh/architecture/api-design/catalog-table.md @@ -112,29 +112,21 @@ Column 通常由以下信息构成: ### 4.1 数据源 → 转换器 → 目标端流程 -``` -┌──────────────┐ -│数据源(source) │ -│ │ -│ 生产 │ -│ CatalogTable │ -└──────┬───────┘ - │ - ▼ (输入模式) -┌──────────────┐ -│ 转换器 │ -│ │ -│ 修改 │ -│ CatalogTable │ -└──────┬───────┘ - │ - ▼ (输出模式) -┌──────────────┐ -│ 目标端 │ -│ │ -│ 验证 │ -│ CatalogTable │ -└──────────────┘ +```mermaid +flowchart LR + source["数据源
产出 CatalogTable"] + transform["转换器
更新 CatalogTable"] + sink["目标端
校验 CatalogTable"] + + source -- "输入模式" --> transform + transform -- "输出模式" --> sink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source,sink layerBlue; + class transform layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 4.2 数据源模式生产 diff --git a/docs/zh/architecture/api-design/sink-architecture.md b/docs/zh/architecture/api-design/sink-architecture.md index 6d36021b6cf4..b879609d776a 100644 --- a/docs/zh/architecture/api-design/sink-architecture.md +++ b/docs/zh/architecture/api-design/sink-architecture.md @@ -39,48 +39,34 @@ SeaTunnel 的数据 Sink 旨在： ### 2.1 整体架构 -``` -┌────────────────────────────────────────────────────────────────┐ -│ 执行引擎任务侧（数据面） │ -│ │ -│ ┌──────────────────────────────────────────────────────┐ │ -│ │ SinkWriter │ │ -│ │ │ │ -│ │ • 从上游接收记录 │ │ -│ │ • 缓冲并写入数据 │ │ -│ │ • 在 checkpoint 边界产出 commitInfo │ │ -│ │ • 快照写入器状态 │ │ -│ └──────────────────────────────────────────────────────┘ │ -│ │ │ -│ │ checkpoint 完成通知触发 │ -│ ▼ │ -│ ┌──────────────────────────────────────────────────────┐ │ -│ │ SinkCommitter（可选） │ │ -│ │ │ │ -│ │ • 使 prepare 的变更对外可见 │ │ -│ │ • 失败可重试，要求幂等 │ │ -│ └──────────────────────────────────────────────────────┘ │ -│ │ -└────────────────────────────────────────────────────────────────┘ - │ - │ （可选：聚合提交任务，单实例） - ▼ -┌────────────────────────────────────────────────────────────────┐ -│ 执行引擎协调侧（控制面） │ -│ │ -│ ┌──────────────────────────────────────────────────────┐ │ -│ │ SinkAggregatedCommitter（可选）│ │ -│ │ │ │ -│ │ • 聚合多个 writer 的 commitInfo │ │ -│ │ • 执行一次全局提交（单线程语义） │ -│ └──────────────────────────────────────────────────────┘ │ -│ │ -└────────────────────────────────────────────────────────────────┘ - │ - ▼ - 外部数据系统 - (数据库 / 文件 / 消息队列) +```mermaid +flowchart LR + subgraph worker["执行引擎任务侧（数据面）"] + writer["SinkWriter<IN, CommitInfoT, StateT>
接收上游记录
缓冲并写入数据
在 checkpoint 边界产出 CommitInfo
快照写入器状态"] + committer["SinkCommitter<CommitInfoT>（可选）
由 createCommitter() 创建
在 checkpoint 成功后触发提交
独立提交每个 writer 的变更"] + end + + subgraph coordinator["执行引擎协调侧（仅聚合提交路径）"] + aggregatedTask["SinkAggregatedCommitterTask（可选）
收集各 writer 的 CommitInfo
在协调端执行单次全局提交"] + aggregated["SinkAggregatedCommitter<CommitInfoT, AggregatedCommitInfoT>（可选）
聚合多个 writer 的 CommitInfo
执行一次全局提交"] + end + + sink["外部数据系统
数据库 / 文件 / 消息队列"] + + writer -- "工作节点本地提交路径" --> committer + committer --> sink + writer -. "聚合提交路径" .-> aggregatedTask + aggregatedTask --> aggregated + aggregated --> sink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class worker,coordinator layerBlue; + class writer,committer layerCyan; + class aggregatedTask,aggregated,sink layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.2 核心组件 diff --git a/docs/zh/architecture/api-design/source-architecture.md b/docs/zh/architecture/api-design/source-architecture.md index 8ff8896beb4f..c8c8ff80e27e 100644 --- a/docs/zh/architecture/api-design/source-architecture.md +++ b/docs/zh/architecture/api-design/source-architecture.md @@ -39,44 +39,27 @@ SeaTunnel 的源端 Source 端读取 API 旨在： ### 2.1 整体架构 -``` -┌──────────────────────────────────────────────────────────────┐ -│ 协调端（master/coordinator 侧） │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ SourceSplitEnumerator │ │ -│ │ │ │ -│ │ • 在 run() 中发现/生成分片（实现自定义） │ │ -│ │ • 分配分片给读取器 │ │ -│ │ • 处理读取器注册 │ │ -│ │ • 处理分片请求 │ │ -│ │ • 从失败的读取器回收分片 │ │ -│ │ • 快照枚举器状态 │ │ -│ │ • 发送/接收自定义事件 │ │ -│ └────────────────────────────────────────────────────┘ │ -│ │ │ -└────────────────────────────┼───────────────────────────────────┘ - │ (分片分配) - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ TaskExecutionService（工作节点侧） │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ SourceReader │ │ -│ │ │ │ -│ │ • 接收分配的分片 │ │ -│ │ • 从分片读取数据 │ │ -│ │ • 向下游发送记录 │ │ -│ │ • 快照读取器状态（分片进度） │ │ -│ │ • 处理分片完成 │ │ -│ │ • 发送/接收自定义事件 │ │ -│ └────────────────────────────────────────────────────┘ │ -│ │ │ -└────────────────────────────┼─────────────────────────────────┘ - │ - ▼ - SeaTunnelRow - (到转换/数据 Sink ) +```mermaid +flowchart TD + subgraph coordinator["协调端（master / coordinator 侧）"] + enumerator["SourceSplitEnumerator<SplitT, StateT>
发现或生成分片
分配分片给读取器
处理读取器注册与分片请求
快照枚举器状态"] + end + + subgraph worker["TaskExecutionService（工作节点侧）"] + reader["SourceReader<T, SplitT>
接收分配的分片
从分片读取数据
向下游发送记录
快照读取进度"] + end + + row["SeaTunnelRow
发送到转换 / 数据 Sink"] + + enumerator -- "分片分配" --> reader + reader --> row + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + + class coordinator,worker layerBlue; + class enumerator,reader,row layerCyan; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.2 核心组件 diff --git a/docs/zh/architecture/api-design/translation-layer.md b/docs/zh/architecture/api-design/translation-layer.md index 5f8cff1c23ef..7db68c39b6cb 100644 --- a/docs/zh/architecture/api-design/translation-layer.md +++ b/docs/zh/architecture/api-design/translation-layer.md @@ -29,29 +29,34 @@ SeaTunnel 的转换层旨在: ### 1.3 架构概览 -``` -┌──────────────────────────────────────────────────────────────┐ -│ SeaTunnel API 层 │ -│ (引擎独立的连接器接口) │ -│ │ -│ SeaTunnelSource SeaTunnelSink SeaTunnelTransform │ -└──────────────────────────────────────────────────────────────┘ - │ - │ 转换层 - ┌─────────────┼─────────────┐ - ▼ ▼ ▼ -┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ -│ Flink 适配器 │ │ Spark 适配器 │ │ Zeta (原生) │ -│ │ │ │ │ │ -│ FlinkSource │ │ SparkSource │ │ 直接 │ -│ FlinkSink │ │ SparkSink │ │ 执行 │ -└──────────────────┘ └──────────────────┘ └──────────────────┘ - │ │ │ - ▼ ▼ ▼ -┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ -│ Apache Flink │ │ Apache Spark │ │ SeaTunnel Engine │ -│ 运行时 │ │ 运行时 │ │ (Zeta) │ -└──────────────────┘ └──────────────────┘ └──────────────────┘ +```mermaid +flowchart TB + api["SeaTunnel API 层
引擎无关的连接器接口
SeaTunnelSource / SeaTunnelSink / SeaTunnelTransform"] + + flinkAdapter["Flink 适配器
FlinkSource / FlinkSink"] + sparkAdapter["Spark 适配器
SparkSource / SparkSink"] + zetaAdapter["Zeta（原生）
直接执行"] + + flinkRuntime["Apache Flink 运行时"] + sparkRuntime["Apache Spark 运行时"] + zetaRuntime["SeaTunnel Engine (Zeta)"] + + api --> flinkAdapter + api --> sparkAdapter + api --> zetaAdapter + + flinkAdapter --> flinkRuntime + sparkAdapter --> sparkRuntime + zetaAdapter --> zetaRuntime + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class api layerBlue; + class flinkAdapter,sparkAdapter,zetaAdapter layerCyan; + class flinkRuntime,sparkRuntime,zetaRuntime layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 1.4 推荐阅读路径 diff --git a/docs/zh/architecture/engine/dag-execution.md b/docs/zh/architecture/engine/dag-execution.md index 765f47d04272..8a468d7b9dcb 100644 --- a/docs/zh/architecture/engine/dag-execution.md +++ b/docs/zh/architecture/engine/dag-execution.md @@ -29,47 +29,25 @@ SeaTunnel 的 DAG 执行模型旨在: ### 1.3 执行模型概览 -``` -用户配置 (HOCON) - │ - ▼ -┌─────────────────────┐ -│ LogicalDag │ 逻辑计划 (做什么) -│ • LogicalVertex │ - 数据 Source/tranform 转换器/Sink 目标端动作 -│ • LogicalEdge │ - 数据依赖关系 -│ • Parallelism │ - 逻辑并行度 -└─────────────────────┘ - │ (计划生成) - ▼ -┌─────────────────────┐ -│ PhysicalPlan │ 物理计划 (如何执行) -│ • SubPlan[] │ - 多个流水线 -│ • Resources │ - 资源需求 -│ • Scheduling │ - 部署策略 -└─────────────────────┘ - │ (流水线分割) - ▼ -┌─────────────────────┐ -│ SubPlan (Pipeline) │ 独立执行单元 -│ • PhysicalVertex[] │ - 并行任务实例 -│ • CheckpointCoord │ - 独立检查点 -│ • PipelineLocation │ - 唯一标识符 -└─────────────────────┘ - │ (任务部署) - ▼ -┌─────────────────────┐ -│ PhysicalVertex │ 已部署任务组 -│ • TaskGroup │ - 共址任务(融合) -│ • SlotProfile │ - 分配的资源槽位 -│ • ExecutionState │ - 运行状态 -└─────────────────────┘ - │ (执行) - ▼ -┌─────────────────────┐ -│ SeaTunnelTask │ 实际执行 -│ • Source/Transform │ - 数据处理 -│ • /Sink Logic │ - 状态管理 -└─────────────────────┘ +```mermaid +flowchart TD + config["用户配置
HOCON"] + logical["LogicalDag
LogicalVertex / LogicalEdge
逻辑并行度"] + physical["PhysicalPlan
SubPlan 列表
资源需求
调度策略"] + pipeline["SubPlan（Pipeline）
独立执行单元
PhysicalVertex 列表
CheckpointCoordinator"] + vertex["PhysicalVertex
TaskGroup
SlotProfile
ExecutionState"] + task["SeaTunnelTask
实际执行 Source / Transform / Sink"] + + config --> logical --> physical --> pipeline --> vertex --> task + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class config,logical layerBlue; + class physical,pipeline layerCyan; + class vertex,task layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## 2. LogicalDag: 用户意图 @@ -159,14 +137,19 @@ sink { ``` 生成的 LogicalDag: -``` -Vertex 1 (JDBC 数据源, parallelism=4) - │ - ▼ -Vertex 2 (SQL 转换器, parallelism=4) - │ - ▼ -Vertex 3 (Elasticsearch 目标端, parallelism=4) +```mermaid +flowchart TD + v1["Vertex 1
JDBC 数据源
parallelism = 4"] + v2["Vertex 2
SQL 转换器
parallelism = 4"] + v3["Vertex 3
Elasticsearch 目标端
parallelism = 4"] + v1 --> v2 --> v3 + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class v1,v3 layerBlue; + class v2 layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## 3. PhysicalPlan: 执行策略 @@ -200,9 +183,9 @@ sink { Elasticsearch { } } ``` 生成: **1 个流水线** -``` -流水线 1: [JDBC 数据源] → [SQL 转换器] → [Elasticsearch 目标端] -``` +生成结果: + +- `流水线 1`: `JDBC 数据源 → SQL 转换器 → Elasticsearch 目标端` **示例 2: 多个数据源**: ```hocon @@ -221,10 +204,10 @@ sink { ``` 生成: **2 个流水线** -``` -流水线 1: [JDBC 数据源] → [SQL 转换器] → [Elasticsearch 目标端] -流水线 2: [Kafka 数据源] → [SQL 转换器] → [Elasticsearch 目标端] -``` +生成结果: + +- `流水线 1`: `JDBC 数据源 → SQL 转换器 → Elasticsearch 目标端` +- `流水线 2`: `Kafka 数据源 → SQL 转换器 → Elasticsearch 目标端` **示例 3: 多个目标端**: ```hocon @@ -239,9 +222,19 @@ sink { ``` 生成: **通常为 1 个流水线（包含分支）** -``` -流水线 1: [MySQL-CDC 数据源] → [Elasticsearch 目标端] - └──────→ [JDBC 目标端] +生成结果: + +```mermaid +flowchart LR + source["MySQL-CDC 数据源"] --> elastic["
Elasticsearch
"] + source --> jdbc["
JDBC
"] + + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source layerCyan; + class elastic,jdbc layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 3.3 PhysicalPlan 生成 @@ -274,15 +267,9 @@ SubPlan(流水线)通常包含: 每个并行度为 N 的 LogicalVertex 生成 N 个 PhysicalVertices。 **示例**: -``` -LogicalVertex: JDBC 数据源 (parallelism = 4) - ↓ -PhysicalVertices: - - PhysicalVertex (子任务 0, 槽位 1) - - PhysicalVertex (子任务 1, 槽位 2) - - PhysicalVertex (子任务 2, 槽位 3) - - PhysicalVertex (子任务 3, 槽位 4) -``` +| 逻辑顶点 | 生成的物理顶点 | +|----------|----------------| +| `JDBC 数据源 (parallelism = 4)` | `PhysicalVertex` 子任务 `0` 落在槽位 `1`，子任务 `1` 落在槽位 `2`，子任务 `2` 落在槽位 `3`，子任务 `3` 落在槽位 `4` | ### 4.3 协调器顶点 @@ -294,17 +281,10 @@ PhysicalVertices: 说明：`SinkCommitter` 的触发方式取决于引擎实现，并不一定体现为独立的协调器顶点；例如在 SeaTunnel Engine 中，committer 可能在 Sink 任务的 checkpoint 回调中被触发。 **示例**: -``` -JDBC → Transform → Elasticsearch 的 SubPlan: - physicalVertexList: - - JdbcSourceTask (4 个实例) - - TransformTask (4 个实例) - - ElasticsearchSinkTask (4 个实例) - - coordinatorVertexList: - - JdbcSourceSplitEnumerator (1 个实例) - - ElasticsearchSinkAggregatedCommitter (1 个实例，可选) -``` +| 运行时范围 | 实例 | +|------------|------| +| `physicalVertexList` | `JdbcSourceTask × 4`、`TransformTask × 4`、`ElasticsearchSinkTask × 4` | +| `coordinatorVertexList` | `JdbcSourceSplitEnumerator × 1`，以及 `ElasticsearchSinkAggregatedCommitter × 1`（可选） | ### 4.4 独立检查点 @@ -317,15 +297,10 @@ JDBC → Transform → Elasticsearch 的 SubPlan: - 简化屏障对齐 **示例**: -``` -流水线 1 (JDBC → ES): - CheckpointCoordinator 按作业配置的间隔触发 - 仅管理 JDBC 和 ES 任务的检查点 - -流水线 2 (Kafka → JDBC): - CheckpointCoordinator 按作业配置的间隔触发 - 仅管理 Kafka 和 JDBC 任务的检查点 -``` +| 流水线 | 检查点行为 | +|--------|------------| +| `流水线 1 (JDBC → ES)` | 一个 `CheckpointCoordinator` 按作业配置的间隔触发，仅管理 JDBC 和 Elasticsearch 任务 | +| `流水线 2 (Kafka → JDBC)` | 另一个协调器同样按作业配置触发，但只管理 Kafka 和 JDBC 任务 | ## 5. PhysicalVertex: 已部署任务 @@ -355,18 +330,11 @@ TaskGroup 的关键点: 3. 不需要数据混洗 **示例(带融合)**: -``` -LogicalDag: - Source (parallelism=4) → Transform (parallelism=4) → Sink (parallelism=4) - -不融合: - 12 个独立任务(4 + 4 + 4) - Source → Transform 和 Transform → Sink 有网络开销 - -融合后: - 4 个 TaskGroups,每个包含: - [SourceTask → TransformTask → SinkTask] (单线程,共享内存) -``` +| 模式 | 执行形态 | 影响 | +|------|----------|------| +| 逻辑 DAG | `Source (4) → Transform (4) → Sink (4)` | 两种模式都保留相同的业务拓扑 | +| 不融合 | `12` 个独立任务，阶段之间走网络传输 | 序列化和网络开销更高 | +| 融合后 | `4` 个 `TaskGroup`，每组执行 `SourceTask → TransformTask → SinkTask` | 本地性更好，网络成本更低 | **优势**: - 减少网络序列化/反序列化 @@ -481,17 +449,11 @@ sink { ### 7.3 资源分配 **槽位计算**: -``` -所需槽位 = 所有任务并行度之和 +槽位估算经验: -示例: - Source (parallelism=4) + Transform (parallelism=4) + Sink (parallelism=2) - = 需要 10 个槽位 - -融合后: - TaskGroup (parallelism=4, fusion[Source+Transform]) + Sink (parallelism=2) - = 需要 6 个槽位 -``` +- `所需槽位 = 所有任务并行度之和` +- 不融合示例: `Source (4) + Transform (4) + Sink (2) = 10 个槽位` +- 融合示例: `TaskGroup (4, Source+Transform 融合) + Sink (2) = 6 个槽位` 说明：资源画像/槽位资源的具体字段、单位与配置路径以引擎侧配置与实现为准；文档不在此给出不存在或不稳定的配置项示例。 @@ -515,15 +477,10 @@ sink { **关键见解**: 流水线故障是隔离的。 **示例**: -``` -有 2 个流水线的作业: - 流水线 1: JDBC → ES (RUNNING) - 流水线 2: Kafka → JDBC (FAILED) - -结果: - 流水线 2 从检查点重启 - 流水线 1 继续不受影响 -``` +| 流水线 | 状态 | 恢复结果 | +|--------|------|----------| +| `流水线 1` | `RUNNING` | 持续运行，不受其他流水线影响 | +| `流水线 2` | `FAILED` | 从最近一次检查点重启 | **优势**: - 减少爆炸半径 @@ -550,32 +507,21 @@ sink { ### 9.2 可视化 -``` -作业: mysql-to-es -│ -├── 流水线 1 (mysql-cdc → elasticsearch) -│ ├── PhysicalVertex 0 [RUNNING] @ worker-1:slot-1 -│ ├── PhysicalVertex 1 [RUNNING] @ worker-2:slot-1 -│ ├── PhysicalVertex 2 [RUNNING] @ worker-3:slot-1 -│ └── PhysicalVertex 3 [RUNNING] @ worker-4:slot-1 -│ -└── 流水线 2 (mysql-cdc → jdbc) - ├── PhysicalVertex 0 [RUNNING] @ worker-1:slot-2 - └── PhysicalVertex 1 [RUNNING] @ worker-2:slot-2 -``` +| 流水线 | 顶点部署 | +|--------|----------| +| `流水线 1 (mysql-cdc → elasticsearch)` | `PhysicalVertex 0 @ worker-1:slot-1`，`1 @ worker-2:slot-1`，`2 @ worker-3:slot-1`，`3 @ worker-4:slot-1` | +| `流水线 2 (mysql-cdc → jdbc)` | `PhysicalVertex 0 @ worker-1:slot-2`，`1 @ worker-2:slot-2` | ## 10. 最佳实践 ### 10.1 并行度配置 **经验法则**: -``` -并行度 = min( - 数据分区数, - 可用槽位数, - 目标吞吐量 / 单任务吞吐量 -) -``` +并行度优先取以下几个约束中的较小值: + +- 数据分区数 +- 可用槽位数 +- `目标吞吐量 / 单任务吞吐量` **示例**: - **JDBC 数据源**: 设置为数据库分区数(例如 8 个分区 → parallelism=8) diff --git a/docs/zh/architecture/engine/engine-architecture.md b/docs/zh/architecture/engine/engine-architecture.md index ca1980498889..7122b9efaf54 100644 --- a/docs/zh/architecture/engine/engine-architecture.md +++ b/docs/zh/architecture/engine/engine-architecture.md @@ -42,73 +42,41 @@ SeaTunnel 引擎（Zeta）设计为原生执行引擎，具有： ### 2.1 主-工架构 +```mermaid +flowchart TB + subgraph master["主节点"] + direction TB + coordinator["CoordinatorService
管理运行作业 / 生命周期 / IMap 状态"] + jobmaster["JobMaster（每个作业一个）
生成物理计划 / 请求资源 / 部署任务 / 协调检查点"] + checkpoint["CheckpointManager
每条管道一个"] + resource["ResourceManager
槽位分配 / 工作节点注册 / 负载均衡"] + coordinator --> jobmaster + jobmaster --> checkpoint + jobmaster --> resource + end + + subgraph worker["工作节点"] + direction TB + execution["TaskExecutionService
部署执行任务 / 生命周期 / 心跳 / 槽位资源"] + source["SourceFlowLifeCycle
SourceReader / SeaTunnelSourceCollector"] + transform["TransformFlowLifeCycle
转换链"] + sink["SinkFlowLifeCycle
SinkWriter"] + execution --> source --> transform --> sink + end + + resource -. "Hazelcast 集群" .-> execution + jobmaster -. "任务部署" .-> execution + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class coordinator,jobmaster,checkpoint layerBlue; + class resource,execution layerCyan; + class source,transform,sink layerPurple; + style master fill:#081425,stroke:#5db8e2,stroke-width:1.5px,color:#f8fbff; + style worker fill:#081425,stroke:#8d7cf6,stroke-width:1.5px,color:#f8fbff; ``` -┌─────────────────────────────────────────────────────────────────┐ -│ 主节点 │ -│ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ CoordinatorService │ │ -│ │ • 管理所有运行中的作业 │ │ -│ │ • 作业提交和生命周期管理 │ │ -│ │ • 维护作业状态（IMap） │ │ -│ │ • 资源管理器工厂 │ │ -│ └───────────────────────────────────────────────────────┘ │ -│ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ JobMaster（每个作业一个） │ │ -│ │ • 生成物理执行计划 │ │ -│ │ • 从 ResourceManager 请求资源 │ │ -│ │ • 将任务部署到工作节点 │ │ -│ │ • 协调检查点 │ │ -│ │ • 处理故障转移和恢复 │ │ -│ └───────────────────────────────────────────────────────┘ │ -│ │ │ │ -│ │ (任务部署) │ (资源请求) │ -│ ▼ ▼ │ -│ ┌─────────────────┐ ┌────────────────────────────┐ │ -│ │ CheckpointManager│ │ ResourceManager │ │ -│ │ (每个管道) │ │ • 槽位分配 │ │ -│ └─────────────────┘ │ • 工作节点注册 │ │ -│ │ • 负载均衡 │ │ -│ └────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ - │ - │ (Hazelcast 集群) - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ 工作节点 │ -│ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ TaskExecutionService │ │ -│ │ • 部署和执行任务 │ │ -│ │ • 管理任务生命周期 │ │ -│ │ • 报告心跳 │ │ -│ │ • 槽位资源管理 │ │ -│ └───────────────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ SeaTunnelTask（每个工作节点多个） │ │ -│ │ │ │ -│ │ ┌─────────────────────────────────────────────┐ │ │ -│ │ │ SourceFlowLifeCycle │ │ │ -│ │ │ • SourceReader │ │ │ -│ │ │ • SeaTunnelSourceCollector │ │ │ -│ │ └─────────────────────────────────────────────┘ │ │ -│ │ │ │ │ -│ │ ▼ │ │ -│ │ ┌─────────────────────────────────────────────┐ │ │ -│ │ │ TransformFlowLifeCycle │ │ │ -│ │ │ • 转换链 │ │ │ -│ │ └─────────────────────────────────────────────┘ │ │ -│ │ │ │ │ -│ │ ▼ │ │ -│ │ ┌─────────────────────────────────────────────┐ │ │ -│ │ │ SinkFlowLifeCycle │ │ │ -│ │ │ • SinkWriter │ │ │ -│ │ └─────────────────────────────────────────────┘ │ │ -│ └───────────────────────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ ``` ### 2.2 核心组件 @@ -142,9 +110,7 @@ SeaTunnel 引擎（Zeta）设计为原生执行引擎，具有： - 处理任务失败并重新调度 **生命周期**： -``` -Created → Initialized → Scheduled → Running → Finished/Failed/Canceled -``` +`Created → Initialized → Scheduled → Running → Finished / Failed / Canceled` **关键操作**： 1. `init()`：生成物理计划，创建检查点协调器 @@ -172,50 +138,26 @@ Created → Initialized → Scheduled → Running → Finished/Failed/Canceled ### 3.1 执行计划转换 -``` -用户配置（HOCON） - │ - ▼ -┌───────────────┐ -│ LogicalDag │ • 逻辑顶点（数据源/转换/数据 Sink ） -│ │ • 逻辑边（数据流） -│ │ • 并行度（每个顶点） -└───────────────┘ - │ (JobMaster.generatePhysicalPlan()) - ▼ -┌───────────────┐ -│ PhysicalPlan │ • SubPlan 列表（管道） -│ │ • JobImmutableInformation -│ │ • 资源要求 -└───────────────┘ - │ - ▼ -┌───────────────┐ -│ SubPlan │ • 管道（独立执行单元） -│ (Pipeline) │ • PhysicalVertex 列表 -│ │ • CheckpointCoordinator -└───────────────┘ - │ - ▼ -┌───────────────┐ -│PhysicalVertex │ • TaskGroup（共存任务） -│ │ • 分配的 SlotProfile -│ │ • ExecutionState -└───────────────┘ - │ - ▼ -┌───────────────┐ -│ TaskGroup │ • 多个 SeaTunnelTask 实例 -│ │ • 共享网络缓冲区 -│ │ • 线程池 -└───────────────┘ - │ - ▼ -┌───────────────┐ -│ SeaTunnelTask │ • 单个任务执行 -│ │ • 数据源/转换/数据 Sink 生命周期 -│ │ • 任务状态机 -└───────────────┘ +```mermaid +flowchart TD + config["用户配置
HOCON"] + logical["LogicalDag
逻辑顶点 / 逻辑边 / 并行度"] + plan["PhysicalPlan
SubPlan 列表 / JobImmutableInformation / 资源要求"] + subplan["SubPlan（Pipeline）
独立执行单元 / PhysicalVertex 列表 / CheckpointCoordinator"] + vertex["PhysicalVertex
TaskGroup / SlotProfile / ExecutionState"] + group["TaskGroup
多个 SeaTunnelTask 实例 / 共享缓冲区 / 线程池"] + task["SeaTunnelTask
任务执行 / 生命周期 / 状态机"] + + config --> logical --> plan --> subplan --> vertex --> group --> task + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class config,logical,plan layerBlue; + class subplan,vertex layerCyan; + class group,task layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 3.2 LogicalDag @@ -303,43 +245,36 @@ sink { ### 4.1 任务状态机 -``` - [Created] - │ - ▼ - [INIT] ────────────────────────────────────┐ - │ │ - ▼ │ -[WAITING_RESTORE]（如果恢复中） │ - │ │ - ▼ │ - [READY_START] │ - │ │ - ▼ │ - [STARTING] ──────────────┐ │ - │ │ │ - ▼ ▼ ▼ - [RUNNING] ──────────> [FAILED] ─────> (重启) - │ - ▼ -[PREPARE_CLOSE] - │ - ▼ - [CLOSED] - │ - ▼ - [CANCELED]（如果作业取消） +```mermaid +stateDiagram-v2 + [*] --> CREATED + CREATED --> INIT + INIT --> WAITING_RESTORE: 恢复路径 + INIT --> READY_START: 无需恢复 + WAITING_RESTORE --> READY_START + READY_START --> STARTING + STARTING --> RUNNING + RUNNING --> PREPARE_CLOSE: 正常完成 + PREPARE_CLOSE --> CLOSED + INIT --> CANCELLING: 外部取消 + WAITING_RESTORE --> CANCELLING + READY_START --> CANCELLING + STARTING --> CANCELLING + RUNNING --> CANCELLING + PREPARE_CLOSE --> CANCELLING + CANCELLING --> CANCELED ``` **状态转换**： -1. **CREATED → INIT**：任务已创建，初始化资源 -2. **INIT → WAITING_RESTORE**：从检查点恢复 -3. **WAITING_RESTORE → READY_START**：状态已恢复 -4. **READY_START → STARTING**：打开数据源/转换/数据 Sink -5. **STARTING → RUNNING**：数据处理已启动 -6. **RUNNING → PREPARE_CLOSE**：正常完成 -7. **PREPARE_CLOSE → CLOSED**：资源已清理 -8. **RUNNING → FAILED**：发生异常 +1. **CREATED → INIT**：任务已创建，并完成运行时资源初始化 +2. **INIT → WAITING_RESTORE / READY_START**：根据是否需要恢复，进入恢复路径或直接启动路径 +3. **WAITING_RESTORE → READY_START**：状态恢复完成，准备打开各生命周期组件 +4. **READY_START → STARTING → RUNNING**：收到启动信号后进入正式处理阶段 +5. **RUNNING → PREPARE_CLOSE → CLOSED**：正常完成并清理资源 +6. **活动状态 → CANCELLING → CANCELED**：外部取消路径，与正常完成链路分开处理 + +**失败说明**： +- `FAILED` 是运行时对不可恢复错误的结果标记，但“失败后是否重启”由更高层的恢复逻辑决定，不应在这个任务状态机图里画成 `FAILED → ...` 的直接边。 ### 4.2 SeaTunnelTask 执行 diff --git a/docs/zh/architecture/engine/resource-management.md b/docs/zh/architecture/engine/resource-management.md index 0b1c44dbd966..81677581bc27 100644 --- a/docs/zh/architecture/engine/resource-management.md +++ b/docs/zh/architecture/engine/resource-management.md @@ -29,55 +29,25 @@ SeaTunnel 的资源管理系统旨在: ### 1.3 架构概览 -``` -┌──────────────────────────────────────────────────────────────┐ -│ JobMaster │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ 请求资源 │ │ -│ │ • 计算所需槽位 │ │ -│ │ • （可选）表达资源需求（以当前引擎实现为准） │ │ -│ │ • 应用标签过滤器(可选) │ │ -│ └────────────────────────────────────────────────────┘ │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ ResourceManager │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ 工作节点注册表 │ │ -│ │ • WorkerProfile (每个工作节点) │ │ -│ │ - 总资源 │ │ -│ │ - 可用资源 │ │ -│ │ - 已分配槽位 │ │ -│ │ - 未分配槽位 │ │ -│ └────────────────────────────────────────────────────┘ │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ 分配策略 │ │ -│ │ • RandomStrategy / SlotRatioStrategy / SystemLoadStrategy│ │ -│ └────────────────────────────────────────────────────┘ │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ 槽位管理 │ │ -│ │ • 分配槽位 │ │ -│ │ • 释放槽位 │ │ -│ │ • 跟踪槽位使用 │ │ -│ └────────────────────────────────────────────────────┘ │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ 工作节点 │ -│ │ -│ Worker 1 Worker 2 Worker N │ -│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ -│ │ Slot 1 │ │ Slot 1 │ │ Slot 1 │ │ -│ │ Slot 2 │ │ Slot 2 │ │ Slot 2 │ │ -│ │ ... │ │ ... │ │ ... │ │ -│ └──────────┘ └──────────┘ └──────────┘ │ -└──────────────────────────────────────────────────────────────┘ +```mermaid +flowchart TD + jobmaster["JobMaster
计算所需槽位
表达资源需求（可选）
应用标签过滤器（可选）"] + registry["工作节点注册表
WorkerProfile：总资源 / 可用资源 / 已分配槽位 / 未分配槽位"] + strategy["分配策略
Random / SlotRatio / SystemLoad"] + slots["槽位管理
分配槽位 / 释放槽位 / 跟踪使用情况"] + workers["工作节点池
Worker 1 / Worker 2 / Worker N
每个节点维护多个 Slot"] + + jobmaster --> registry + registry --> strategy --> slots --> workers + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class jobmaster layerBlue; + class registry,strategy,slots layerCyan; + class workers layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## 2. 核心概念 diff --git a/docs/zh/architecture/fault-tolerance/checkpoint-mechanism.md b/docs/zh/architecture/fault-tolerance/checkpoint-mechanism.md index 3f234e8a3d38..dce94220d3e5 100644 --- a/docs/zh/architecture/fault-tolerance/checkpoint-mechanism.md +++ b/docs/zh/architecture/fault-tolerance/checkpoint-mechanism.md @@ -44,83 +44,27 @@ SeaTunnel 的检查点基于 **Chandy-Lamport 分布式快照算法**： ### 2.1 检查点架构 -``` -┌─────────────────────────────────────────────────────────────────┐ -│ JobMaster（每个作业一个，内部按 pipeline 管理） │ -│ │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ CheckpointCoordinator │ │ -│ │ │ │ -│ │ • 触发检查点（定期/手动） │ │ -│ │ • 生成检查点 ID │ │ -│ │ • 跟踪待处理的检查点 │ │ -│ │ • 收集任务确认 │ │ -│ │ • 持久化完成的检查点 │ │ -│ │ • 清理旧检查点 │ │ -│ └───────────────────────────────────────────────────────┘ │ -│ │ │ -│ │ (触发屏障) │ -│ ▼ │ -└─────────────────────────────────────────────────────────────────┘ - │ - │ (CheckpointBarrier) - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ 工作节点 │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ SourceTask 1 │ │ SourceTask 2 │ │ SourceTask N │ │ -│ │ │ │ │ │ │ │ -│ │ 1. 接收 │ │ 1. 接收 │ │ 1. 接收 │ │ -│ │ 屏障 │ │ 屏障 │ │ 屏障 │ │ -│ │ 2. 快照 │ │ 2. 快照 │ │ 2. 快照 │ │ -│ │ 状态 │ │ 状态 │ │ 状态 │ │ -│ │ 3. ACK │ │ 3. ACK │ │ 3. ACK │ │ -│ │ 4. 转发 │ │ 4. 转发 │ │ 4. 转发 │ │ -│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ -│ │ │ │ │ -│ │ (屏障传播) │ │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ Transform 1 │ │ Transform 2 │ │ Transform N │ │ -│ │ │ │ │ │ │ │ -│ │ 1. 接收 │ │ 1. 接收 │ │ 1. 接收 │ │ -│ │ 屏障 │ │ 屏障 │ │ 屏障 │ │ -│ │ 2. 快照 │ │ 2. 快照 │ │ 2. 快照 │ │ -│ │ 状态 │ │ 状态 │ │ 状态 │ │ -│ │ 3. ACK │ │ 3. ACK │ │ 3. ACK │ │ -│ │ 4. 转发 │ │ 4. 转发 │ │ 4. 转发 │ │ -│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ SinkTask 1 │ │ SinkTask 2 │ │ SinkTask N │ │ -│ │ │ │ │ │ │ │ -│ │ 1. 接收 │ │ 1. 接收 │ │ 1. 接收 │ │ -│ │ 屏障 │ │ 屏障 │ │ 屏障 │ │ -│ │ 2. 准备 │ │ 2. 准备 │ │ 2. 准备 │ │ -│ │ 提交 │ │ 提交 │ │ 提交 │ │ -│ │ 3. 快照 │ │ 3. 快照 │ │ 3. 快照 │ │ -│ │ 状态 │ │ 状态 │ │ 状态 │ │ -│ │ 4. ACK │ │ 4. ACK │ │ 4. ACK │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ - │ - │ (收到所有 ACK) - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ CheckpointStorage │ -│ （例如 localfile/hdfs 等，取决于插件与配置） │ -│ │ -│ CompletedCheckpoint { │ -│ checkpointId: 123 │ -│ taskStates: { │ -│ SourceTask-1: { splits: [...], offsets: [...] } │ -│ SinkTask-1: { commitInfo: XidInfo(...) } │ -│ ... │ -│ } │ -│ } │ -└─────────────────────────────────────────────────────────────────┘ +```mermaid +flowchart TD + coordinator["CheckpointCoordinator
触发检查点 / 生成 checkpointId / 跟踪 PendingCheckpoint
收集 ACK / 持久化 CompletedCheckpoint / 清理旧检查点"] + source["SourceTask
接收屏障 / 快照状态 / ACK / 转发屏障"] + transform["TransformTask
接收屏障 / 快照状态 / ACK / 转发屏障"] + sink["SinkTask
接收屏障 / prepareCommit / 快照状态 / ACK"] + storage["CheckpointStorage
CompletedCheckpoint
checkpointId + taskStates"] + + coordinator -->|触发屏障| source + source -->|屏障传播| transform + transform -->|屏障传播| sink + sink -->|收到所有 ACK 后持久化| storage + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class coordinator,storage layerBlue; + class source,transform layerCyan; + class sink layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.2 关键数据结构 diff --git a/docs/zh/architecture/fault-tolerance/exactly-once.md b/docs/zh/architecture/fault-tolerance/exactly-once.md index 4719405b21ba..a86a513633e6 100644 --- a/docs/zh/architecture/fault-tolerance/exactly-once.md +++ b/docs/zh/architecture/fault-tolerance/exactly-once.md @@ -16,15 +16,10 @@ title: 精确一次语义 - **精确一次**: 每条记录恰好处理一次(理想但复杂) **实际影响**: -``` -场景: 金融交易处理 - -至少一次: - 交易 $100 处理两次 → 用户被收费 $200 ❌ - -精确一次: - 交易 $100 处理一次 → 用户被收费 $100 ✅ -``` +| 场景 | 结果 | +|------|------| +| 至少一次 | `交易 $100` 被处理两次，用户被收费 `$200` | +| 精确一次 | `交易 $100` 只处理一次，用户被收费 `$100` | ### 1.2 设计目标 @@ -75,48 +70,26 @@ SeaTunnel 的精确一次语义旨在: ### 3.1 端到端流水线 -``` -┌──────────────────────────────────────────────────────────────┐ -│ 数据源 │ -│ • 从外部系统读取 │ -│ • 跟踪偏移量/位置 │ -│ • 在检查点中快照偏移量 │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ 检查点屏障 -┌──────────────────────────────────────────────────────────────┐ -│ 转换器 │ -│ • 处理记录 │ -│ • 快照转换器状态(如果有) │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ 检查点屏障 -┌──────────────────────────────────────────────────────────────┐ -│ 目标端写入器 │ -│ • 缓冲写入 │ -│ • prepareCommit(checkpointId) → 生成 CommitInfo (阶段 1) │ -│ • 快照写入器状态 │ -└──────────────────────────────┬───────────────────────────────┘ - │ - │ CommitInfo - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ CheckpointCoordinator │ -│ • 收集所有 CommitInfos │ -│ • 持久化 CompletedCheckpoint │ -│ • 触发提交/回调（触发点取决于执行引擎实现） │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ 目标端提交器 │ -│ • commit(CommitInfos) → 应用变更 (阶段 2) │ -│ • 必须是幂等的 │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ - 外部目标端 - (变更可见) +```mermaid +flowchart TD + source["数据源
读取外部系统 / 跟踪偏移量 / 检查点时快照进度"] + transform["转换器
处理记录 / 快照算子状态"] + writer["目标端写入器
缓冲写入 / prepareCommit(checkpointId)
快照 writer 状态"] + coordinator["CheckpointCoordinator
收集 CommitInfo / 持久化 CompletedCheckpoint
触发提交回调"] + committer["目标端提交器
commit(CommitInfos)
必须幂等"] + sink["外部目标端
变更变为可见"] + + source -->|检查点屏障| transform -->|检查点屏障| writer + writer -->|CommitInfo| coordinator --> committer --> sink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source,sink layerBlue; + class transform,writer layerCyan; + class coordinator,committer layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 3.2 关键组件 diff --git a/docs/zh/architecture/features/multi-table.md b/docs/zh/architecture/features/multi-table.md index 611656226c37..cbc3299161b9 100644 --- a/docs/zh/architecture/features/multi-table.md +++ b/docs/zh/architecture/features/multi-table.md @@ -165,16 +165,10 @@ MultiTableSink 是一个“按表路由 + 可多副本并行写入”的 Sink: **解决方案**: 每张表多个副本写入器用于并行写入。 -``` -无副本: - orders 表(1000 写入/秒) → [单个写入器] → 瓶颈 - -有副本(replicaNum=4): - orders 表(1000 写入/秒) → [写入器 0] (250 写入/秒) - → [写入器 1] (250 写入/秒) - → [写入器 2] (250 写入/秒) - → [写入器 3] (250 写入/秒) -``` +| 模式 | 写入形态 | 结果 | +|------|----------|------| +| 无副本 | `orders` 表 `1000` 写入/秒全部落到单个写入器 | 单点写入器成为瓶颈 | +| `replicaNum = 4` | `orders` 表流量平均分散到 `4` 个写入器，每个约 `250` 写入/秒 | 吞吐更平稳，可横向扩展 | ### 5.2 副本配置 @@ -223,43 +217,38 @@ sink { ### 7.1 完整流水线 -``` -┌──────────────────────────────────────────────────────────────┐ -│ MySQL CDC 数据源 │ -│ • 从 100 张表捕获变更 │ -│ • 用 TablePath 标记每行 │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ▼ - ┌─────────────────────────────────────┐ - │ SeaTunnelRow (带 TablePath) │ - │ tableId: "my_db.public.orders" │ - │ fields: [1, "order-001", 99.99] │ - └─────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ MultiTableSinkWriter │ -│ • 从行中提取 TablePath │ -│ • 选择副本（按主键哈希或随机） │ -│ • 路由到正确的写入器 │ -└──────────────────────────────┬───────────────────────────────┘ - │ - ┌──────────────────┼──────────────────┐ - ▼ ▼ ▼ -┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ orders │ │ users │ │ products │ -│ 写入器 0 │ │ 写入器 0 │ │ 写入器 0 │ -│ 写入器 1 │ │ 写入器 1 │ │ 写入器 1 │ -│ 写入器 2 │ │ │ │ │ -│ 写入器 3 │ │ │ │ │ -└──────────────┘ └──────────────┘ └──────────────┘ - │ │ │ - ▼ ▼ ▼ -┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ PostgreSQL │ │ PostgreSQL │ │ PostgreSQL │ -│ orders │ │ users │ │ products │ -└──────────────┘ └──────────────┘ └──────────────┘ +```mermaid +flowchart TD + source["MySQL CDC 数据源
从 100 张表捕获变更
每行携带 TablePath"] + row["SeaTunnelRow
tableId = my_db.public.orders
fields = [1, order-001, 99.99]"] + writer["MultiTableSinkWriter
提取 TablePath
选择副本（主键哈希或随机）
路由到正确写入器"] + + subgraph tables["按表拆分的写入器副本"] + direction LR + orders["orders
写入器 0 / 1 / 2 / 3"] + users["users
写入器 0 / 1"] + products["products
写入器 0 / 1"] + end + + sink["PostgreSQL 目标表
orders / users / products"] + + source --> row --> writer + writer --> orders + writer --> users + writer --> products + orders --> sink + users --> sink + products --> sink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source,row layerBlue; + class writer layerCyan; + class orders,users,products,sink layerPurple; + style tables fill:#081425,stroke:#5db8e2,stroke-width:1.5px,color:#f8fbff; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 7.2 写入流程 diff --git a/docs/zh/architecture/overview.md b/docs/zh/architecture/overview.md index c8947478859d..bdaa164d2a31 100644 --- a/docs/zh/architecture/overview.md +++ b/docs/zh/architecture/overview.md @@ -42,49 +42,26 @@ SeaTunnel 设计为分布式多模态数据集成工具，具有以下核心目 SeaTunnel 采用分层架构，实现关注点分离和灵活性： -``` -┌─────────────────────────────────────────────────────────────────┐ -│ 用户配置层 │ -│ (HOCON 配置 / SQL) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ SeaTunnel API 层 │ -│ (数据源 API / 数据 Sink API / 转换 API / 表 API) │ -│ │ -│ • SeaTunnelSource • CatalogTable │ -│ • SeaTunnelSink • TableSchema │ -│ • SeaTunnelTransform • SchemaChangeEvent │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ 连接器生态系统 │ -│ │ -│ [Jdbc] [Kafka] [MySQL-CDC] [Elasticsearch] [Iceberg] ... │ -│ (连接器生态) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ 转换层 │ -│ (将 SeaTunnel API 适配到引擎特定 API) │ -│ │ -│ • FlinkSource/FlinkSink • SparkSource/SparkSink │ -│ • 上下文适配器 • 序列化适配器 │ -└─────────────────────────────────────────────────────────────────┘ - │ - ┌─────────────────────┼─────────────────────┐ - ▼ ▼ ▼ -┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ SeaTunnel │ │ Apache │ │ Apache │ -│ Engine (Zeta)│ │ Flink │ │ Spark │ -│ │ │ │ │ │ -│ • 主节点 │ │ • JobManager │ │ • Driver │ -│ • 工作节点 │ │ • TaskManager│ │ • Executor │ -│ • 检查点 │ │ • State │ │ • RDD/DS │ -└──────────────┘ └──────────────┘ └──────────────┘ +```mermaid +flowchart TD + config["用户配置层
HOCON 配置 / SQL / Web UI"] + api["SeaTunnel API 层
数据源 API / 数据 Sink API / 转换 API / 表 API"] + connectors["连接器生态系统
JDBC / Kafka / MySQL-CDC / Elasticsearch / Iceberg / ..."] + translation["转换层
Flink 适配器 / Spark 适配器 / 上下文包装器 / 序列化适配器"] + + config --> api --> connectors --> translation + translation --> zeta["SeaTunnel Engine (Zeta)
主节点 / 工作节点 / 检查点"] + translation --> flink["Apache Flink
JobManager / TaskManager / State"] + translation --> spark["Apache Spark
Driver / Executor / RDD / Dataset"] + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class config,api layerBlue; + class connectors,translation layerCyan; + class zeta,flink,spark layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 2.1 层级职责 @@ -112,10 +89,10 @@ API 层提供引擎独立的抽象： **关键设计**：协调（枚举器）与执行（读取器）分离，实现高效的并行处理和容错。 #### 数据 Sink API -- **SeaTunnelSink**：创建写入器和提交器的工厂接口 +- **SeaTunnelSink**：创建写入器和可选提交策略的工厂接口 - **SinkWriter**：工作节点侧组件，负责写入数据 -- **SinkCommitter**：多个写入器的提交操作协调器 -- **SinkAggregatedCommitter**：聚合提交的全局协调器 +- **SinkCommitter**：工作节点侧的可选提交器，负责独立提交单个 writer 的变更 +- **SinkAggregatedCommitter**：协调端聚合提交路径上的可选全局提交器 **关键设计**：两阶段提交协议（prepareCommit → commit）在外部系统支持事务/幂等提交且启用 checkpoint 的前提下，可提供一致性语义。 @@ -145,8 +122,16 @@ API 层提供引擎独立的抽象： - **FlowLifeCycle**：管理数据源 Source/转换/数据 Sink 组件的生命周期 #### 执行模型 -``` -LogicalDag → PhysicalPlan → SubPlan (管道) → PhysicalVertex → TaskGroup → SeaTunnelTask +```mermaid +flowchart LR + logical["LogicalDag"] --> plan["PhysicalPlan"] --> pipeline["SubPlan
（管道）"] --> vertex["PhysicalVertex"] --> taskGroup["TaskGroup"] --> task["SeaTunnelTask"] + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class logical,plan,pipeline layerBlue; + class vertex,taskGroup,task layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 3.3 转换层 @@ -162,20 +147,12 @@ LogicalDag → PhysicalPlan → SubPlan (管道) → PhysicalVertex → TaskGrou 所有连接器遵循标准化结构： -``` -connector-[name]/ -├── src/main/java/.../ -│ ├── [Name]Source.java # 实现 SeaTunnelSource -│ ├── [Name]SourceReader.java # 实现 SourceReader -│ ├── [Name]SourceSplitEnumerator.java -│ ├── [Name]SourceSplit.java -│ ├── [Name]Sink.java # 实现 SeaTunnelSink -│ ├── [Name]SinkWriter.java # 实现 SinkWriter -│ └── config/[Name]Config.java -└── src/main/resources/META-INF/services/ - ├── org.apache.seatunnel.api.table.factory.TableSourceFactory - └── org.apache.seatunnel.api.table.factory.TableSinkFactory -``` +| 区域 | 典型文件 | 职责 | +|------|----------|------| +| Source 入口 | `[Name]Source.java`、`[Name]SourceReader.java`、`[Name]SourceSplitEnumerator.java`、`[Name]SourceSplit.java` | 读取数据、切分任务并暴露统一的 Source 契约 | +| Sink 入口 | `[Name]Sink.java`、`[Name]SinkWriter.java` | 缓冲、写入并向目标系统提交数据 | +| 配置定义 | `config/[Name]Config.java` | 定义连接器参数、校验规则和默认值 | +| SPI 注册 | `META-INF/services/TableSourceFactory`、`META-INF/services/TableSinkFactory` | 注册工厂，供运行时发现和装载 | **发现机制**：Java SPI（服务提供者接口）用于动态连接器加载。 @@ -183,47 +160,28 @@ connector-[name]/ ### 4.1 数据读取 Source 端数据流 -``` -数据源 Source - │ - ▼ -┌─────────────────────┐ -│ SourceSplitEnumerator│ (主节点侧) -│ • 生成分片 │ -│ • 分配给读取器 │ -└─────────────────────┘ - │ (分片分配) - ▼ -┌─────────────────────┐ -│ SourceReader │ (工作节点侧) -│ • 从分片读取 │ -│ • 发送记录 │ -└─────────────────────┘ - │ - ▼ - SeaTunnelRow - │ - ▼ - 转换链（可选） - │ - ▼ - SeaTunnelRow - │ - ▼ -┌─────────────────────┐ -│ SinkWriter │ (工作节点侧) -│ • 缓冲记录 │ -│ • 准备提交 │ -└─────────────────────┘ - │ (CommitInfo) - ▼ -┌─────────────────────┐ -│ SinkCommitter │ (协调器) -│ • 提交变更 │ -└─────────────────────┘ - │ - ▼ -数据 Sink +```mermaid +flowchart TD + source["数据源 Source"] --> enumerator["SourceSplitEnumerator
主节点侧
生成分片 / 分配读取器"] + enumerator -->|分片分配| reader["SourceReader
工作节点侧
从分片读取 / 发送记录"] + reader --> rowIn["SeaTunnelRow"] + rowIn --> transform["转换链
（可选）"] + transform --> rowOut["SeaTunnelRow"] + rowOut --> writer["SinkWriter
工作节点侧
缓冲记录 / 准备提交"] + writer -->|"可选的工作节点本地提交"| committer["SinkCommitter
工作节点侧
独立提交各 writer 的变更"] + writer -. "可选的聚合提交路径" .-> aggregatedTask["SinkAggregatedCommitterTask
协调器侧
收集各 writer 的 CommitInfo"] + aggregatedTask --> aggregated["SinkAggregatedCommitter
协调器侧
执行一次全局提交"] + committer --> sink["数据 Sink"] + aggregated --> sink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source,sink,rowIn,rowOut layerBlue; + class enumerator,reader,transform layerCyan; + class writer,committer,aggregatedTask,aggregated layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ### 4.2 基于分片的并行度 @@ -237,10 +195,28 @@ connector-[name]/ 作业被划分为**管道**（SubPlan）： -``` -管道 1: [数据 Source A] → [转换 1] → [数据 Sink A] - ↓ -管道 2: [数据 Source B] ───────→ [转换 2] → [数据 Sink B] +下图表示同一个作业中的两个独立子计划，它们之间不存在直接的数据记录流转。 + +```mermaid +flowchart TB + subgraph pipeline1["管道 1"] + direction LR + sourceA["数据 Source A"] --> transformA["转换 1"] --> sinkA["数据 Sink A"] + end + + subgraph pipeline2["管道 2"] + direction LR + sourceB["数据 Source B"] --> transformB["转换 2"] --> sinkB["数据 Sink B"] + end + + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class sourceA,sourceB,transformA,transformB layerCyan; + class sinkA,sinkB layerPurple; + style pipeline1 fill:#081425,stroke:#5db8e2,stroke-width:1.5px,color:#f8fbff; + style pipeline2 fill:#081425,stroke:#5db8e2,stroke-width:1.5px,color:#f8fbff; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` 每个管道： @@ -288,25 +264,48 @@ sequenceDiagram 4. **提交阶段** - SinkWriter 准备提交信息 - - SinkCommitter 协调提交 + - 默认由工作节点侧 `SinkCommitter` 独立提交各 writer 的变更；如果启用聚合提交，则改由协调端执行一次全局提交 - 状态持久化到检查点存储 ### 5.3 状态机 **任务状态转换**： -``` -CREATED → INIT → WAITING_RESTORE → READY_START → STARTING → RUNNING - ↓ - FAILED ← ─────────────────────── → PREPARE_CLOSE → CLOSED - ↓ - CANCELED +```mermaid +stateDiagram-v2 + direction LR + [*] --> CREATED + CREATED --> INIT + INIT --> WAITING_RESTORE: 恢复路径 + INIT --> READY_START: 无需恢复 + WAITING_RESTORE --> READY_START + READY_START --> STARTING + STARTING --> RUNNING + RUNNING --> PREPARE_CLOSE: 正常完成 + PREPARE_CLOSE --> CLOSED + INIT --> CANCELLING: 外部取消 + WAITING_RESTORE --> CANCELLING + READY_START --> CANCELLING + STARTING --> CANCELLING + RUNNING --> CANCELLING + PREPARE_CLOSE --> CANCELLING + CANCELLING --> CANCELED ``` +**失败说明**： +- `FAILED` 是运行时对不可恢复错误的结果标记，但“失败后是否重启”由更高层的恢复逻辑决定，不应在这个任务状态机图里画成 `FAILED → ...` 的直接边。 + **作业状态转换**： -``` -CREATED → SCHEDULED → RUNNING → FINISHED - ↓ ↓ - FAILED CANCELING → CANCELED +```mermaid +stateDiagram-v2 + direction LR + [*] --> CREATED + CREATED --> SCHEDULED + SCHEDULED --> RUNNING + RUNNING --> FINISHED + SCHEDULED --> FAILED + RUNNING --> FAILED + RUNNING --> CANCELING + CANCELING --> CANCELED ``` ## 6. 关键特性 @@ -328,10 +327,10 @@ CREATED → SCHEDULED → RUNNING → FINISHED **两阶段提交协议**： 1. **准备阶段**：SinkWriter 在检查点期间准备提交信息 -2. **提交阶段**：SinkCommitter 在检查点完成后提交 +2. **提交阶段**：默认由工作节点侧 `SinkCommitter` 独立提交各 writer 的变更；如果启用聚合提交，则在 checkpoint 成功后由协调端执行一次全局提交 3. **中止处理**：在提交前失败时回滚 -**幂等性**：SinkCommitter 操作必须是幂等的以处理重试 +**幂等性**：`SinkCommitter` 与 `SinkAggregatedCommitter` 的提交操作都必须保持幂等，以便在重试场景下不重复生效 ### 6.3 动态资源管理 @@ -356,49 +355,23 @@ CREATED → SCHEDULED → RUNNING → FINISHED ## 7. 模块结构 -``` -seatunnel/ -├── seatunnel-api/ # 核心 API 定义 -│ ├── source/ # 数据源 API -│ ├── sink/ # 数据 Sink API -│ ├── transform/ # 转换 API -│ └── table/ # 表和模式 API -│ -├── seatunnel-connectors-v2/ # 连接器实现 -│ ├── connector-jdbc/ # JDBC 连接器 -│ ├── connector-kafka/ # Kafka 连接器 -│ ├── connector-cdc/ # CDC 连接器集合 -│ │ ├── connector-cdc-mysql/ # MySQL CDC 连接器 -│ └── ... # 更多连接器 -│ -├── seatunnel-transforms-v2/ # 转换实现 -│ ├── src/ # Transform 实现源码（如：SQL、Filter 等） -│ └── ... -│ -├── seatunnel-engine/ # SeaTunnel Engine (Zeta) -│ ├── seatunnel-engine-core/ # 核心执行逻辑 -│ ├── seatunnel-engine-server/ # 服务器组件（主节点/工作节点） -│ └── seatunnel-engine-storage/ # 检查点存储 -│ -├── seatunnel-translation/ # 引擎转换层 -│ ├── seatunnel-translation-flink/ -│ └── seatunnel-translation-spark/ -│ -├── seatunnel-formats/ # 数据格式处理器 -│ ├── seatunnel-format-json/ -│ ├── seatunnel-format-avro/ -│ └── ... -│ -├── seatunnel-core/ # 作业提交和 CLI -└── seatunnel-e2e/ # 端到端测试 -``` +| 模块 | 代表子目录 | 职责 | +|------|------------|------| +| `seatunnel-api` | `source`、`sink`、`transform`、`table` | 定义核心 API、表模型与跨引擎抽象 | +| `seatunnel-connectors-v2` | `connector-jdbc`、`connector-kafka`、`connector-cdc-mysql` | 提供各类数据源与目标端连接器实现 | +| `seatunnel-transforms-v2` | `src`（SQL、Filter 等） | 提供通用转换能力 | +| `seatunnel-engine` | `seatunnel-engine-core`、`seatunnel-engine-server`、`seatunnel-engine-storage` | 承载 Zeta 执行、调度和检查点存储 | +| `seatunnel-translation` | `seatunnel-translation-flink`、`seatunnel-translation-spark` | 负责多引擎适配层 | +| `seatunnel-formats` | `seatunnel-format-json`、`seatunnel-format-avro` | 处理不同数据格式 | +| `seatunnel-core` | CLI 与提交入口 | 负责作业提交和命令行能力 | +| `seatunnel-e2e` | 端到端测试套件 | 保障关键链路回归 | ## 8. 设计原则 ### 8.1 关注点分离 - **API vs 实现**：清晰的 API 边界支持多种实现 -- **协调 vs 执行**：枚举器/提交器（主节点）与读取器/写入器（工作节点）分离 +- **协调 vs 执行**：枚举器与聚合提交编排负责协调，读取器与写入器负责工作节点上的实际执行 - **逻辑 vs 物理**：LogicalDag（用户意图）与 PhysicalPlan（执行细节）分离 ### 8.2 插件架构 diff --git a/docs/zh/engines/overview.md b/docs/zh/engines/overview.md index e7240a2565d4..63e59cd98b39 100644 --- a/docs/zh/engines/overview.md +++ b/docs/zh/engines/overview.md @@ -127,21 +127,30 @@ SeaTunnel 支持多种执行引擎，您可以根据实际场景选择最合适 ## 决策流程图 -``` -开始 - │ - ▼ -是否有现有的 Flink/Spark 基础设施？ - │ - ├─ 是 ──► 是否想要复用？ - │ │ - │ ├─ 是 (Flink) ──► 使用 Flink 引擎 - │ │ - │ ├─ 是 (Spark) ──► 使用 Spark 引擎 - │ │ - │ └─ 否 ──► 使用 SeaTunnel Engine - │ - └─ 否 ──► 使用 SeaTunnel Engine（推荐） +```mermaid +flowchart TD + start["开始"] + infra{"是否已经有
Flink 或 Spark 基础设施？"} + reuse{"是否希望继续复用？"} + flink["使用 Flink 引擎"] + spark["使用 Spark 引擎"] + zeta["使用 SeaTunnel Engine
（默认更推荐）"] + + start --> infra + infra -- "是" --> reuse + infra -- "否" --> zeta + reuse -- "是，Flink" --> flink + reuse -- "是，Spark" --> spark + reuse -- "否" --> zeta + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class start,infra,reuse layerBlue; + class flink,spark layerCyan; + class zeta layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## 配置示例 diff --git a/docs/zh/introduction/how-it-works.md b/docs/zh/introduction/how-it-works.md index 0794fe60d4c6..5ea92db16f6d 100644 --- a/docs/zh/introduction/how-it-works.md +++ b/docs/zh/introduction/how-it-works.md @@ -10,30 +10,29 @@ SeaTunnel 是一个分布式多模态数据集成工具，采用插件化架构 这一页适合作为“快速开始”和“架构章节”之间的桥接页。当你已经知道 SeaTunnel 是什么，但还没形成“作业配置、插件体系、执行引擎如何连起来”的整体模型时，建议先读这里。 -``` -┌─────────────────────────────────────────────────────────────┐ -│ 作业配置 │ -│ (HOCON / SQL / Web UI) │ -└─────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ SeaTunnel 核心层 │ -│ (作业解析器、协调器、调度器) │ -└─────────────────────────────────────────────────────────────┘ - │ - ┌─────────────────────┼─────────────────────┐ - ▼ ▼ ▼ -┌───────────────┐ ┌───────────────┐ ┌───────────────┐ -│ Source │────▶│ Transform │────▶│ Sink │ -│ 数据源连接器 │ │ (可选) │ │ 目标连接器 │ -└───────────────┘ └───────────────┘ └───────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ 执行引擎 │ -│ SeaTunnel Engine (Zeta) / Flink / Spark │ -└─────────────────────────────────────────────────────────────┘ +```mermaid +flowchart TD + config["作业配置
HOCON / SQL / Web UI"] + core["SeaTunnel 核心层
作业解析器 / 协调器 / 调度器"] + source["Source 数据源连接器"] + transform["Transform（可选）"] + sink["Sink 目标连接器"] + engine["执行引擎
SeaTunnel Engine (Zeta) / Flink / Spark"] + + config --> core + core --> source + source --> transform + transform --> sink + sink --> engine + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class config,core layerBlue; + class source,transform,sink layerCyan; + class engine layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` ## 推荐阅读路径 @@ -73,14 +72,19 @@ SeaTunnel 是一个分布式多模态数据集成工具，采用插件化架构 ## 数据流 -``` -Source ──▶ [分片] ──▶ Reader ──▶ Transform ──▶ Writer ──▶ Sink - │ │ │ - │ ▼ │ - │ Checkpoint/状态 │ - │ │ │ - └──────────────────────┴────────────────────────┘ - 容错机制 +```mermaid +flowchart LR + source["Source"] --> split["分片"] --> reader["Reader"] --> transform["Transform"] --> writer["Writer"] --> sink["Sink"] + reader -. "Checkpoint / 状态" .-> recovery["容错机制"] + writer -. "Checkpoint / 提交" .-> recovery + source -. "重放 / 重读" .-> recovery + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + + class source,split,reader,transform,writer,sink layerBlue; + class recovery layerCyan; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` **核心特性：** @@ -90,17 +94,16 @@ Source ──▶ [分片] ──▶ Reader ──▶ Transform ──▶ Writer ## 模块结构 -``` -seatunnel/ -├── seatunnel-api/ # 核心 API 定义 -├── seatunnel-connectors-v2/ # Source & Sink 连接器 -├── seatunnel-transforms-v2/ # Transform 插件 -├── seatunnel-engine/ # SeaTunnel Engine (Zeta) -├── seatunnel-translation/ # 引擎适配器 (Flink/Spark) -├── seatunnel-core/ # 作业提交 & CLI -├── seatunnel-formats/ # 数据格式处理 -└── seatunnel-e2e/ # 端到端测试 -``` +| 模块 | 职责 | +|------|------| +| `seatunnel-api` | 核心 API 定义 | +| `seatunnel-connectors-v2` | Source 和 Sink 连接器 | +| `seatunnel-transforms-v2` | Transform 插件 | +| `seatunnel-engine` | SeaTunnel Engine (Zeta) | +| `seatunnel-translation` | Flink 和 Spark 的引擎适配器 | +| `seatunnel-core` | 作业提交与 CLI | +| `seatunnel-formats` | 数据格式处理 | +| `seatunnel-e2e` | 端到端测试 | ## 作业执行流程 diff --git a/docs/zh/transforms/multi-table-transform-and-join-boundary.md b/docs/zh/transforms/multi-table-transform-and-join-boundary.md index 37fcb42c6347..cac6e3328129 100644 --- a/docs/zh/transforms/multi-table-transform-and-join-boundary.md +++ b/docs/zh/transforms/multi-table-transform-and-join-boundary.md @@ -18,12 +18,28 @@ SeaTunnel 的**多表 Transform**功能允许单个 Transform 节点在一条流 单个 Source（例如 MySQL-CDC）同时发送来自**多张表**的记录，下游的每个 Transform 或 Sink 需声明它适用于哪张（些）表。 -``` -MySQL-CDC ──► FieldMapper (orders 表) ──► Kafka Sink (orders topic) - │ - ├──► FieldMapper (users 表) ──► Kafka Sink (users topic) - │ - └──► (未匹配的表直接透传) ──► Elasticsearch Sink +```mermaid +flowchart LR + source["MySQL-CDC"] + ordersMap["FieldMapper
orders 表"] + ordersSink["Kafka Sink
orders topic"] + usersMap["FieldMapper
users 表"] + usersSink["Kafka Sink
users topic"] + passthrough["未匹配的表
直接透传"] + esSink["Elasticsearch Sink"] + + source --> ordersMap --> ordersSink + source --> usersMap --> usersSink + source --> passthrough --> esSink + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class source layerBlue; + class ordersMap,usersMap,passthrough layerCyan; + class ordersSink,usersSink,esSink layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` --- @@ -203,18 +219,23 @@ PostgreSQL-CDC 流进行 JOIN）。 **EtLT**（Extract、轻量 transform、Load，再在数仓中 Transform）是当 SeaTunnel Transform 层无法完成全量转换需求时的推荐架构模式： -``` -CDC Source - │ - ▼ -轻量 Transform（字段重命名、类型转换、行过滤） - │ - ▼ -数据湖 / 数仓（Hudi / Iceberg / ClickHouse） - │ - ▼ -重型 Transform（JOIN、聚合、复杂 SQL） -在 dbt / Flink SQL / Spark SQL 中执行 +```mermaid +flowchart TD + cdc["CDC Source"] + light["轻量 Transform
字段重命名 / 类型转换 / 行过滤"] + lake["数据湖 / 数仓
Hudi / Iceberg / ClickHouse"] + heavy["重型 Transform
JOIN / 聚合 / 复杂 SQL
在 dbt / Flink SQL / Spark SQL 中执行"] + + cdc --> light --> lake --> heavy + + classDef layerBlue fill:#0f1d33,stroke:#5db8e2,stroke-width:2px,color:#f8fbff; + classDef layerCyan fill:#0c2530,stroke:#2dd4bf,stroke-width:2px,color:#f8fbff; + classDef layerPurple fill:#1f1a34,stroke:#8d7cf6,stroke-width:2px,color:#f8fbff; + + class cdc layerBlue; + class light layerCyan; + class lake,heavy layerPurple; + linkStyle default stroke:#5db8e2,stroke-width:2px; ``` **适合在 SeaTunnel Transform 层完成的操作**： From 3f88dc9719622c669ed05a3719c02775628addb4 Mon Sep 17 00:00:00 2001 From: Kang Myeong Gwan Date: Sat, 20 Jun 2026 07:06:10 +0900 Subject: [PATCH 035/375] [Feature][Transform-V2] Add Base64 SQL functions (#11114) --- docs/en/transforms/sql-functions.md | 37 +++++- docs/zh/transforms/sql-functions.md | 38 +++++- .../resources/sql_transform/func_string.conf | 32 ++++- .../transform/sql/zeta/ZetaSQLFunction.java | 6 + .../transform/sql/zeta/ZetaSQLType.java | 2 + .../sql/zeta/functions/StringFunction.java | 43 +++++++ .../transform/sql/SQLStringFunctionsTest.java | 75 ++++++++++++ .../zeta/functions/StringFunctionTest.java | 111 ++++++++++++++++++ 8 files changed, 340 insertions(+), 4 deletions(-) diff --git a/docs/en/transforms/sql-functions.md b/docs/en/transforms/sql-functions.md index 2c6bd6bc17c6..28b837206763 100644 --- a/docs/en/transforms/sql-functions.md +++ b/docs/en/transforms/sql-functions.md @@ -94,6 +94,42 @@ Example: RAWTOHEX(DATA) +### TO_BASE64 + +```TO_BASE64(value[, charset]) -> STRING``` + +Encodes a string or bytes to Base64. + +The default charset is `UTF-8`. You can specify another charset. + +For bytes input, the charset argument is not supported because the value is already raw bytes. + +Returns **NULL** if value is **NULL**. + +Example: + +TO_BASE64(NAME) + +TO_BASE64(NAME, 'UTF-16') + +TO_BASE64(BINARY_PAYLOAD) + +### FROM_BASE64 + +```FROM_BASE64(value[, charset]) -> STRING``` + +Decodes a Base64 string to text. + +The default charset is `UTF-8`. You can specify another charset. + +Returns **NULL** if value is **NULL**. + +Example: + +FROM_BASE64(ENCODED_NAME) + +FROM_BASE64(TO_BASE64(NAME, 'UTF-16'), 'UTF-16') + ### INSERT ```INSERT(originalString, startInt, lengthInt, addString) -> STRING``` @@ -1350,4 +1386,3 @@ Normalizes a vector to unit length (magnitude = 1). This is useful for computing ```sql SELECT id, VECTOR_NORMALIZE(embedding) as normalized_embedding FROM table ``` - diff --git a/docs/zh/transforms/sql-functions.md b/docs/zh/transforms/sql-functions.md index 5a2536fa42e9..9349ee129159 100644 --- a/docs/zh/transforms/sql-functions.md +++ b/docs/zh/transforms/sql-functions.md @@ -94,6 +94,42 @@ HEXTORAW(DATA) RAWTOHEX(DATA) +### TO_BASE64 + +```TO_BASE64(value[, charset]) -> STRING``` + +将字符串或字节编码为 Base64。 + +默认字符集为 `UTF-8`。可以指定其他字符集。 + +对于字节输入，不支持 charset 参数，因为该值已经是原始字节。 + +如果 value 为 **NULL**，返回 **NULL**。 + +示例: + +TO_BASE64(NAME) + +TO_BASE64(NAME, 'UTF-16') + +TO_BASE64(BINARY_PAYLOAD) + +### FROM_BASE64 + +```FROM_BASE64(value[, charset]) -> STRING``` + +将 Base64 字符串解码为文本。 + +默认字符集为 `UTF-8`。可以指定其他字符集。 + +如果 value 为 **NULL**，返回 **NULL**。 + +示例: + +FROM_BASE64(ENCODED_NAME) + +FROM_BASE64(TO_BASE64(NAME, 'UTF-16'), 'UTF-16') + ### INSERT ```INSERT(originalString, startInt, lengthInt, addString) -> STRING``` @@ -1353,4 +1389,4 @@ SELECT id, VECTOR_REDUCE(embedding, 64, 'SPARSE_RANDOM_PROJECTION') as reduced_e **示例:** ```sql SELECT id, VECTOR_NORMALIZE(embedding) as normalized_embedding FROM table -``` \ No newline at end of file +``` diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_string.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_string.conf index 3f0ad98b0b2b..192b6229e435 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_string.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_string.conf @@ -54,7 +54,7 @@ transform { sql { plugin_input = "fake" plugin_output = "fake1" - query = "select ascii(c1) as c1_1, ascii(c2) as c2_1, bit_length(c4) as c4_1, length(c4) as c4_2, octet_length(c4) as c4_3, char(c5) as c5_1, concat(c1,id,'!') as c1_2, hextoraw(c6) as c6_1, rawtohex(c7) as c7_1, insert(name,2,2,'**') as name1, lower(name) as name2, upper(name) as name3, left(name, 3) as name4, right(name, 4) as name5, lpad(name, 10, '*') as name6, rpad(name, 10, '*') as name7, ltrim(c8, '*') as c8_1, rtrim(c8, '*') as c8_2, trim(c8, '*') as c8_3, regexp_replace(c9, 'w+', 'W', 'i') as c9_1, regexp_like(name, '[A-Z ]*', 'i') as name8, regexp_substr(c10, '\\d{4}') as c10_1, regexp_substr(c10, '(\\d{4})-(\\d{2})-(\\d{2})', 1, 1, null, 2) as c10_2, repeat(name||' ',3) as name9, replace(name,' ','_') as name10, soundex(name) as name11, name || space(3) as name12, substring(name, 1, 3) as name13, to_char(id) as id1, to_char(c11,'yyyy-MM-dd') as c11_1, translate(name, 'ing', 'ING') as name14, des_decrypt('1234567890', des_encrypt('1234567890', name)) as name15,UUID() as uuid from dual" + query = "select ascii(c1) as c1_1, ascii(c2) as c2_1, bit_length(c4) as c4_1, length(c4) as c4_2, octet_length(c4) as c4_3, char(c5) as c5_1, concat(c1,id,'!') as c1_2, hextoraw(c6) as c6_1, rawtohex(c7) as c7_1, insert(name,2,2,'**') as name1, lower(name) as name2, upper(name) as name3, left(name, 3) as name4, right(name, 4) as name5, lpad(name, 10, '*') as name6, rpad(name, 10, '*') as name7, ltrim(c8, '*') as c8_1, rtrim(c8, '*') as c8_2, trim(c8, '*') as c8_3, regexp_replace(c9, 'w+', 'W', 'i') as c9_1, regexp_like(name, '[A-Z ]*', 'i') as name8, regexp_substr(c10, '\\d{4}') as c10_1, regexp_substr(c10, '(\\d{4})-(\\d{2})-(\\d{2})', 1, 1, null, 2) as c10_2, repeat(name||' ',3) as name9, replace(name,' ','_') as name10, soundex(name) as name11, name || space(3) as name12, substring(name, 1, 3) as name13, to_char(id) as id1, to_char(c11,'yyyy-MM-dd') as c11_1, translate(name, 'ing', 'ING') as name14, des_decrypt('1234567890', des_encrypt('1234567890', name)) as name15, UUID() as uuid, to_base64(name) as name16, from_base64('Sm95IERpbmc=') as name17, to_base64(name, 'UTF-16') as name18, from_base64('/v8ASgBvAHkAIABEAGkAbgBn', 'UTF-16') as name19 from dual" } } @@ -295,8 +295,36 @@ sink { rule_type = NOT_NULL } ] + }, + { + field_name = "name16" + field_type = "string" + field_value = [ + {equals_to = "Sm95IERpbmc="} + ] + }, + { + field_name = "name17" + field_type = "string" + field_value = [ + {equals_to = "Joy Ding"} + ] + }, + { + field_name = "name18" + field_type = "string" + field_value = [ + {equals_to = "/v8ASgBvAHkAIABEAGkAbgBn"} + ] + }, + { + field_name = "name19" + field_type = "string" + field_value = [ + {equals_to = "Joy Ding"} + ] } ] } } -} \ No newline at end of file +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java index e33943461483..d6a21edc1e6b 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java @@ -96,6 +96,8 @@ public class ZetaSQLFunction { public static final String CONCAT_WS = "CONCAT_WS"; public static final String HEXTORAW = "HEXTORAW"; public static final String RAWTOHEX = "RAWTOHEX"; + public static final String TO_BASE64 = "TO_BASE64"; + public static final String FROM_BASE64 = "FROM_BASE64"; public static final String INSERT = "INSERT"; public static final String LOWER = "LOWER"; public static final String LCASE = "LCASE"; @@ -475,6 +477,10 @@ public Object executeFunctionExpr( return StringFunction.hextoraw(args); case RAWTOHEX: return StringFunction.rawtohex(args); + case TO_BASE64: + return StringFunction.toBase64(args); + case FROM_BASE64: + return StringFunction.fromBase64(args); case INSERT: return StringFunction.insert(args); case LOWER: diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java index 5988f0869d81..7da359c8f5a5 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java @@ -327,6 +327,8 @@ private SeaTunnelDataType getFunctionType(Function function) { case ZetaSQLFunction.CONCAT_WS: case ZetaSQLFunction.HEXTORAW: case ZetaSQLFunction.RAWTOHEX: + case ZetaSQLFunction.TO_BASE64: + case ZetaSQLFunction.FROM_BASE64: case ZetaSQLFunction.INSERT: case ZetaSQLFunction.LOWER: case ZetaSQLFunction.LCASE: diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunction.java index 51fbe627dbaf..ece5c3c7d142 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunction.java @@ -22,12 +22,14 @@ import org.apache.seatunnel.common.exception.CommonErrorCode; import org.apache.seatunnel.common.utils.DateTimeUtils; import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.EncodingUtils; import org.apache.seatunnel.transform.exception.TransformException; import org.apache.seatunnel.transform.sql.zeta.ZetaSQLFunction; import org.apache.groovy.parser.antlr4.util.StringUtils; import java.lang.reflect.Array; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; @@ -36,6 +38,7 @@ import java.time.temporal.Temporal; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -189,6 +192,46 @@ public static String rawtohex(List args) { return buff.toString(); } + public static String toBase64(List args) { + if (args.size() > 2) { + throw new IllegalArgumentException("TO_BASE64 requires one or two arguments"); + } + Object arg = args.get(0); + if (arg == null) { + return null; + } + if (arg instanceof byte[]) { + if (args.size() == 2) { + throw new IllegalArgumentException( + "TO_BASE64 does not support charset for bytes input"); + } + return Base64.getEncoder().encodeToString((byte[]) arg); + } + Charset charset = getBase64Charset(args); + return Base64.getEncoder().encodeToString(arg.toString().getBytes(charset)); + } + + public static String fromBase64(List args) { + if (args.size() > 2) { + throw new IllegalArgumentException("FROM_BASE64 requires one or two arguments"); + } + Object arg = args.get(0); + if (arg == null) { + return null; + } + Charset charset = getBase64Charset(args); + try { + return new String(Base64.getDecoder().decode(arg.toString()), charset); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid Base64 content", e); + } + } + + private static Charset getBase64Charset(List args) { + String charsetName = args.size() == 2 ? (String) args.get(1) : null; + return EncodingUtils.tryParseCharset(charsetName); + } + public static String insert(List args) { String s1 = (String) args.get(0); int start = ((Number) args.get(1)).intValue(); diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLStringFunctionsTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLStringFunctionsTest.java index e63102e19a9d..3c6c54a6d41f 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLStringFunctionsTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLStringFunctionsTest.java @@ -21,6 +21,7 @@ import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; @@ -29,6 +30,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -293,6 +295,79 @@ public void testRawtohexWithBytesColumn() { Assertions.assertEquals("010a", outRow.getField(0)); } + @Test + public void testToBase64() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"data"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + + SeaTunnelRow outRow = + runSql( + "select TO_BASE64(data) as encoded_data," + + " TO_BASE64(data, 'UTF-16') as utf16_encoded_data" + + " from dual", + rowType, + "SeaTunnel"); + + Assertions.assertEquals("U2VhVHVubmVs", outRow.getField(0)); + Assertions.assertEquals("/v8AUwBlAGEAVAB1AG4AbgBlAGw=", outRow.getField(1)); + } + + @Test + public void testFromBase64() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"data"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + + SeaTunnelRow outRow = + runSql( + "select FROM_BASE64(data) as decoded_data," + + " FROM_BASE64('/v8AUwBlAGEAVAB1AG4AbgBlAGw=', 'UTF-16') as utf16_decoded_data" + + " from dual", + rowType, + "U2VhVHVubmVs"); + + Assertions.assertEquals("SeaTunnel", outRow.getField(0)); + Assertions.assertEquals("SeaTunnel", outRow.getField(1)); + } + + @Test + public void testToBase64WithBytesColumn() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"data"}, + new SeaTunnelDataType[] {PrimitiveByteArrayType.INSTANCE}); + + byte[] bytes = "SeaTunnel".getBytes(StandardCharsets.UTF_8); + SeaTunnelRow outRow = + runSql("select TO_BASE64(data) as encoded_data from dual", rowType, bytes); + + Assertions.assertEquals("U2VhVHVubmVs", outRow.getField(0)); + } + + @Test + public void testToBase64BytesColumnRejectsCharset() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"data"}, + new SeaTunnelDataType[] {PrimitiveByteArrayType.INSTANCE}); + + byte[] bytes = "SeaTunnel".getBytes(StandardCharsets.UTF_8); + TransformException exception = + Assertions.assertThrows( + TransformException.class, + () -> + runSql( + "select TO_BASE64(data, 'UTF-16') as encoded_data from dual", + rowType, + bytes)); + + Assertions.assertInstanceOf(IllegalArgumentException.class, exception.getCause()); + Assertions.assertEquals( + "TO_BASE64 does not support charset for bytes input", + exception.getCause().getMessage()); + } + @Test public void testSoundex() { SeaTunnelRowType rowType = diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunctionTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunctionTest.java index f0e52a2d87ad..00a858ce4e6b 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunctionTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunctionTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -29,6 +30,8 @@ import java.time.ZoneOffset; import java.time.temporal.Temporal; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.List; @@ -275,6 +278,114 @@ public void testHexToRawAndRawToHex() { Assertions.assertEquals("010a", StringFunction.rawtohex(args)); } + @Test + public void testToBase64() { + List args = new ArrayList<>(); + args.add("SeaTunnel"); + Assertions.assertEquals("U2VhVHVubmVs", StringFunction.toBase64(args)); + + args.clear(); + args.add("hello"); + args.add("ISO-8859-1"); + Assertions.assertEquals("aGVsbG8=", StringFunction.toBase64(args)); + + args.clear(); + args.add("SeaTunnel"); + args.add("UTF-16"); + Assertions.assertEquals("/v8AUwBlAGEAVAB1AG4AbgBlAGw=", StringFunction.toBase64(args)); + } + + @Test + public void testFromBase64() { + List args = new ArrayList<>(); + args.add("U2VhVHVubmVs"); + Assertions.assertEquals("SeaTunnel", StringFunction.fromBase64(args)); + + args.clear(); + args.add("aGVsbG8="); + args.add("ISO-8859-1"); + Assertions.assertEquals("hello", StringFunction.fromBase64(args)); + + args.clear(); + args.add("/v8AUwBlAGEAVAB1AG4AbgBlAGw="); + args.add("UTF-16"); + Assertions.assertEquals("SeaTunnel", StringFunction.fromBase64(args)); + } + + @Test + public void testToBase64WithBytesInput() { + List args = new ArrayList<>(); + args.add("SeaTunnel".getBytes(StandardCharsets.UTF_8)); + Assertions.assertEquals("U2VhVHVubmVs", StringFunction.toBase64(args)); + } + + @Test + public void testToBase64BytesInputRejectsCharset() { + IllegalArgumentException bytesWithCharset = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + StringFunction.toBase64( + Arrays.asList( + "SeaTunnel".getBytes(StandardCharsets.UTF_8), + "UTF-16"))); + Assertions.assertEquals( + "TO_BASE64 does not support charset for bytes input", + bytesWithCharset.getMessage()); + } + + @Test + public void testBase64WithNullInputAndNullCharset() { + List args = new ArrayList<>(); + args.clear(); + args.add(null); + Assertions.assertNull(StringFunction.toBase64(args)); + Assertions.assertNull(StringFunction.fromBase64(args)); + + Assertions.assertEquals( + "U2VhVHVubmVs", StringFunction.toBase64(Arrays.asList("SeaTunnel", null))); + Assertions.assertEquals( + "SeaTunnel", StringFunction.fromBase64(Arrays.asList("U2VhVHVubmVs", null))); + } + + @Test + public void testBase64RejectsInvalidCharset() { + Assertions.assertThrows( + RuntimeException.class, + () -> StringFunction.toBase64(Arrays.asList("SeaTunnel", "invalid"))); + Assertions.assertThrows( + RuntimeException.class, + () -> StringFunction.fromBase64(Arrays.asList("U2VhVHVubmVs", "invalid"))); + } + + @Test + public void testFromBase64RejectsInvalidContent() { + IllegalArgumentException invalidBase64 = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> StringFunction.fromBase64(Collections.singletonList("not-base64!"))); + Assertions.assertEquals("Invalid Base64 content", invalidBase64.getMessage()); + } + + @Test + public void testBase64RejectsInvalidArgumentCount() { + IllegalArgumentException invalidToBase64Args = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> StringFunction.toBase64(Arrays.asList("SeaTunnel", "UTF-8", "x"))); + Assertions.assertEquals( + "TO_BASE64 requires one or two arguments", invalidToBase64Args.getMessage()); + + IllegalArgumentException invalidFromBase64Args = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + StringFunction.fromBase64( + Arrays.asList("U2VhVHVubmVs", "UTF-8", "x"))); + Assertions.assertEquals( + "FROM_BASE64 requires one or two arguments", invalidFromBase64Args.getMessage()); + } + @Test public void testInsertFunction() { List args = new ArrayList<>(); From be5d0f0279d72ba6f6b64e22f44bfc4391117816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=82=AF?= <123237285+programmerloverun@users.noreply.github.com> Date: Sat, 20 Jun 2026 13:04:10 +0800 Subject: [PATCH 036/375] =?UTF-8?q?[Feature][Connector-V2]=20Add=20Google?= =?UTF-8?q?=20Cloud=20Bigtable=20Source=20and=20Sink=20con=E2=80=A6=20(#10?= =?UTF-8?q?849)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: leijiong Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Cursor --- .dlc.json | 3 +- config/plugin_config | 1 + .../changelog/connector-google-bigtable.md | 7 + docs/en/connectors/sink/GoogleBigtable.md | 176 +++++++++++ docs/en/connectors/source/GoogleBigtable.md | 139 ++++++++ .../changelog/connector-google-bigtable.md | 7 + docs/zh/connectors/sink/GoogleBigtable.md | 105 ++++++ docs/zh/connectors/source/GoogleBigtable.md | 59 ++++ plugin-mapping.properties | 2 + .../connector-google-bigtable/pom.xml | 64 ++++ .../bigtable/client/BigtableClient.java | 173 ++++++++++ .../bigtable/config/BigtableBaseOptions.java | 61 ++++ .../bigtable/config/BigtableParameters.java | 94 ++++++ .../bigtable/config/BigtableSinkOptions.java | 90 ++++++ .../config/BigtableSourceOptions.java | 64 ++++ .../bigtable/constant/BigtableIdentifier.java | 24 ++ .../exception/BigtableConnectorErrorCode.java | 51 +++ .../exception/BigtableConnectorException.java | 32 ++ .../format/BigtableDeserializationFormat.java | 119 +++++++ .../seatunnel/bigtable/sink/BigtableSink.java | 127 ++++++++ .../bigtable/sink/BigtableSinkFactory.java | 67 ++++ .../bigtable/sink/BigtableSinkWriter.java | 298 ++++++++++++++++++ .../bigtable/source/BigtableSource.java | 81 +++++ .../source/BigtableSourceFactory.java | 78 +++++ .../bigtable/source/BigtableSourceReader.java | 254 +++++++++++++++ .../bigtable/source/BigtableSourceSplit.java | 60 ++++ .../source/BigtableSourceSplitEnumerator.java | 176 +++++++++++ .../bigtable/source/BigtableSourceState.java | 36 +++ .../state/BigtableAggregatedCommitInfo.java | 24 ++ .../bigtable/state/BigtableCommitInfo.java | 24 ++ .../bigtable/state/BigtableSinkState.java | 24 ++ .../sink/BigtableSinkSaveModeTest.java | 153 +++++++++ .../bigtable/sink/BigtableSinkWriterTest.java | 129 ++++++++ .../source/BigtableSourceReaderTest.java | 267 ++++++++++++++++ .../BigtableSourceSplitEnumeratorTest.java | 40 +++ seatunnel-connectors-v2/pom.xml | 1 + seatunnel-dist/pom.xml | 6 + 37 files changed, 3115 insertions(+), 1 deletion(-) create mode 100644 docs/en/connectors/changelog/connector-google-bigtable.md create mode 100644 docs/en/connectors/sink/GoogleBigtable.md create mode 100644 docs/en/connectors/source/GoogleBigtable.md create mode 100644 docs/zh/connectors/changelog/connector-google-bigtable.md create mode 100644 docs/zh/connectors/sink/GoogleBigtable.md create mode 100644 docs/zh/connectors/source/GoogleBigtable.md create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/pom.xml create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/client/BigtableClient.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableBaseOptions.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableParameters.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableSinkOptions.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableSourceOptions.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/constant/BigtableIdentifier.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/exception/BigtableConnectorErrorCode.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/exception/BigtableConnectorException.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/format/BigtableDeserializationFormat.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSink.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkFactory.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkWriter.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSource.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceFactory.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceReader.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplit.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplitEnumerator.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceState.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableAggregatedCommitInfo.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableCommitInfo.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableSinkState.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkSaveModeTest.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkWriterTest.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceReaderTest.java create mode 100644 seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplitEnumeratorTest.java diff --git a/.dlc.json b/.dlc.json index b25f2512f20f..a43af666416d 100644 --- a/.dlc.json +++ b/.dlc.json @@ -42,6 +42,7 @@ 0, 200, 401, - 403 + 403, + 406 ] } diff --git a/config/plugin_config b/config/plugin_config index fc6bf5c7c2c2..5f560a4a4615 100644 --- a/config/plugin_config +++ b/config/plugin_config @@ -47,6 +47,7 @@ connector-file-sftp connector-file-obs connector-google-sheets connector-google-firestore +connector-google-bigtable connector-graphql connector-hive connector-http-base diff --git a/docs/en/connectors/changelog/connector-google-bigtable.md b/docs/en/connectors/changelog/connector-google-bigtable.md new file mode 100644 index 000000000000..3fc3a897f9e7 --- /dev/null +++ b/docs/en/connectors/changelog/connector-google-bigtable.md @@ -0,0 +1,7 @@ +
Change Log + +| Change | Commit | Version | +| --- | --- | --- | +|[Feature][Connector-V2] Add Google Cloud Bigtable Source and Sink connector|https://github.com/apache/seatunnel/commit/8e57c04|dev| + +
diff --git a/docs/en/connectors/sink/GoogleBigtable.md b/docs/en/connectors/sink/GoogleBigtable.md new file mode 100644 index 000000000000..55b96cf39802 --- /dev/null +++ b/docs/en/connectors/sink/GoogleBigtable.md @@ -0,0 +1,176 @@ +import ChangeLog from '../changelog/connector-google-bigtable.md'; + +# GoogleBigtable + +> Google Bigtable sink connector + +## Description + +Writes data to Google Cloud Bigtable using the native Bigtable Data v2 Java client. + +## Key features + +- [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [x] [batch](../../introduction/concepts/connector-v2-features.md) + +## Options + +| name | type | required | default value | +|---------------------|---------|----------|---------------| +| project_id | string | yes | - | +| instance_id | string | yes | - | +| table | string | yes | - | +| rowkey_column | list | yes | - | +| column_family | config | yes | - | +| credentials_path | string | no | - | +| rowkey_delimiter | string | no | "" | +| version_column | string | no | - | +| null_mode | string | no | skip | +| batch_mutation_size | int | no | 100 | +| common-options | | no | - | + +### project_id [string] + +Google Cloud project ID. Example: `"my-gcp-project"` + +### instance_id [string] + +Bigtable instance ID. Example: `"my-bigtable-instance"` + +### table [string] + +The Bigtable table name to write to. Example: `"my-table"` + +### rowkey_column [list] + +Column names used to compose the Bigtable row key. Example: `["id"]` or `["tenant", "id"]`. + +When multiple columns are specified they are joined with `rowkey_delimiter`. + +### column_family [config] + +Mapping from column name to column family name. Use `all_columns` as key to set a default family for all unmapped columns. + +```hocon +column_family { + name = "info" + age = "stats" +} +``` + +or to put everything in one family: + +```hocon +column_family { + all_columns = "cf" +} +``` + +### credentials_path [string] + +Path to the Google Cloud service account JSON key file. + +If not set, [Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/application-default-credentials) will be used — this works automatically on GCE/GKE or when `GOOGLE_APPLICATION_CREDENTIALS` is set in the environment. + +### rowkey_delimiter [string] + +Delimiter used to join multiple row-key column values. Default is `""` (empty string, no delimiter). + +### version_column [string] + +Column name whose `BIGINT` value is used as the Bigtable cell timestamp (microseconds since epoch). If not set, the current system time is used. + +### null_mode [string] + +How to handle `null` field values. Supported: `skip` (default), `empty`. + +- `skip` — the cell is omitted from the mutation +- `empty` — an empty byte array is written to the cell + +### batch_mutation_size [int] + +Number of row mutations to accumulate before sending a BulkMutation to Bigtable. Default is `100`. Increase for higher throughput at the cost of higher per-task memory usage. + +### common options + +Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. + +## Data Types + +All SeaTunnel types are supported: + +| SeaTunnel type | Storage format in Bigtable | +|------------------------------|---------------------------------| +| TINYINT | 1-byte binary | +| SMALLINT | 2-byte big-endian binary | +| INT | 4-byte big-endian binary | +| BIGINT | 8-byte big-endian binary | +| FLOAT | 4-byte IEEE 754 big-endian | +| DOUBLE | 8-byte IEEE 754 big-endian | +| BOOLEAN | 1-byte (1 = true, 0 = false) | +| BYTES | Raw bytes | +| STRING | UTF-8 text | +| DECIMAL | UTF-8 plain string | +| DATE | UTF-8 `yyyy-MM-dd` | +| TIME | UTF-8 `HH:mm:ss` | +| TIMESTAMP | UTF-8 `yyyy-MM-dd HH:mm:ss` | + +## Example + +### Basic — Application Default Credentials + +```hocon +sink { + GoogleBigtable { + project_id = "my-gcp-project" + instance_id = "my-bigtable-instance" + table = "events" + rowkey_column = ["event_id"] + column_family { + all_columns = "cf" + } + } +} +``` + +### Service Account Key File + +```hocon +sink { + GoogleBigtable { + project_id = "my-gcp-project" + instance_id = "my-bigtable-instance" + table = "events" + credentials_path = "/secrets/sa-key.json" + rowkey_column = ["tenant_id", "event_id"] + rowkey_delimiter = "#" + column_family { + all_columns = "data" + } + batch_mutation_size = 500 + } +} +``` + +### Multiple Column Families + +```hocon +sink { + GoogleBigtable { + project_id = "my-gcp-project" + instance_id = "my-bigtable-instance" + table = "user_profile" + rowkey_column = ["user_id"] + column_family { + name = "identity" + email = "identity" + age = "stats" + last_login = "stats" + } + } +} +``` + +## Changelog + + diff --git a/docs/en/connectors/source/GoogleBigtable.md b/docs/en/connectors/source/GoogleBigtable.md new file mode 100644 index 000000000000..b511aa38c786 --- /dev/null +++ b/docs/en/connectors/source/GoogleBigtable.md @@ -0,0 +1,139 @@ +import ChangeLog from '../changelog/connector-google-bigtable.md'; + +# GoogleBigtable + +> Google Bigtable source connector + +## Description + +Reads data from Google Cloud Bigtable using the native Bigtable Data v2 Java client. + +## Key features + +- [x] [batch](../../introduction/concepts/connector-v2-features.md) +- [ ] [stream](../../introduction/concepts/connector-v2-features.md) +- [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [ ] [parallelism](../../introduction/concepts/connector-v2-features.md) + +## Options + +| name | type | required | default value | +|------------------|--------|----------|---------------| +| project_id | string | yes | - | +| instance_id | string | yes | - | +| table | string | yes | - | +| credentials_path | string | no | - | +| rowkey_column | list | no | - | +| start_rowkey | string | no | - | +| end_rowkey | string | no | - | +| start_timestamp | long | no | - | +| end_timestamp | long | no | - | +| max_versions | int | no | 1 | +| scan_row_limit | int | no | -1 | +| common-options | | no | - | + +### project_id [string] + +Google Cloud project ID. + +### instance_id [string] + +Bigtable instance ID. + +### table [string] + +Bigtable table name to read from. + +### credentials_path [string] + +Path to the Google Cloud service account JSON key file. If omitted, Application Default Credentials (ADC) are used. + +### rowkey_column [list] + +Optional list of field names that should receive the row key value. Declare a field named `rowkey` in your schema to capture the raw row key bytes as a `BYTES` or `STRING` field. + +### start_rowkey [string] + +Inclusive start row key for the scan. If not set, the scan starts from the beginning of the table. + +### end_rowkey [string] + +Exclusive end row key for the scan. If not set, the scan reads to the end of the table. + +### start_timestamp [long] + +Inclusive start timestamp filter (microseconds since epoch). + +### end_timestamp [long] + +Exclusive end timestamp filter (microseconds since epoch). + +### max_versions [int] + +Maximum number of cell versions to return per column qualifier. Default `1` returns only the latest version. + +### scan_row_limit [int] + +Maximum number of rows to return. `-1` (default) means no limit. + +### common options + +Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details. + +## Schema Mapping + +Field names in the SeaTunnel schema must follow the pattern `familyName:qualifier`, for example `cf:name` or `stats:age`. The special field name `rowkey` maps to the Bigtable row key. + +| Schema field name | Mapped Bigtable cell | +|-------------------|-----------------------------| +| `rowkey` | Row key | +| `cf:name` | Column family `cf`, qualifier `name` | +| `stats:age` | Column family `stats`, qualifier `age` | + +## Example + +### Read all rows — Application Default Credentials + +```hocon +source { + GoogleBigtable { + project_id = "my-gcp-project" + instance_id = "my-bigtable-instance" + table = "events" + schema { + fields { + rowkey = BYTES + "cf:type" = STRING + "cf:ts" = BIGINT + } + } + } +} +``` + +### Scan a row-key range with a service account + +```hocon +source { + GoogleBigtable { + project_id = "my-gcp-project" + instance_id = "my-bigtable-instance" + table = "events" + credentials_path = "/secrets/sa-key.json" + start_rowkey = "2024-01-01#" + end_rowkey = "2024-02-01#" + max_versions = 1 + schema { + fields { + rowkey = STRING + "cf:type" = STRING + "cf:data" = STRING + } + } + } +} +``` + +## Changelog + + diff --git a/docs/zh/connectors/changelog/connector-google-bigtable.md b/docs/zh/connectors/changelog/connector-google-bigtable.md new file mode 100644 index 000000000000..3fc3a897f9e7 --- /dev/null +++ b/docs/zh/connectors/changelog/connector-google-bigtable.md @@ -0,0 +1,7 @@ +
Change Log + +| Change | Commit | Version | +| --- | --- | --- | +|[Feature][Connector-V2] Add Google Cloud Bigtable Source and Sink connector|https://github.com/apache/seatunnel/commit/8e57c04|dev| + +
diff --git a/docs/zh/connectors/sink/GoogleBigtable.md b/docs/zh/connectors/sink/GoogleBigtable.md new file mode 100644 index 000000000000..65fd4f927d5a --- /dev/null +++ b/docs/zh/connectors/sink/GoogleBigtable.md @@ -0,0 +1,105 @@ +import ChangeLog from '../changelog/connector-google-bigtable.md'; + +# GoogleBigtable + +> Google Bigtable Sink 连接器 + +## 描述 + +使用原生 Bigtable Data v2 Java 客户端将数据写入 Google Cloud Bigtable。 + +## 主要特性 + +- [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [x] [batch](../../introduction/concepts/connector-v2-features.md) + +## 参数 + +| 参数名 | 类型 | 是否必填 | 默认值 | +|--------------------|---------|--------|------| +| project_id | string | 是 | - | +| instance_id | string | 是 | - | +| table | string | 是 | - | +| rowkey_column | list | 是 | - | +| column_family | config | 是 | - | +| credentials_path | string | 否 | - | +| rowkey_delimiter | string | 否 | "" | +| version_column | string | 否 | - | +| null_mode | string | 否 | skip | +| batch_mutation_size| int | 否 | 100 | +| common-options | | 否 | - | + +### project_id [string] + +Google Cloud 项目 ID，例如 `"my-gcp-project"`。 + +### instance_id [string] + +Bigtable 实例 ID，例如 `"my-bigtable-instance"`。 + +### table [string] + +写入的 Bigtable 表名，例如 `"my-table"`。 + +### rowkey_column [list] + +用于构造行键的列名列表，例如 `["id"]` 或 `["tenant_id", "event_id"]`。多列时用 `rowkey_delimiter` 拼接。 + +### column_family [config] + +列名到列族的映射配置。可使用 `all_columns` 作为默认列族： + +```hocon +column_family { + all_columns = "cf" +} +``` + +也可以为不同列指定不同列族： + +```hocon +column_family { + name = "info" + age = "stats" +} +``` + +### credentials_path [string] + +Google Cloud 服务账号 JSON 密钥文件路径。未设置时使用应用默认凭证（ADC）。 + +### rowkey_delimiter [string] + +多列行键的拼接分隔符，默认为空字符串 `""`。 + +### version_column [string] + +用作 Bigtable Cell 时间戳（微秒）的 BIGINT 列名。未设置时使用当前系统时间。 + +### null_mode [string] + +空值写入策略：`skip`（默认，跳过该 Cell）或 `empty`（写入空字节数组）。 + +### batch_mutation_size [int] + +每次批量提交的行数，默认 `100`。 + +## 示例 + +```hocon +sink { + GoogleBigtable { + project_id = "my-gcp-project" + instance_id = "my-bigtable-instance" + table = "events" + rowkey_column = ["event_id"] + column_family { + all_columns = "cf" + } + } +} +``` + +## Changelog + + diff --git a/docs/zh/connectors/source/GoogleBigtable.md b/docs/zh/connectors/source/GoogleBigtable.md new file mode 100644 index 000000000000..ea1b4e819067 --- /dev/null +++ b/docs/zh/connectors/source/GoogleBigtable.md @@ -0,0 +1,59 @@ +import ChangeLog from '../changelog/connector-google-bigtable.md'; + +# GoogleBigtable + +> Google Bigtable Source 连接器 + +## 描述 + +使用原生 Bigtable Data v2 Java 客户端从 Google Cloud Bigtable 读取数据。 + +## 主要特性 + +- [x] [batch](../../introduction/concepts/connector-v2-features.md) +- [ ] [parallelism](../../introduction/concepts/connector-v2-features.md) +- [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) + +## 参数 + +| 参数名 | 类型 | 是否必填 | 默认值 | +|-----------------|--------|--------|------| +| project_id | string | 是 | - | +| instance_id | string | 是 | - | +| table | string | 是 | - | +| credentials_path| string | 否 | - | +| rowkey_column | list | 否 | - | +| start_rowkey | string | 否 | - | +| end_rowkey | string | 否 | - | +| start_timestamp | long | 否 | - | +| end_timestamp | long | 否 | - | +| max_versions | int | 否 | 1 | +| scan_row_limit | int | 否 | -1 | +| common-options | | 否 | - | + +### Schema 映射 + +字段名须使用 `列族:列限定符` 格式，例如 `cf:name`、`stats:age`。特殊字段名 `rowkey` 映射到行键。 + +## 示例 + +```hocon +source { + GoogleBigtable { + project_id = "my-gcp-project" + instance_id = "my-bigtable-instance" + table = "events" + schema { + fields { + rowkey = BYTES + "cf:type" = STRING + "cf:ts" = BIGINT + } + } + } +} +``` + +## Changelog + + diff --git a/plugin-mapping.properties b/plugin-mapping.properties index 800f50b7bef3..5e0a4460ceb8 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -90,6 +90,8 @@ seatunnel.source.MyHours = connector-http-myhours seatunnel.sink.InfluxDB = connector-influxdb seatunnel.source.GoogleSheets = connector-google-sheets seatunnel.sink.GoogleFirestore = connector-google-firestore +seatunnel.source.GoogleBigtable = connector-google-bigtable +seatunnel.sink.GoogleBigtable = connector-google-bigtable seatunnel.sink.Tablestore = connector-tablestore seatunnel.source.Tablestore = connector-tablestore seatunnel.source.Lemlist = connector-http-lemlist diff --git a/seatunnel-connectors-v2/connector-google-bigtable/pom.xml b/seatunnel-connectors-v2/connector-google-bigtable/pom.xml new file mode 100644 index 000000000000..5f6e03aaaaab --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/pom.xml @@ -0,0 +1,64 @@ + + + + 4.0.0 + + org.apache.seatunnel + seatunnel-connectors-v2 + ${revision} + + + connector-google-bigtable + SeaTunnel : Connectors V2 : Google Bigtable + + + 2.39.1 + + + + + + org.apache.seatunnel + connector-common + ${project.version} + + + + com.google.cloud + google-cloud-bigtable + ${bigtable.version} + + + + org.apache.seatunnel + seatunnel-format-json + ${project.version} + + + + org.mockito + mockito-junit-jupiter + test + + + + + diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/client/BigtableClient.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/client/BigtableClient.java new file mode 100644 index 000000000000..d4d21d97188f --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/client/BigtableClient.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.client; + +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorErrorCode; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorException; + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.BulkMutation; +import com.google.cloud.bigtable.data.v2.models.Mutation; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.Serializable; +import java.util.List; + +/** + * Wrapper around the native Google Cloud Bigtable Java client. + * + *

Provides basic CRUD operations and batched mutation support used by both the Sink and Source + * connectors. + */ +@Slf4j +public class BigtableClient implements Serializable, AutoCloseable { + + private transient BigtableDataClient dataClient; + private final BigtableParameters parameters; + + private BigtableClient(BigtableDataClient dataClient, BigtableParameters parameters) { + this.dataClient = dataClient; + this.parameters = parameters; + } + + /** + * Creates a new BigtableClient instance using the provided parameters. + * + * @param parameters Bigtable connection parameters + * @return a connected BigtableClient + */ + public static BigtableClient createInstance(BigtableParameters parameters) { + try { + BigtableDataSettings.Builder settingsBuilder = + BigtableDataSettings.newBuilder() + .setProjectId(parameters.getProjectId()) + .setInstanceId(parameters.getInstanceId()); + + if (parameters.getCredentialsPath() != null + && !parameters.getCredentialsPath().isEmpty()) { + try (FileInputStream credStream = + new FileInputStream(parameters.getCredentialsPath())) { + GoogleCredentials credentials = + ServiceAccountCredentials.fromStream(credStream); + settingsBuilder.stubSettings().setCredentialsProvider(() -> credentials); + } + } + + BigtableDataClient client = BigtableDataClient.create(settingsBuilder.build()); + return new BigtableClient(client, parameters); + } catch (IOException e) { + throw new BigtableConnectorException( + BigtableConnectorErrorCode.CONNECTION_FAILED, + "Failed to create Bigtable client for project=" + + parameters.getProjectId() + + ", instance=" + + parameters.getInstanceId(), + e); + } + } + + /** + * Applies a single row mutation to Bigtable. + * + * @param rowMutation the row mutation to apply + */ + public void mutateRow(RowMutation rowMutation) { + try { + dataClient.mutateRow(rowMutation); + } catch (Exception e) { + throw new BigtableConnectorException( + BigtableConnectorErrorCode.WRITE_FAILED, + "Failed to mutate row in table " + parameters.getTable(), + e); + } + } + + /** + * Applies a batch of row mutations to Bigtable using BulkMutation for efficiency. + * + * @param mutations list of (rowKey, Mutation) pairs to apply + */ + public void bulkMutate(List mutations) { + if (mutations.isEmpty()) { + return; + } + try { + BulkMutation bulk = BulkMutation.create(parameters.getTable()); + for (RowKeyMutation entry : mutations) { + bulk.add(entry.getRowKey(), entry.getMutation()); + } + dataClient.bulkMutateRows(bulk); + } catch (Exception e) { + throw new BigtableConnectorException( + BigtableConnectorErrorCode.WRITE_FAILED, + "Failed to bulk mutate " + + mutations.size() + + " rows in table " + + parameters.getTable(), + e); + } + } + + /** + * Returns the underlying data client (for use in Source reader streaming). + * + * @return BigtableDataClient + */ + public BigtableDataClient getDataClient() { + return dataClient; + } + + @Override + public void close() { + if (dataClient != null) { + try { + dataClient.close(); + dataClient = null; + } catch (Exception e) { + log.error("Failed to close Bigtable data client", e); + } + } + } + + /** Holds a row key and its associated Mutation for bulk operations. */ + public static class RowKeyMutation implements Serializable { + private final ByteString rowKey; + private final Mutation mutation; + + public RowKeyMutation(ByteString rowKey, Mutation mutation) { + this.rowKey = rowKey; + this.mutation = mutation; + } + + public ByteString getRowKey() { + return rowKey; + } + + public Mutation getMutation() { + return mutation; + } + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableBaseOptions.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableBaseOptions.java new file mode 100644 index 000000000000..01850ee69861 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableBaseOptions.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.options.ConnectorCommonOptions; + +import java.util.List; + +public class BigtableBaseOptions extends ConnectorCommonOptions { + + public static final Option PROJECT_ID = + Options.key("project_id") + .stringType() + .noDefaultValue() + .withDescription("Google Cloud project ID"); + + public static final Option INSTANCE_ID = + Options.key("instance_id") + .stringType() + .noDefaultValue() + .withDescription("Bigtable instance ID"); + + public static final Option TABLE = + Options.key("table") + .stringType() + .noDefaultValue() + .withDescription("Bigtable table name"); + + public static final Option> ROWKEY_COLUMNS = + Options.key("rowkey_column") + .listType() + .noDefaultValue() + .withDescription( + "Column names used to compose the Bigtable row key. " + + "If multiple columns are specified they are joined with rowkey_delimiter."); + + public static final Option CREDENTIALS_PATH = + Options.key("credentials_path") + .stringType() + .noDefaultValue() + .withDescription( + "Path to the Google Cloud service account JSON key file. " + + "If not set, Application Default Credentials (ADC) will be used."); +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableParameters.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableParameters.java new file mode 100644 index 000000000000..69a005976a29 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableParameters.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.config; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; + +import lombok.Builder; +import lombok.Getter; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +@Builder +@Getter +public class BigtableParameters implements Serializable { + + private String projectId; + private String instanceId; + private String table; + private List rowkeyColumns; + private String credentialsPath; + + // Sink-specific + private Map columnFamily; + @Builder.Default private String rowkeyDelimiter = ""; + private String versionColumn; + + @Builder.Default + private BigtableSinkOptions.NullMode nullMode = BigtableSinkOptions.NullMode.SKIP; + + @Builder.Default private int batchMutationSize = 100; + + // Source-specific + private String startRowkey; + private String endRowkey; + private Long startTimestamp; + private Long endTimestamp; + @Builder.Default private int maxVersions = 1; + @Builder.Default private int scanRowLimit = -1; + + public static BigtableParameters buildWithConfig(ReadonlyConfig config) { + BigtableParametersBuilder builder = BigtableParameters.builder(); + builder.projectId(config.get(BigtableBaseOptions.PROJECT_ID)); + builder.instanceId(config.get(BigtableBaseOptions.INSTANCE_ID)); + builder.table(config.get(BigtableBaseOptions.TABLE)); + builder.rowkeyColumns(config.get(BigtableBaseOptions.ROWKEY_COLUMNS)); + + config.getOptional(BigtableBaseOptions.CREDENTIALS_PATH) + .ifPresent(builder::credentialsPath); + config.getOptional(BigtableSinkOptions.COLUMN_FAMILY).ifPresent(builder::columnFamily); + config.getOptional(BigtableSinkOptions.ROWKEY_DELIMITER) + .ifPresent(builder::rowkeyDelimiter); + config.getOptional(BigtableSinkOptions.VERSION_COLUMN).ifPresent(builder::versionColumn); + config.getOptional(BigtableSinkOptions.NULL_MODE).ifPresent(builder::nullMode); + config.getOptional(BigtableSinkOptions.BATCH_MUTATION_SIZE) + .ifPresent(builder::batchMutationSize); + return builder.build(); + } + + public static BigtableParameters buildWithSourceConfig(ReadonlyConfig config) { + BigtableParametersBuilder builder = BigtableParameters.builder(); + builder.projectId(config.get(BigtableBaseOptions.PROJECT_ID)); + builder.instanceId(config.get(BigtableBaseOptions.INSTANCE_ID)); + builder.table(config.get(BigtableBaseOptions.TABLE)); + + config.getOptional(BigtableBaseOptions.ROWKEY_COLUMNS).ifPresent(builder::rowkeyColumns); + config.getOptional(BigtableBaseOptions.CREDENTIALS_PATH) + .ifPresent(builder::credentialsPath); + config.getOptional(BigtableSourceOptions.START_ROW_KEY).ifPresent(builder::startRowkey); + config.getOptional(BigtableSourceOptions.END_ROW_KEY).ifPresent(builder::endRowkey); + config.getOptional(BigtableSourceOptions.START_TIMESTAMP) + .ifPresent(builder::startTimestamp); + config.getOptional(BigtableSourceOptions.END_TIMESTAMP).ifPresent(builder::endTimestamp); + config.getOptional(BigtableSourceOptions.MAX_VERSIONS).ifPresent(builder::maxVersions); + config.getOptional(BigtableSourceOptions.SCAN_ROW_LIMIT).ifPresent(builder::scanRowLimit); + return builder.build(); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableSinkOptions.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableSinkOptions.java new file mode 100644 index 000000000000..1c3b4bb0b6f8 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableSinkOptions.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.sink.DataSaveMode; +import org.apache.seatunnel.api.sink.SchemaSaveMode; + +import java.util.Map; + +import static org.apache.seatunnel.api.sink.DataSaveMode.APPEND_DATA; + +public class BigtableSinkOptions extends BigtableBaseOptions { + + public static final Option> COLUMN_FAMILY = + Options.key("column_family") + .mapType() + .noDefaultValue() + .withDescription( + "Mapping from column name to column family. " + + "Use \"all_columns\" as key to set a default family for unmapped columns."); + + public static final Option ROWKEY_DELIMITER = + Options.key("rowkey_delimiter") + .stringType() + .defaultValue("") + .withDescription( + "Delimiter used to join multiple rowkey column values. Default is empty string."); + + public static final Option VERSION_COLUMN = + Options.key("version_column") + .stringType() + .noDefaultValue() + .withDescription( + "Column name whose long value is used as the Bigtable cell timestamp. " + + "If not set, the current system time is used."); + + public static final Option NULL_MODE = + Options.key("null_mode") + .enumType(NullMode.class) + .defaultValue(NullMode.SKIP) + .withDescription( + "How to handle null field values: SKIP (default) omits the cell; EMPTY writes an empty byte array."); + + public static final Option BATCH_MUTATION_SIZE = + Options.key("batch_mutation_size") + .intType() + .defaultValue(100) + .withDescription( + "Number of mutations to accumulate before flushing to Bigtable. Default is 100."); + + public static final Option SCHEMA_SAVE_MODE = + Options.key("schema_save_mode") + .singleChoice( + SchemaSaveMode.class, + java.util.Arrays.asList(SchemaSaveMode.RECREATE_SCHEMA)) + .defaultValue(SchemaSaveMode.RECREATE_SCHEMA) + .withDescription( + "Schema save mode. Only RECREATE_SCHEMA is currently supported. " + + "Table and column families must be created manually before running the job."); + + public static final Option DATA_SAVE_MODE = + Options.key("data_save_mode") + .singleChoice(DataSaveMode.class, java.util.Arrays.asList(APPEND_DATA)) + .defaultValue(APPEND_DATA) + .withDescription( + "Data save mode. Only APPEND_DATA is currently supported. " + + "DROP_DATA and ERROR_WHEN_DATA_EXISTS are not yet implemented."); + + public enum NullMode { + SKIP, + EMPTY; + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableSourceOptions.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableSourceOptions.java new file mode 100644 index 000000000000..9ef2f88a37b0 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/config/BigtableSourceOptions.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; + +public class BigtableSourceOptions extends BigtableBaseOptions { + + public static final Option START_ROW_KEY = + Options.key("start_rowkey") + .stringType() + .noDefaultValue() + .withDescription("Bigtable scan start row key (inclusive)."); + + public static final Option END_ROW_KEY = + Options.key("end_rowkey") + .stringType() + .noDefaultValue() + .withDescription("Bigtable scan end row key (exclusive)."); + + public static final Option START_TIMESTAMP = + Options.key("start_timestamp") + .longType() + .noDefaultValue() + .withDescription( + "Start timestamp (inclusive) for scan time range in microseconds since epoch."); + + public static final Option END_TIMESTAMP = + Options.key("end_timestamp") + .longType() + .noDefaultValue() + .withDescription( + "End timestamp (exclusive) for scan time range in microseconds since epoch."); + + public static final Option MAX_VERSIONS = + Options.key("max_versions") + .intType() + .defaultValue(1) + .withDescription( + "Maximum number of cell versions to return per column. Default is 1 (latest only)."); + + public static final Option SCAN_ROW_LIMIT = + Options.key("scan_row_limit") + .intType() + .defaultValue(-1) + .withDescription( + "Maximum number of rows to scan. -1 (default) means no limit."); +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/constant/BigtableIdentifier.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/constant/BigtableIdentifier.java new file mode 100644 index 000000000000..6bd287b27ab9 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/constant/BigtableIdentifier.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.constant; + +public class BigtableIdentifier { + public static final String IDENTIFIER_NAME = "GoogleBigtable"; + + private BigtableIdentifier() {} +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/exception/BigtableConnectorErrorCode.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/exception/BigtableConnectorErrorCode.java new file mode 100644 index 000000000000..0f16864b7243 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/exception/BigtableConnectorErrorCode.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.exception; + +import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; + +public enum BigtableConnectorErrorCode implements SeaTunnelErrorCode { + CONNECTION_FAILED("Bigtable-01", "Build Bigtable connection failed"), + TABLE_NOT_FOUND("Bigtable-02", "Bigtable table not found"), + TABLE_CREATE_FAILED("Bigtable-03", "Bigtable table create failed"), + TABLE_DELETE_FAILED("Bigtable-04", "Bigtable table delete failed"), + TABLE_TRUNCATE_FAILED("Bigtable-05", "Bigtable table truncate failed"), + TABLE_QUERY_FAILED("Bigtable-06", "Bigtable table query failed"), + WRITE_FAILED("Bigtable-07", "Bigtable write failed"), + READ_FAILED("Bigtable-08", "Bigtable read failed"), + CREDENTIALS_FAILED("Bigtable-09", "Failed to load Bigtable credentials"), + ; + + private final String code; + private final String description; + + BigtableConnectorErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return code; + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/exception/BigtableConnectorException.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/exception/BigtableConnectorException.java new file mode 100644 index 000000000000..d9b73e77f3a7 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/exception/BigtableConnectorException.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.exception; + +import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; + +public class BigtableConnectorException extends SeaTunnelRuntimeException { + + public BigtableConnectorException(BigtableConnectorErrorCode errorCode, String errorMessage) { + super(errorCode, errorMessage); + } + + public BigtableConnectorException( + BigtableConnectorErrorCode errorCode, String errorMessage, Throwable cause) { + super(errorCode, errorMessage, cause); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/format/BigtableDeserializationFormat.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/format/BigtableDeserializationFormat.java new file mode 100644 index 000000000000..5a7254ba3e76 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/format/BigtableDeserializationFormat.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.format; + +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorErrorCode; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorException; + +import com.google.protobuf.ByteString; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; + +/** + * Deserializes raw {@link ByteString} cell values returned by Bigtable into typed {@link + * SeaTunnelRow} fields. + * + *

Numeric types are stored as big-endian binary; string-like types (DATE, TIME, TIMESTAMP, + * DECIMAL, STRING) are stored as UTF-8 text, matching the encoding used by {@code + * BigtableSinkWriter}. + */ +public class BigtableDeserializationFormat { + + private final DateUtils.Formatter dateFormat = DateUtils.Formatter.YYYY_MM_DD; + private final DateTimeUtils.Formatter datetimeFormat = + DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS; + private final TimeUtils.Formatter timeFormat = TimeUtils.Formatter.HH_MM_SS; + + /** + * Deserializes an array of raw cell bytes into a {@link SeaTunnelRow}. + * + * @param rawCells one entry per field in {@code rowType}; may be {@code null} for absent cells + * @param rowType the target row schema + * @return the deserialized row + */ + public SeaTunnelRow deserialize(ByteString[] rawCells, SeaTunnelRowType rowType) { + SeaTunnelRow row = new SeaTunnelRow(rowType.getTotalFields()); + for (int i = 0; i < rowType.getTotalFields(); i++) { + SeaTunnelDataType fieldType = rowType.getFieldType(i); + row.setField(i, deserializeCell(fieldType, rawCells[i])); + } + return row; + } + + private Object deserializeCell(SeaTunnelDataType fieldType, ByteString cell) { + if (cell == null || cell.isEmpty()) { + return null; + } + byte[] bytes = cell.toByteArray(); + switch (fieldType.getSqlType()) { + case TINYINT: + return bytes[0]; + case SMALLINT: + return (short) ((bytes[0] & 0xFF) << 8 | (bytes[1] & 0xFF)); + case INT: + return ByteBuffer.wrap(bytes).getInt(); + case BIGINT: + return ByteBuffer.wrap(bytes).getLong(); + case FLOAT: + return ByteBuffer.wrap(bytes).getFloat(); + case DOUBLE: + return ByteBuffer.wrap(bytes).getDouble(); + case BOOLEAN: + return bytes[0] != 0; + case BYTES: + return bytes; + case DECIMAL: + String decStr = new String(bytes, StandardCharsets.UTF_8); + try { + return new BigDecimal(decStr); + } catch (NumberFormatException e) { + return new BigDecimal(ByteBuffer.wrap(bytes).getFloat()); + } + case DATE: + return LocalDate.parse( + new String(bytes, StandardCharsets.UTF_8), + DateTimeFormatter.ofPattern(dateFormat.getValue())); + case TIME: + return LocalTime.parse( + new String(bytes, StandardCharsets.UTF_8), + DateTimeFormatter.ofPattern(timeFormat.getValue())); + case TIMESTAMP: + return LocalDateTime.parse( + new String(bytes, StandardCharsets.UTF_8), + DateTimeFormatter.ofPattern(datetimeFormat.getValue())); + case STRING: + return new String(bytes, StandardCharsets.UTF_8); + default: + throw new BigtableConnectorException( + BigtableConnectorErrorCode.READ_FAILED, + "Unsupported data type: " + fieldType.getSqlType()); + } + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSink.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSink.java new file mode 100644 index 000000000000..468d0f8c521b --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSink.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.sink.DataSaveMode; +import org.apache.seatunnel.api.sink.SchemaSaveMode; +import org.apache.seatunnel.api.sink.SeaTunnelSink; +import org.apache.seatunnel.api.sink.SinkWriter; +import org.apache.seatunnel.api.sink.SupportMultiTableSink; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableSinkOptions; +import org.apache.seatunnel.connectors.seatunnel.bigtable.constant.BigtableIdentifier; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorErrorCode; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorException; +import org.apache.seatunnel.connectors.seatunnel.bigtable.state.BigtableAggregatedCommitInfo; +import org.apache.seatunnel.connectors.seatunnel.bigtable.state.BigtableCommitInfo; +import org.apache.seatunnel.connectors.seatunnel.bigtable.state.BigtableSinkState; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +@Slf4j +public class BigtableSink + implements SeaTunnelSink< + SeaTunnelRow, + BigtableSinkState, + BigtableCommitInfo, + BigtableAggregatedCommitInfo>, + SupportMultiTableSink { + + private final ReadonlyConfig config; + private final CatalogTable catalogTable; + private final BigtableParameters parameters; + private final SeaTunnelRowType rowType; + private final List rowkeyColumnIndexes = new ArrayList<>(); + private int versionColumnIndex = -1; + private final SchemaSaveMode schemaSaveMode; + private final DataSaveMode dataSaveMode; + + public BigtableSink(ReadonlyConfig config, CatalogTable catalogTable) { + this.config = config; + this.catalogTable = catalogTable; + this.parameters = BigtableParameters.buildWithConfig(config); + this.rowType = catalogTable.getSeaTunnelRowType(); + if (parameters.getVersionColumn() != null) { + this.versionColumnIndex = rowType.indexOf(parameters.getVersionColumn()); + } + this.schemaSaveMode = config.get(BigtableSinkOptions.SCHEMA_SAVE_MODE); + this.dataSaveMode = config.get(BigtableSinkOptions.DATA_SAVE_MODE); + } + + @Override + public String getPluginName() { + return BigtableIdentifier.IDENTIFIER_NAME; + } + + @Override + public BigtableSinkWriter createWriter(SinkWriter.Context context) throws IOException { + for (String rowkeyColumn : parameters.getRowkeyColumns()) { + rowkeyColumnIndexes.add(rowType.indexOf(rowkeyColumn)); + } + if (parameters.getVersionColumn() != null) { + this.versionColumnIndex = rowType.indexOf(parameters.getVersionColumn()); + } + handleSaveMode(); + return new BigtableSinkWriter(rowType, parameters, rowkeyColumnIndexes, versionColumnIndex); + } + + /** + * Validates and applies the configured {@link SchemaSaveMode} and {@link DataSaveMode}. + * + *

Only {@link SchemaSaveMode#RECREATE_SCHEMA} and {@link DataSaveMode#APPEND_DATA} are + * currently supported. Unsupported modes throw immediately so users are never misled by + * accepted-but-no-op settings. Full Admin API support (table creation / truncation) can be + * added in a follow-up once a BigtableCatalog is available. + */ + private void handleSaveMode() { + if (schemaSaveMode == SchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST) { + throw new BigtableConnectorException( + BigtableConnectorErrorCode.TABLE_CREATE_FAILED, + "schema_save_mode=CREATE_SCHEMA_WHEN_NOT_EXIST is not yet supported by the " + + "Bigtable connector. Please create the table and column families " + + "manually and set schema_save_mode=RECREATE_SCHEMA."); + } + if (dataSaveMode == DataSaveMode.DROP_DATA) { + throw new BigtableConnectorException( + BigtableConnectorErrorCode.TABLE_TRUNCATE_FAILED, + "data_save_mode=DROP_DATA is not yet supported by the Bigtable connector. " + + "Please truncate the table manually or use data_save_mode=APPEND_DATA."); + } + if (dataSaveMode == DataSaveMode.ERROR_WHEN_DATA_EXISTS) { + throw new BigtableConnectorException( + BigtableConnectorErrorCode.TABLE_QUERY_FAILED, + "data_save_mode=ERROR_WHEN_DATA_EXISTS is not yet supported by the Bigtable " + + "connector. Please use data_save_mode=APPEND_DATA."); + } + log.info("Bigtable sink save mode: schema={}, data={}", schemaSaveMode, dataSaveMode); + } + + @Override + public Optional getWriteCatalogTable() { + return Optional.ofNullable(catalogTable); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkFactory.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkFactory.java new file mode 100644 index 000000000000..aed77c072942 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.options.SinkConnectorCommonOptions; +import org.apache.seatunnel.api.table.connector.TableSink; +import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.TableSinkFactory; +import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableBaseOptions; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableSinkOptions; +import org.apache.seatunnel.connectors.seatunnel.bigtable.constant.BigtableIdentifier; + +import com.google.auto.service.AutoService; + +@AutoService(Factory.class) +public class BigtableSinkFactory implements TableSinkFactory { + + @Override + public String factoryIdentifier() { + return BigtableIdentifier.IDENTIFIER_NAME; + } + + @Override + public OptionRule optionRule() { + return OptionRule.builder() + .required( + BigtableBaseOptions.PROJECT_ID, + BigtableBaseOptions.INSTANCE_ID, + BigtableBaseOptions.TABLE, + BigtableBaseOptions.ROWKEY_COLUMNS, + BigtableSinkOptions.COLUMN_FAMILY) + .optional( + BigtableBaseOptions.CREDENTIALS_PATH, + BigtableSinkOptions.ROWKEY_DELIMITER, + BigtableSinkOptions.VERSION_COLUMN, + BigtableSinkOptions.NULL_MODE, + BigtableSinkOptions.BATCH_MUTATION_SIZE, + BigtableSinkOptions.SCHEMA_SAVE_MODE, + BigtableSinkOptions.DATA_SAVE_MODE, + SinkConnectorCommonOptions.MULTI_TABLE_SINK_REPLICA) + .build(); + } + + @Override + public TableSink createSink(TableSinkFactoryContext context) { + ReadonlyConfig readonlyConfig = context.getOptions(); + return () -> new BigtableSink(readonlyConfig, context.getCatalogTable()); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkWriter.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkWriter.java new file mode 100644 index 000000000000..8ca5312e19b2 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkWriter.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.sink; + +import org.apache.seatunnel.api.sink.SinkWriter; +import org.apache.seatunnel.api.sink.SupportMultiTableSinkWriter; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; +import org.apache.seatunnel.connectors.seatunnel.bigtable.client.BigtableClient; +import org.apache.seatunnel.connectors.seatunnel.bigtable.client.BigtableClient.RowKeyMutation; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableSinkOptions; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorErrorCode; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorException; +import org.apache.seatunnel.connectors.seatunnel.bigtable.state.BigtableCommitInfo; +import org.apache.seatunnel.connectors.seatunnel.bigtable.state.BigtableSinkState; + +import com.google.cloud.bigtable.data.v2.models.Mutation; +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Writes {@link SeaTunnelRow} records to Google Cloud Bigtable. + * + *

Each row is converted to a Bigtable mutation using the configured column-family mapping and + * row-key strategy. Mutations are accumulated in a local buffer and flushed in bulk when the buffer + * reaches {@code batchMutationSize}. + */ +@Slf4j +public class BigtableSinkWriter + implements SinkWriter, + SupportMultiTableSinkWriter { + + private static final String ALL_COLUMNS_KEY = "all_columns"; + private static final String DEFAULT_FAMILY = "cf"; + + private final BigtableClient bigtableClient; + private final SeaTunnelRowType rowType; + private final BigtableParameters parameters; + private final List rowkeyColumnIndexes; + private final int versionColumnIndex; + private final String defaultFamily; + + /** Buffer of pending mutations, flushed in batches. */ + private final List buffer; + + public BigtableSinkWriter( + SeaTunnelRowType rowType, + BigtableParameters parameters, + List rowkeyColumnIndexes, + int versionColumnIndex) { + this(rowType, parameters, rowkeyColumnIndexes, versionColumnIndex, null); + } + + BigtableSinkWriter( + SeaTunnelRowType rowType, + BigtableParameters parameters, + List rowkeyColumnIndexes, + int versionColumnIndex, + BigtableClient bigtableClient) { + this.rowType = rowType; + this.parameters = parameters; + this.rowkeyColumnIndexes = rowkeyColumnIndexes; + this.versionColumnIndex = versionColumnIndex; + this.buffer = new ArrayList<>(parameters.getBatchMutationSize()); + + Map familyMap = parameters.getColumnFamily(); + if (familyMap != null && familyMap.containsKey(ALL_COLUMNS_KEY)) { + this.defaultFamily = familyMap.get(ALL_COLUMNS_KEY); + } else { + this.defaultFamily = DEFAULT_FAMILY; + } + + this.bigtableClient = + bigtableClient != null ? bigtableClient : BigtableClient.createInstance(parameters); + } + + @Override + public void write(SeaTunnelRow element) throws IOException { + RowKeyMutation entry = convertRowToMutation(element); // outside lock: no shared state + synchronized (buffer) { + buffer.add(entry); + if (buffer.size() >= parameters.getBatchMutationSize()) { + flush(); + } + } + } + + @Override + public Optional prepareCommit() throws IOException { + synchronized (buffer) { + flush(); + } + return Optional.empty(); + } + + @Override + public void abortPrepare() {} + + @Override + public void close() throws IOException { + try { + synchronized (buffer) { + flush(); + } + } finally { + if (bigtableClient != null) { + bigtableClient.close(); + } + } + } + + private void flush() { + if (buffer.isEmpty()) { + return; + } + List toFlush = new ArrayList<>(buffer); + buffer.clear(); // clear first: prevents re-sending if bulkMutate throws + bigtableClient.bulkMutate(toFlush); + } + + private RowKeyMutation convertRowToMutation(SeaTunnelRow row) { + ByteString rowKey = buildRowKey(row); + if (rowKey.isEmpty()) { + throw new BigtableConnectorException( + BigtableConnectorErrorCode.WRITE_FAILED, + "Row key cannot be empty. Check rowkey_column configuration."); + } + + long timestamp = System.currentTimeMillis() * 1000L; // Bigtable uses microseconds + if (versionColumnIndex != -1) { + Object versionField = row.getField(versionColumnIndex); + if (versionField instanceof Long) { + timestamp = (Long) versionField; + } + } + + Mutation mutation = Mutation.create(); + + List writeColumnIndexes = + IntStream.range(0, row.getArity()) + .boxed() + .filter(idx -> !rowkeyColumnIndexes.contains(idx)) + .filter(idx -> idx != versionColumnIndex) + .collect(Collectors.toList()); + + for (Integer idx : writeColumnIndexes) { + String fieldName = rowType.getFieldName(idx); + String family = resolveFamily(fieldName); + Object fieldValue = row.getField(idx); + ByteString qualifier = ByteString.copyFromUtf8(fieldName); + + if (fieldValue == null) { + if (parameters.getNullMode() == BigtableSinkOptions.NullMode.EMPTY) { + mutation.setCell(family, qualifier, timestamp, ByteString.EMPTY); + } + // SKIP: do nothing + } else { + ByteString valueBytes = convertToByteString(row, idx); + mutation.setCell(family, qualifier, timestamp, valueBytes); + } + } + + return new RowKeyMutation(rowKey, mutation); + } + + private ByteString buildRowKey(SeaTunnelRow row) { + if (rowkeyColumnIndexes.size() == 1) { + return fieldToByteString(row, rowkeyColumnIndexes.get(0)); + } + String delimiter = parameters.getRowkeyDelimiter(); + List parts = new ArrayList<>(); + for (Integer idx : rowkeyColumnIndexes) { + Object field = row.getField(idx); + parts.add(field == null ? "" : field.toString()); + } + return ByteString.copyFromUtf8(String.join(delimiter, parts)); + } + + private ByteString fieldToByteString(SeaTunnelRow row, int index) { + Object field = row.getField(index); + if (field == null) { + return ByteString.EMPTY; + } + SeaTunnelDataType fieldType = rowType.getFieldType(index); + if (fieldType.getSqlType() == org.apache.seatunnel.api.table.type.SqlType.BYTES) { + return ByteString.copyFrom((byte[]) field); + } + return ByteString.copyFromUtf8(field.toString()); + } + + private String resolveFamily(String fieldName) { + Map familyMap = parameters.getColumnFamily(); + if (familyMap == null) { + return defaultFamily; + } + return familyMap.getOrDefault(fieldName, defaultFamily); + } + + private ByteString convertToByteString(SeaTunnelRow row, int index) { + Object field = row.getField(index); + SeaTunnelDataType fieldType = rowType.getFieldType(index); + switch (fieldType.getSqlType()) { + case TINYINT: + return ByteString.copyFrom(new byte[] {(Byte) field}); + case SMALLINT: + short sv = (Short) field; + return ByteString.copyFrom(new byte[] {(byte) (sv >> 8), (byte) sv}); + case INT: + int iv = (Integer) field; + return ByteString.copyFrom(ByteBuffer.allocate(4).putInt(iv).array()); + case BIGINT: + long lv = (Long) field; + return ByteString.copyFrom(ByteBuffer.allocate(8).putLong(lv).array()); + case FLOAT: + float fv = (Float) field; + return ByteString.copyFrom(ByteBuffer.allocate(4).putFloat(fv).array()); + case DOUBLE: + double dv = (Double) field; + return ByteString.copyFrom(ByteBuffer.allocate(8).putDouble(dv).array()); + case BOOLEAN: + return ByteString.copyFrom(new byte[] {(byte) ((Boolean) field ? 1 : 0)}); + case BYTES: + return ByteString.copyFrom((byte[]) field); + case DECIMAL: + BigDecimal bd = + field instanceof BigDecimal + ? (BigDecimal) field + : new BigDecimal(field.toString()); + return ByteString.copyFrom(bd.toPlainString().getBytes(StandardCharsets.UTF_8)); + case DATE: + LocalDate date = + field instanceof LocalDate + ? (LocalDate) field + : DateUtils.parse(field.toString()); + return ByteString.copyFrom( + DateUtils.toString(date, DateUtils.Formatter.YYYY_MM_DD) + .getBytes(StandardCharsets.UTF_8)); + case TIME: + LocalTime time = + field instanceof LocalTime + ? (LocalTime) field + : TimeUtils.parse(field.toString()); + return ByteString.copyFrom( + TimeUtils.toString(time, TimeUtils.Formatter.HH_MM_SS) + .getBytes(StandardCharsets.UTF_8)); + case TIMESTAMP: + LocalDateTime ts = + field instanceof LocalDateTime + ? (LocalDateTime) field + : DateTimeUtils.parse(field.toString()); + return ByteString.copyFrom( + DateTimeUtils.toString(ts, DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS) + .getBytes(StandardCharsets.UTF_8)); + case STRING: + return ByteString.copyFromUtf8(field.toString()); + default: + throw new BigtableConnectorException( + BigtableConnectorErrorCode.WRITE_FAILED, + String.format( + "Bigtable connector does not support column type [%s]", + fieldType.getSqlType())); + } + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSource.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSource.java new file mode 100644 index 000000000000..9f9135f68ed9 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSource.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.source; + +import org.apache.seatunnel.shade.com.google.common.collect.Lists; + +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.source.SupportParallelism; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; +import org.apache.seatunnel.connectors.seatunnel.bigtable.constant.BigtableIdentifier; + +import java.util.List; + +public class BigtableSource + implements SeaTunnelSource, + SupportParallelism { + + private final CatalogTable catalogTable; + private final BigtableParameters parameters; + + BigtableSource(BigtableParameters parameters, CatalogTable catalogTable) { + this.parameters = parameters; + this.catalogTable = catalogTable; + } + + @Override + public String getPluginName() { + return BigtableIdentifier.IDENTIFIER_NAME; + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.BOUNDED; + } + + @Override + public List getProducedCatalogTables() { + return Lists.newArrayList(catalogTable); + } + + @Override + public SourceReader createReader( + SourceReader.Context readerContext) throws Exception { + return new BigtableSourceReader( + parameters, readerContext, catalogTable.getSeaTunnelRowType()); + } + + @Override + public SourceSplitEnumerator createEnumerator( + SourceSplitEnumerator.Context enumeratorContext) throws Exception { + return new BigtableSourceSplitEnumerator(enumeratorContext, parameters); + } + + @Override + public SourceSplitEnumerator restoreEnumerator( + SourceSplitEnumerator.Context enumeratorContext, + BigtableSourceState checkpointState) + throws Exception { + return new BigtableSourceSplitEnumerator(enumeratorContext, parameters, checkpointState); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceFactory.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceFactory.java new file mode 100644 index 000000000000..bb4f4d2ad4de --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceFactory.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.source; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; +import org.apache.seatunnel.api.table.connector.TableSource; +import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.TableSourceFactory; +import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableBaseOptions; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.bigtable.constant.BigtableIdentifier; + +import com.google.auto.service.AutoService; + +import java.io.Serializable; + +@AutoService(Factory.class) +public class BigtableSourceFactory implements TableSourceFactory { + + @Override + public String factoryIdentifier() { + return BigtableIdentifier.IDENTIFIER_NAME; + } + + @Override + public OptionRule optionRule() { + return OptionRule.builder() + .required( + BigtableBaseOptions.PROJECT_ID, + BigtableBaseOptions.INSTANCE_ID, + BigtableBaseOptions.TABLE) + .optional( + BigtableBaseOptions.CREDENTIALS_PATH, + BigtableBaseOptions.ROWKEY_COLUMNS, + BigtableSourceOptions.START_ROW_KEY, + BigtableSourceOptions.END_ROW_KEY, + BigtableSourceOptions.START_TIMESTAMP, + BigtableSourceOptions.END_TIMESTAMP, + BigtableSourceOptions.MAX_VERSIONS, + BigtableSourceOptions.SCAN_ROW_LIMIT) + .build(); + } + + @Override + public Class getSourceClass() { + return BigtableSource.class; + } + + @Override + public + TableSource createSource(TableSourceFactoryContext context) { + return () -> + (SeaTunnelSource) + new BigtableSource( + BigtableParameters.buildWithSourceConfig(context.getOptions()), + CatalogTableUtil.buildWithConfig(context.getOptions())); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceReader.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceReader.java new file mode 100644 index 000000000000..17e41107aac4 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceReader.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.source; + +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.bigtable.client.BigtableClient; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; +import org.apache.seatunnel.connectors.seatunnel.bigtable.format.BigtableDeserializationFormat; + +import com.google.cloud.bigtable.data.v2.models.Filters; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentLinkedDeque; + +/** + * Reads rows from a single {@link BigtableSourceSplit} and emits {@link SeaTunnelRow} records. + * + *

The column schema is derived from the {@link SeaTunnelRowType}. Each field name must follow + * the pattern {@code columnFamily:qualifier}, except the special name {@code rowkey} which maps to + * the Bigtable row key. + */ +@Slf4j +public class BigtableSourceReader implements SourceReader { + + private static final String ROW_KEY_FIELD = "rowkey"; + + private final Deque pendingSplits = new ConcurrentLinkedDeque<>(); + private final Context context; + private final SeaTunnelRowType rowType; + private final BigtableParameters parameters; + private final BigtableDeserializationFormat deserializationFormat; + private volatile boolean noMoreSplits = false; + private BigtableClient bigtableClient; + + /** The split currently being read. Persisted in checkpoint to survive failover. */ + private volatile BigtableSourceSplit currentSplit = null; + + /** + * Set of field names that map to the Bigtable row key. Populated from {@code rowkey_column} + * config; falls back to the literal field name {@value ROW_KEY_FIELD} when not configured. + */ + private final java.util.Set rowKeyFieldNames; + + public BigtableSourceReader( + BigtableParameters parameters, Context context, SeaTunnelRowType rowType) { + this(parameters, context, rowType, null); + } + + BigtableSourceReader( + BigtableParameters parameters, + Context context, + SeaTunnelRowType rowType, + BigtableClient bigtableClient) { + this.parameters = parameters; + this.context = context; + this.rowType = rowType; + this.bigtableClient = bigtableClient; + this.deserializationFormat = new BigtableDeserializationFormat(); + if (parameters.getRowkeyColumns() != null && !parameters.getRowkeyColumns().isEmpty()) { + this.rowKeyFieldNames = new java.util.HashSet<>(parameters.getRowkeyColumns()); + } else { + this.rowKeyFieldNames = java.util.Collections.singleton(ROW_KEY_FIELD); + } + } + + @Override + public void open() throws Exception { + if (bigtableClient == null) { + bigtableClient = BigtableClient.createInstance(parameters); + } + } + + @Override + public void close() throws IOException { + if (bigtableClient != null) { + bigtableClient.close(); + bigtableClient = null; + } + } + + @Override + public void pollNext(Collector output) throws Exception { + final BigtableSourceSplit split = pendingSplits.poll(); + if (Objects.nonNull(split)) { + // Assign currentSplit before reading so checkpoints taken during readSplit() + // include it and can re-enqueue it on restore. + currentSplit = split; + readSplit(split, output); + currentSplit = null; + } else if (noMoreSplits && pendingSplits.isEmpty()) { + log.info("Closed the bounded Bigtable source"); + context.signalNoMoreElement(); + } else { + log.warn("Waiting for Bigtable split, sleeping 1s"); + Thread.sleep(1000L); + } + } + + private void readSplit(BigtableSourceSplit split, Collector output) { + Query query = buildQuery(split); + // Stream rows one at a time to avoid buffering the full result in memory. + // Each collect() acquires the checkpoint lock only for the emit, not for the network read. + bigtableClient + .getDataClient() + .readRows(query) + .forEach( + bigtableRow -> { + SeaTunnelRow seaTunnelRow = convertRow(bigtableRow); + synchronized (output.getCheckpointLock()) { + output.collect(seaTunnelRow); + } + }); + } + + private Query buildQuery(BigtableSourceSplit split) { + Query query = Query.create(parameters.getTable()); + + String startKey = split.getStartRowKey(); + String endKey = split.getEndRowKey(); + if (!startKey.isEmpty() && !endKey.isEmpty()) { + query.range(startKey, endKey); + } else if (!startKey.isEmpty()) { + query.range(startKey, null); + } else if (!endKey.isEmpty()) { + query.range(null, endKey); + } + + if (parameters.getScanRowLimit() > 0) { + query.limit(parameters.getScanRowLimit()); + } + + Filters.Filter filter = buildFilter(); + if (filter != null) { + query.filter(filter); + } + + return query; + } + + private Filters.Filter buildFilter() { + List filters = new ArrayList<>(); + + if (parameters.getMaxVersions() > 0) { + filters.add(Filters.FILTERS.limit().cellsPerColumn(parameters.getMaxVersions())); + } + + Long startTs = parameters.getStartTimestamp(); + Long endTs = parameters.getEndTimestamp(); + if (startTs != null && endTs != null) { + filters.add(Filters.FILTERS.timestamp().range().of(startTs, endTs)); + } else if (startTs != null) { + filters.add(Filters.FILTERS.timestamp().range().startClosed(startTs)); + } else if (endTs != null) { + filters.add(Filters.FILTERS.timestamp().range().endOpen(endTs)); + } + + if (filters.isEmpty()) { + return null; + } + if (filters.size() == 1) { + return filters.get(0); + } + Filters.ChainFilter chain = Filters.FILTERS.chain(); + for (Filters.Filter f : filters) { + chain = chain.filter(f); + } + return chain; + } + + /** + * Converts a Bigtable {@link Row} into a {@link SeaTunnelRow}. + * + *

Field names drive the mapping: + * + *

    + *
  • Fields listed in {@code rowkey_column} config (or the literal {@value ROW_KEY_FIELD} + * when the option is absent) → the row key bytes + *
  • {@code familyName:qualifier} → the latest cell value for that column + *
+ */ + private SeaTunnelRow convertRow(Row bigtableRow) { + // Build a flat lookup: "family:qualifier" -> latest cell value + Map cellMap = new HashMap<>(); + for (RowCell cell : bigtableRow.getCells()) { + String key = cell.getFamily() + ":" + cell.getQualifier().toStringUtf8(); + cellMap.putIfAbsent(key, cell.getValue()); // first cell = latest (sorted desc by ts) + } + + String[] fieldNames = rowType.getFieldNames(); + ByteString[] rawCells = new ByteString[fieldNames.length]; + for (int i = 0; i < fieldNames.length; i++) { + String fieldName = fieldNames[i]; + if (rowKeyFieldNames.contains(fieldName)) { + rawCells[i] = bigtableRow.getKey(); + } else { + rawCells[i] = cellMap.get(fieldName); + } + } + return deserializationFormat.deserialize(rawCells, rowType); + } + + @Override + public List snapshotState(long checkpointId) { + List state = new ArrayList<>(); + // Include the split currently being read so it can be re-enqueued on restore. + if (currentSplit != null) { + state.add(currentSplit); + } + state.addAll(pendingSplits); + return state; + } + + @Override + public void addSplits(List splits) { + pendingSplits.addAll(splits); + } + + @Override + public void handleNoMoreSplits() { + noMoreSplits = true; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) {} +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplit.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplit.java new file mode 100644 index 000000000000..99e678296021 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplit.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.source; + +import org.apache.seatunnel.api.source.SourceSplit; + +import java.io.Serializable; + +public class BigtableSourceSplit implements SourceSplit, Serializable { + + private static final long serialVersionUID = 1L; + public static final String SPLIT_PREFIX = "bigtable_source_split_"; + + private final String splitId; + /** Inclusive start row key (empty means the table start). */ + private final String startRowKey; + /** Exclusive end row key (empty means the table end). */ + private final String endRowKey; + + public BigtableSourceSplit(int splitIndex, String startRowKey, String endRowKey) { + this.splitId = SPLIT_PREFIX + splitIndex; + this.startRowKey = startRowKey; + this.endRowKey = endRowKey; + } + + @Override + public String splitId() { + return splitId; + } + + public String getStartRowKey() { + return startRowKey; + } + + public String getEndRowKey() { + return endRowKey; + } + + @Override + public String toString() { + return String.format( + "{\"split_id\":\"%s\", \"start\":\"%s\", \"end\":\"%s\"}", + splitId, startRowKey, endRowKey); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplitEnumerator.java new file mode 100644 index 000000000000..3d41a2ad3928 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplitEnumerator.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.source; + +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Enumerates {@link BigtableSourceSplit} instances for parallel reading. + * + *

Currently produces a single split covering the full table (or the user-defined row-key range). + * The split is assigned to whichever reader registers first. Future work can partition by Bigtable + * tablet boundaries using the Admin API. + */ +@Slf4j +public class BigtableSourceSplitEnumerator + implements SourceSplitEnumerator { + + private final Context context; + private final BigtableParameters parameters; + private final Set assignedSplits; + private Set pendingSplits; + private boolean initialized = false; + + public BigtableSourceSplitEnumerator( + Context context, BigtableParameters parameters) { + this(context, parameters, new HashSet<>()); + } + + public BigtableSourceSplitEnumerator( + Context context, + BigtableParameters parameters, + BigtableSourceState sourceState) { + this(context, parameters, sourceState.getAssignedSplits()); + } + + private BigtableSourceSplitEnumerator( + Context context, + BigtableParameters parameters, + Set assignedSplits) { + this.context = context; + this.parameters = parameters; + this.assignedSplits = new HashSet<>(assignedSplits); + } + + @Override + public void open() { + this.pendingSplits = new HashSet<>(); + this.initialized = false; + } + + @Override + public void run() throws Exception { + // Splits are assigned lazily when readers register. + } + + @Override + public void close() throws IOException { + // Nothing to close – no persistent connection held here. + } + + @Override + public void addSplitsBack(List splits, int subtaskId) { + if (!splits.isEmpty()) { + pendingSplits.addAll(splits); + if (context.registeredReaders().contains(subtaskId)) { + assignSplit(subtaskId); + } + } + } + + @Override + public int currentUnassignedSplitSize() { + return pendingSplits.size(); + } + + @Override + public void registerReader(int subtaskId) { + initializePendingSplits(); + assignSplit(subtaskId); + } + + @Override + public BigtableSourceState snapshotState(long checkpointId) throws Exception { + return new BigtableSourceState(assignedSplits); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception {} + + @Override + public void handleSplitRequest(int subtaskId) {} + + private void initializePendingSplits() { + if (initialized) { + return; + } + Set tableSplits = buildSplits(); + Set existingIds = + pendingSplits.stream() + .map(BigtableSourceSplit::splitId) + .collect(Collectors.toSet()); + existingIds.addAll( + assignedSplits.stream() + .map(BigtableSourceSplit::splitId) + .collect(Collectors.toSet())); + tableSplits.stream() + .filter(s -> !existingIds.contains(s.splitId())) + .forEach(pendingSplits::add); + initialized = true; + } + + /** + * Builds the set of splits. + * + *

For now a single split spanning the requested row-key range is produced. This is + * sufficient for bounded batch reads. Parallel multi-split support can be added later by + * querying Bigtable tablet boundary information. + */ + private Set buildSplits() { + String startKey = parameters.getStartRowkey() != null ? parameters.getStartRowkey() : ""; + String endKey = parameters.getEndRowkey() != null ? parameters.getEndRowkey() : ""; + return Collections.singleton(new BigtableSourceSplit(0, startKey, endKey)); + } + + private void assignSplit(int taskId) { + List toAssign = new ArrayList<>(); + if (context.currentParallelism() == 1) { + toAssign.addAll(pendingSplits); + } else { + for (BigtableSourceSplit split : pendingSplits) { + int owner = + (split.splitId().hashCode() & Integer.MAX_VALUE) + % context.currentParallelism(); + if (owner == taskId) { + toAssign.add(split); + } + } + } + context.assignSplit(taskId, toAssign); + assignedSplits.addAll(toAssign); + toAssign.forEach(pendingSplits::remove); + log.info( + "SubTask {} assigned [{}]", + taskId, + toAssign.stream() + .map(BigtableSourceSplit::splitId) + .collect(Collectors.joining(","))); + context.signalNoMoreSplits(taskId); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceState.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceState.java new file mode 100644 index 000000000000..e44fd024a970 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceState.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.source; + +import java.io.Serializable; +import java.util.Set; + +public class BigtableSourceState implements Serializable { + + private static final long serialVersionUID = 1L; + + private final Set assignedSplits; + + public BigtableSourceState(Set assignedSplits) { + this.assignedSplits = assignedSplits; + } + + public Set getAssignedSplits() { + return assignedSplits; + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableAggregatedCommitInfo.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableAggregatedCommitInfo.java new file mode 100644 index 000000000000..ca9858ae71ef --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableAggregatedCommitInfo.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.state; + +import java.io.Serializable; + +public class BigtableAggregatedCommitInfo implements Serializable { + private static final long serialVersionUID = 1L; +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableCommitInfo.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableCommitInfo.java new file mode 100644 index 000000000000..98b10945bdb7 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableCommitInfo.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.state; + +import java.io.Serializable; + +public class BigtableCommitInfo implements Serializable { + private static final long serialVersionUID = 1L; +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableSinkState.java b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableSinkState.java new file mode 100644 index 000000000000..e4d636e7f994 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/main/java/org/apache/seatunnel/connectors/seatunnel/bigtable/state/BigtableSinkState.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.state; + +import java.io.Serializable; + +public class BigtableSinkState implements Serializable { + private static final long serialVersionUID = 1L; +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkSaveModeTest.java b/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkSaveModeTest.java new file mode 100644 index 000000000000..9d3bd33ab387 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkSaveModeTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.sink.DataSaveMode; +import org.apache.seatunnel.api.sink.SchemaSaveMode; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.connectors.seatunnel.bigtable.exception.BigtableConnectorException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Unit tests for {@link BigtableSink#handleSaveMode()}. + * + *

Verifies that unsupported save modes fail fast with a {@link BigtableConnectorException} + * instead of silently no-opping (Issue 3 fix). + */ +class BigtableSinkSaveModeTest { + + private CatalogTable catalogTable; + + @BeforeEach + void setUp() { + catalogTable = + CatalogTable.of( + TableIdentifier.of("catalog", "database", "table"), + TableSchema.builder() + .column( + PhysicalColumn.of( + "id", + BasicType.STRING_TYPE, + (Long) null, + false, + null, + "row key")) + .column( + PhysicalColumn.of( + "cf:name", + BasicType.STRING_TYPE, + (Long) null, + true, + null, + "")) + .build(), + Collections.emptyMap(), + Collections.emptyList(), + ""); + } + + private ReadonlyConfig buildConfig(String schemaSaveMode, String dataSaveMode) { + Map map = new HashMap<>(); + map.put("project_id", "p"); + map.put("instance_id", "i"); + map.put("table", "t"); + map.put("rowkey_column", Arrays.asList("id")); + map.put("column_family", Collections.singletonMap("all_columns", "cf")); + if (schemaSaveMode != null) { + map.put("schema_save_mode", schemaSaveMode); + } + if (dataSaveMode != null) { + map.put("data_save_mode", dataSaveMode); + } + return ReadonlyConfig.fromMap(map); + } + + private BigtableSink newSink() { + ReadonlyConfig config = buildConfig("RECREATE_SCHEMA", "APPEND_DATA"); + return new BigtableSink(config, catalogTable); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field field = BigtableSink.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } + + /** Invokes the private handleSaveMode guard without creating a writer or Bigtable client. */ + private static void invokeHandleSaveMode(BigtableSink sink) throws Exception { + Method method = BigtableSink.class.getDeclaredMethod("handleSaveMode"); + method.setAccessible(true); + try { + method.invoke(sink); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw e; + } + } + + @Test + void testSupportedModeDoesNotThrow() { + BigtableSink sink = newSink(); + assertDoesNotThrow(() -> invokeHandleSaveMode(sink)); + } + + @Test + void testDropDataThrows() throws Exception { + BigtableSink sink = newSink(); + setField(sink, "dataSaveMode", DataSaveMode.DROP_DATA); + assertThrows(BigtableConnectorException.class, () -> invokeHandleSaveMode(sink)); + } + + @Test + void testErrorWhenDataExistsThrows() throws Exception { + BigtableSink sink = newSink(); + setField(sink, "dataSaveMode", DataSaveMode.ERROR_WHEN_DATA_EXISTS); + assertThrows(BigtableConnectorException.class, () -> invokeHandleSaveMode(sink)); + } + + @Test + void testCreateSchemaWhenNotExistThrows() throws Exception { + BigtableSink sink = newSink(); + setField(sink, "schemaSaveMode", SchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST); + assertThrows(BigtableConnectorException.class, () -> invokeHandleSaveMode(sink)); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkWriterTest.java b/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkWriterTest.java new file mode 100644 index 000000000000..1d8432190db2 --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/sink/BigtableSinkWriterTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.sink; + +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.bigtable.client.BigtableClient; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableSinkOptions; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class BigtableSinkWriterTest { + + private BigtableClient mockClient; + private SeaTunnelRowType rowType; + private BigtableParameters parameters; + + @BeforeEach + void setUp() { + mockClient = Mockito.mock(BigtableClient.class); + + rowType = + new SeaTunnelRowType( + new String[] {"id", "name", "age"}, + new org.apache.seatunnel.api.table.type.SeaTunnelDataType[] { + BasicType.STRING_TYPE, BasicType.STRING_TYPE, BasicType.INT_TYPE + }); + + Map familyMap = new HashMap<>(); + familyMap.put("all_columns", "cf"); + + parameters = + BigtableParameters.builder() + .projectId("test-project") + .instanceId("test-instance") + .table("test-table") + .rowkeyColumns(Arrays.asList("id")) + .columnFamily(familyMap) + .batchMutationSize(10) + .nullMode(BigtableSinkOptions.NullMode.SKIP) + .build(); + } + + @Test + void testWriteFlushesOnBatchSize() throws IOException { + List rowkeyIndexes = Arrays.asList(0); + BigtableSinkWriter writer = + new BigtableSinkWriter(rowType, parameters, rowkeyIndexes, -1, mockClient); + + // Write batchMutationSize rows to trigger a flush + for (int i = 0; i < parameters.getBatchMutationSize(); i++) { + SeaTunnelRow row = new SeaTunnelRow(3); + row.setField(0, "key-" + i); + row.setField(1, "name-" + i); + row.setField(2, i); + writer.write(row); + } + + verify(mockClient, times(1)).bulkMutate(anyList()); + } + + @Test + void testCloseFlushesRemainingRows() throws IOException { + List rowkeyIndexes = Arrays.asList(0); + BigtableSinkWriter writer = + new BigtableSinkWriter(rowType, parameters, rowkeyIndexes, -1, mockClient); + + // Write fewer rows than batch size + SeaTunnelRow row = new SeaTunnelRow(3); + row.setField(0, "key-1"); + row.setField(1, "alice"); + row.setField(2, 30); + writer.write(row); + + // No flush yet + verify(mockClient, times(0)).bulkMutate(anyList()); + + // Close should flush + writer.close(); + verify(mockClient, times(1)).bulkMutate(anyList()); + verify(mockClient, times(1)).close(); + } + + @Test + void testNullFieldSkipped() throws IOException { + List rowkeyIndexes = Arrays.asList(0); + BigtableSinkWriter writer = + new BigtableSinkWriter(rowType, parameters, rowkeyIndexes, -1, mockClient); + + SeaTunnelRow row = new SeaTunnelRow(3); + row.setField(0, "key-null"); + row.setField(1, null); // should be skipped + row.setField(2, 25); + writer.write(row); + writer.close(); + + // Verify a mutation was still produced (null field just omitted) + verify(mockClient, times(1)).bulkMutate(anyList()); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceReaderTest.java b/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceReaderTest.java new file mode 100644 index 000000000000..d1e88520282c --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceReaderTest.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.source; + +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.bigtable.client.BigtableClient; +import org.apache.seatunnel.connectors.seatunnel.bigtable.config.BigtableParameters; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.protobuf.ByteString; + +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link BigtableSourceReader}. + * + *

Covers: + * + *

    + *
  • Checkpoint state includes the in-flight split (Issue 1 fix) + *
  • Streaming read path — rows emitted via forEach, not buffered (Issue 2 fix) + *
  • rowkey_column config drives row-key field mapping (Issue 3 fix) + *
+ */ +class BigtableSourceReaderTest { + + private BigtableClient mockClient; + private BigtableDataClient mockDataClient; + private SourceReader.Context mockContext; + private SeaTunnelRowType rowType; + private BigtableParameters parameters; + + @BeforeEach + void setUp() { + mockClient = mock(BigtableClient.class); + mockDataClient = mock(BigtableDataClient.class); + when(mockClient.getDataClient()).thenReturn(mockDataClient); + + mockContext = mock(SourceReader.Context.class); + + rowType = + new SeaTunnelRowType( + new String[] {"rowkey", "cf:name"}, + new org.apache.seatunnel.api.table.type.SeaTunnelDataType[] { + BasicType.STRING_TYPE, BasicType.STRING_TYPE + }); + + parameters = BigtableParameters.builder().projectId("p").instanceId("i").table("t").build(); + } + + /** Matches production lifecycle: open() initializes the client before pollNext(). */ + private BigtableSourceReader createOpenedReader( + BigtableParameters params, SeaTunnelRowType type) throws Exception { + BigtableSourceReader reader = + new BigtableSourceReader(params, mockContext, type, mockClient); + reader.open(); + return reader; + } + + @SuppressWarnings("unchecked") + private void mockReadStream(Runnable duringForEach) { + ServerStream fakeStream = mock(ServerStream.class); + Mockito.doAnswer( + invocation -> { + if (duringForEach != null) { + duringForEach.run(); + } + return null; + }) + .when(fakeStream) + .forEach(any()); + when(mockDataClient.readRows(any(Query.class))).thenReturn(fakeStream); + } + + // ------------------------------------------------------------------------- + // Issue 1: snapshotState must include the currently-being-read split + // ------------------------------------------------------------------------- + + /** + * When a split is being read (between addSplits and end of readSplit), snapshotState must + * include it so that a failover can re-enqueue it. + */ + @Test + void testSnapshotStateIncludesInFlightSplit() throws Exception { + BigtableSourceSplit split = new BigtableSourceSplit(0, "a", "z"); + final List[] capturedState = new List[1]; + + BigtableSourceReader reader = createOpenedReader(parameters, rowType); + reader.addSplits(Collections.singletonList(split)); + + mockReadStream(() -> capturedState[0] = reader.snapshotState(1L)); + + Collector collector = mock(Collector.class); + when(collector.getCheckpointLock()).thenReturn(new Object()); + + reader.pollNext(collector); + + assertTrue( + capturedState[0].stream().anyMatch(s -> s.splitId().equals(split.splitId())), + "snapshotState taken during readSplit() must include the in-flight split"); + } + + /** + * After readSplit() completes, currentSplit is cleared. A snapshot taken after that must NOT + * re-include the already-finished split. + */ + @Test + void testSnapshotStateAfterReadDoesNotDuplicateSplit() throws Exception { + BigtableSourceSplit split = new BigtableSourceSplit(0, "", ""); + + BigtableSourceReader reader = createOpenedReader(parameters, rowType); + reader.addSplits(Collections.singletonList(split)); + + mockReadStream(null); + + Collector collector = mock(Collector.class); + when(collector.getCheckpointLock()).thenReturn(new Object()); + + reader.pollNext(collector); + + List state = reader.snapshotState(2L); + assertTrue(state.isEmpty(), "State after completed read must be empty"); + } + + // ------------------------------------------------------------------------- + // Issue 2: rows must be emitted via streaming forEach, not buffered + // ------------------------------------------------------------------------- + + /** + * Verifies that each row is emitted individually via output.collect() inside the forEach + * lambda, rather than being buffered first. + */ + @SuppressWarnings("unchecked") + @Test + void testRowsEmittedStreamingNotBuffered() throws Exception { + BigtableSourceSplit split = new BigtableSourceSplit(0, "", ""); + + Row fakeRow = mock(Row.class); + RowCell cell = mock(RowCell.class); + when(cell.getFamily()).thenReturn("cf"); + when(cell.getQualifier()).thenReturn(ByteString.copyFromUtf8("name")); + when(cell.getValue()).thenReturn(ByteString.copyFromUtf8("alice")); + when(fakeRow.getCells()).thenReturn(Collections.singletonList(cell)); + when(fakeRow.getKey()).thenReturn(ByteString.copyFromUtf8("row-1")); + + ServerStream fakeStream = mock(ServerStream.class); + Mockito.doAnswer( + invocation -> { + Consumer action = invocation.getArgument(0); + action.accept(fakeRow); + return null; + }) + .when(fakeStream) + .forEach(any()); + when(mockDataClient.readRows(any(Query.class))).thenReturn(fakeStream); + + BigtableSourceReader reader = createOpenedReader(parameters, rowType); + reader.addSplits(Collections.singletonList(split)); + + Object lock = new Object(); + Collector collector = mock(Collector.class); + when(collector.getCheckpointLock()).thenReturn(lock); + + reader.pollNext(collector); + + ArgumentCaptor captor = ArgumentCaptor.forClass(SeaTunnelRow.class); + verify(collector).collect(captor.capture()); + assertEquals("alice", captor.getValue().getField(1)); + } + + // ------------------------------------------------------------------------- + // Issue 3: rowkey_column config drives field mapping + // ------------------------------------------------------------------------- + + /** + * When rowkey_column is configured, the named field should receive the row key value, not the + * default literal "rowkey". + */ + @SuppressWarnings("unchecked") + @Test + void testRowkeyColumnConfigMapsCorrectField() throws Exception { + SeaTunnelRowType customRowType = + new SeaTunnelRowType( + new String[] {"id", "cf:value"}, + new org.apache.seatunnel.api.table.type.SeaTunnelDataType[] { + BasicType.STRING_TYPE, BasicType.STRING_TYPE + }); + + BigtableParameters paramsWithRowkeyCol = + BigtableParameters.builder() + .projectId("p") + .instanceId("i") + .table("t") + .rowkeyColumns(Collections.singletonList("id")) + .build(); + + Row fakeRow = mock(Row.class); + RowCell cell = mock(RowCell.class); + when(cell.getFamily()).thenReturn("cf"); + when(cell.getQualifier()).thenReturn(ByteString.copyFromUtf8("value")); + when(cell.getValue()).thenReturn(ByteString.copyFromUtf8("hello")); + when(fakeRow.getCells()).thenReturn(Collections.singletonList(cell)); + when(fakeRow.getKey()).thenReturn(ByteString.copyFromUtf8("my-key")); + + ServerStream fakeStream = mock(ServerStream.class); + Mockito.doAnswer( + invocation -> { + Consumer action = invocation.getArgument(0); + action.accept(fakeRow); + return null; + }) + .when(fakeStream) + .forEach(any()); + when(mockDataClient.readRows(any(Query.class))).thenReturn(fakeStream); + + BigtableSourceReader reader = createOpenedReader(paramsWithRowkeyCol, customRowType); + reader.addSplits(Collections.singletonList(new BigtableSourceSplit(0, "", ""))); + + Object lock = new Object(); + Collector collector = mock(Collector.class); + when(collector.getCheckpointLock()).thenReturn(lock); + + reader.pollNext(collector); + + ArgumentCaptor captor = ArgumentCaptor.forClass(SeaTunnelRow.class); + verify(collector).collect(captor.capture()); + assertEquals("my-key", captor.getValue().getField(0)); + assertEquals("hello", captor.getValue().getField(1)); + } +} diff --git a/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplitEnumeratorTest.java b/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplitEnumeratorTest.java new file mode 100644 index 000000000000..b9d764b34a2e --- /dev/null +++ b/seatunnel-connectors-v2/connector-google-bigtable/src/test/java/org/apache/seatunnel/connectors/seatunnel/bigtable/source/BigtableSourceSplitEnumeratorTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.bigtable.source; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class BigtableSourceSplitEnumeratorTest { + + @Test + void testSingleSplitFullRange() { + BigtableSourceSplit split = new BigtableSourceSplit(0, "", ""); + assertEquals("bigtable_source_split_0", split.splitId()); + assertEquals("", split.getStartRowKey()); + assertEquals("", split.getEndRowKey()); + } + + @Test + void testSplitWithRowKeyRange() { + BigtableSourceSplit split = new BigtableSourceSplit(0, "aaa", "zzz"); + assertEquals("aaa", split.getStartRowKey()); + assertEquals("zzz", split.getEndRowKey()); + } +} diff --git a/seatunnel-connectors-v2/pom.xml b/seatunnel-connectors-v2/pom.xml index 6e0266fa6def..bb47dfb58814 100644 --- a/seatunnel-connectors-v2/pom.xml +++ b/seatunnel-connectors-v2/pom.xml @@ -77,6 +77,7 @@ connector-tdengine connector-selectdb-cloud connector-hbase + connector-google-bigtable connector-rocketmq connector-amazonsqs connector-paimon diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index ec8fd7a9518f..92d25d8aa86f 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -418,6 +418,12 @@ ${project.version} provided + + org.apache.seatunnel + connector-google-bigtable + ${project.version} + provided + org.apache.seatunnel connector-datahub From f823b312edcc8033076f6e328fac8148da74df6c Mon Sep 17 00:00:00 2001 From: yzeng1618 Date: Sat, 20 Jun 2026 13:07:29 +0800 Subject: [PATCH 037/375] [Improve][Transform-V2] Improve embedding model invocation reliability (#10863) Co-authored-by: zengyi --- docs/en/transforms/embedding.md | 50 ++- docs/zh/transforms/embedding.md | 38 +- .../nlpmodel/ModelInvocationCache.java | 40 +++ .../nlpmodel/ModelInvocationCacheKey.java | 203 +++++++++++ .../nlpmodel/ModelInvocationContext.java | 56 +++ .../nlpmodel/ModelInvocationErrorType.java | 29 ++ .../nlpmodel/ModelInvocationException.java | 182 ++++++++++ .../nlpmodel/ModelInvocationMetrics.java | 74 ++++ .../nlpmodel/ModelInvocationOptions.java | 76 ++++ .../nlpmodel/ModelInvocationRuntime.java | 331 ++++++++++++++++++ .../nlpmodel/ModelTransformConfig.java | 27 ++ .../transform/nlpmodel/ProviderAdapter.java | 57 +++ .../embedding/EmbeddingTransform.java | 17 +- .../embedding/EmbeddingTransformFactory.java | 7 +- .../embedding/multimodal/MultimodalModel.java | 5 + .../embedding/remote/AbstractModel.java | 9 + .../embedding/remote/amazon/BedrockModel.java | 208 ++++++++++- .../embedding/remote/custom/CustomModel.java | 126 ++++++- .../embedding/remote/doubao/DoubaoModel.java | 209 +++++++++-- .../embedding/remote/openai/OpenAIModel.java | 119 +++++-- .../remote/qianfan/QianfanModel.java | 202 ++++++++--- .../embedding/remote/zhipu/ZhipuModel.java | 114 ++++-- .../EmbeddingModelDimensionTest.java | 111 ++++++ .../nlpmodel/ModelInvocationCacheKeyTest.java | 144 ++++++++ .../ModelInvocationRuntimeCacheTest.java | 226 ++++++++++++ .../ModelInvocationRuntimeMetricsTest.java | 317 +++++++++++++++++ .../nlpmodel/ModelInvocationRuntimeTest.java | 233 ++++++++++++ 27 files changed, 3064 insertions(+), 146 deletions(-) create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCache.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKey.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationContext.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationErrorType.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationException.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationMetrics.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationOptions.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntime.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ProviderAdapter.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKeyTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeCacheTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeMetricsTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeTest.java diff --git a/docs/en/transforms/embedding.md b/docs/en/transforms/embedding.md index 27238c99e350..98dfa0af01be 100644 --- a/docs/en/transforms/embedding.md +++ b/docs/en/transforms/embedding.md @@ -28,6 +28,10 @@ different API endpoints. | custom_response_parse | string | no | | Specifies how to parse the response from the model using JsonPath. Example: `$.choices[*].message.content`. | | custom_request_headers | map | no | | Custom headers for the request to the model. | | custom_request_body | map | no | | Custom body for the request. Supports placeholders like `${model}`, `${input}`. | +| model_retry_max_attempts | int | no | 1 | Maximum attempts for one remote model request. The default value `1` keeps the previous no-retry behavior. | +| model_retry_backoff_ms | long | no | 1000 | Initial backoff in milliseconds before retrying a remote model request. | +| model_retry_max_backoff_ms | long | no | 10000 | Maximum backoff in milliseconds before retrying a remote model request. | +| model_request_timeout_ms | int | no | 20000 | Request timeout in milliseconds for remote model calls. | ## Precision Support @@ -52,9 +56,49 @@ The secret key used for additional authentication. Some providers may require th ### single_vectorized_input_number -Specifies how many inputs are processed in a single vectorization request. The default is 1. Adjust based on your -processing -capacity and the model provider's API limitations. +Specifies how many model inputs are included in one remote vectorization request. The default is 1. Adjust based on your +processing capacity and the model provider's API limitations. + +This is request-level batching inside one row's vectorization inputs. It is not row-level transform micro-batching, and +it does not mean that the transform will collect multiple SeaTunnel rows before calling the provider. + +### Model invocation reliability + +Embedding providers use a common model invocation runtime for remote calls. The provider is still responsible for +provider-specific request body, headers, authentication, response parsing, and provider error conversion. The common +runtime handles timeout propagation, retry, error classification, response count validation, safe logs, metrics hooks, +and the cache boundary. + +The default retry behavior is compatible with previous jobs: `model_retry_max_attempts = 1` means each request is tried +once. When you configure a value greater than 1, retryable failures such as rate limiting, timeout, and temporary remote +service errors can be retried with backoff. Authentication failures, configuration errors, response parse failures, and +response count mismatches are not retried. + +For every request batch, the number of returned vectors must match the number of inputs. If the provider returns fewer or +more vectors than requested, the request fails and the transform does not emit possibly misaligned vectors. + +Retries are performed for the same remote request payload. The transform only emits vectors after a successful response, +but providers can still charge or apply side effects per attempt. Downstream sink idempotency is not changed by this +option. + +The runtime records safe diagnostic context such as provider, model, batch size, attempt number, error category, +retryable flag, and elapsed time. It does not log API keys, secret keys, full source text chunks, binary payloads, or full +provider response bodies. + +Bedrock now uses the same common runtime path as the other embedding providers, so retry, timeout, response parsing, +and response-count validation behave consistently across providers. + +The runtime also has a cache boundary. When a cache implementation is wired in, keys are built from provider, model, +output configuration, modality, format, normalized metadata, and a SHA-256 digest of normalized input content. The +default production wiring still uses `ModelInvocationCache.NOOP`, so existing jobs keep the previous behavior unless an +integration layer enables caching. Existing binary multimodal cache state is unchanged and still only reassembles file +chunks before vectorization. + +Compatibility notes: + +- The default values for `model_retry_max_attempts`, `model_retry_backoff_ms`, and `model_request_timeout_ms` remain unchanged. +- No user-facing config names were renamed or removed in this update. +- The cache integration is additive and does not change the default execution path. ### vectorization_fields diff --git a/docs/zh/transforms/embedding.md b/docs/zh/transforms/embedding.md index 9422181449c5..b8ace6ca6b8b 100644 --- a/docs/zh/transforms/embedding.md +++ b/docs/zh/transforms/embedding.md @@ -26,6 +26,10 @@ Embedding 转换插件利用 embedding 模型将文本和多模态数据转换 | custom_response_parse | string | 否 | | 使用 JsonPath 解析模型响应的方式。示例：`$.choices[*].message.content`。 | | custom_request_headers | map | 否 | | 发送到模型的请求的自定义头信息。 | | custom_request_body | map | 否 | | 请求体的自定义配置。支持占位符如 `${model}`、`${input}`。 | +| model_retry_max_attempts | int | 否 | 1 | 单个远程模型请求的最大尝试次数。默认值 `1` 表示保持原有不自动重试行为。 | +| model_retry_backoff_ms | long | 否 | 1000 | 远程模型请求重试前的初始退避时间，单位毫秒。 | +| model_retry_max_backoff_ms | long | 否 | 10000 | 远程模型请求重试前的最大退避时间，单位毫秒。 | +| model_request_timeout_ms | int | 否 | 20000 | 远程模型调用的请求超时时间，单位毫秒。 | ## 精度支持 @@ -49,7 +53,39 @@ Embedding 转换插件利用 embedding 模型将文本和多模态数据转换 ### single_vectorized_input_number -指定单次请求向量化的输入数量。默认值为1。根据处理能力和模型提供商的API限制进行调整。 +指定一个远程向量化请求中包含的模型输入数量。默认值为1。根据处理能力和模型提供商的API限制进行调整。 + +这是 request-level 的批处理语义，只作用于一行数据中的多个待向量化输入。它不是 row-level transform micro-batching， +也不表示 Transform 会先收集多行 SeaTunnel row 再调用模型提供商。 + +### 模型调用可靠性 + +Embedding provider 通过通用模型调用运行时执行远程调用。Provider 仍然负责 provider-specific 的请求体、请求头、认证、 +响应解析以及 provider 错误转换；通用运行时负责超时传递、重试、错误分类、响应数量校验、安全日志、指标 hook 和缓存边界。 + +默认重试行为与已有任务兼容：`model_retry_max_attempts = 1` 表示每个请求只尝试一次。配置为大于 1 后，限流、超时、 +临时远端服务错误等可重试失败可以按退避策略重试。认证失败、配置错误、响应解析失败和返回 vector 数量不匹配不会重试。 + +每个 request batch 都必须为每个输入返回且仅返回一个 vector。如果 provider 返回的 vector 数量少于或多于输入数量， +该请求会失败，Transform 不会输出可能错位的向量。 + +重试会对同一个远程请求 payload 再次尝试。Transform 只有在拿到成功响应后才输出向量，但 provider 仍可能按每次尝试计费或产生 +provider 侧副作用。该配置不会改变下游 Sink 的幂等语义。 + +运行时会记录 provider、model、batch size、attempt number、error category、retryable flag、elapsed time 等安全诊断上下文。 +日志不会记录 API key、secret key、完整源文本 chunk、二进制 payload 或完整 provider response body。 + +Bedrock 现在也走统一的 common runtime 路径，因此 retry、timeout、响应解析和返回数量校验在各个 provider 之间保持一致。 + +运行时也提供了一个 cache 边界。当接入 cache 实现时，key 由 provider、model、输出配置、modality、format、规范化后的 metadata， +以及规范化输入内容的 SHA-256 摘要组成。默认的生产 wiring 仍然使用 `ModelInvocationCache.NOOP`，因此在接入层显式启用缓存之前， +现有任务的行为保持不变。现有的 binary multimodal cache 行为不变，仍然只用于向量化前的文件分片重组。 + +兼容性说明： + +- `model_retry_max_attempts`、`model_retry_backoff_ms` 和 `model_request_timeout_ms` 的默认值保持不变。 +- 本次更新没有重命名或删除任何用户可见的配置项。 +- cache 集成是增量能力，不会改变默认执行路径。 ### vectorization_fields diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCache.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCache.java new file mode 100644 index 000000000000..a939de89ad76 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCache.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import java.util.Optional; + +public interface ModelInvocationCache { + + ModelInvocationCache NOOP = new NoopModelInvocationCache(); + + Optional get(String key); + + void put(String key, T value); + + class NoopModelInvocationCache implements ModelInvocationCache { + + @Override + public Optional get(String key) { + return Optional.empty(); + } + + @Override + public void put(String key, T value) {} + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKey.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKey.java new file mode 100644 index 000000000000..d9ae64dfba40 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKey.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Map; +import java.util.TreeMap; + +/** + * Builds a stable cache key for embedding invocation inputs. + * + *

The key is designed to avoid exposing raw content while still distinguishing provider, model, + * output configuration, modality/format, and metadata that participate in cache identity. + */ +public final class ModelInvocationCacheKey { + + private static final String PREFIX = "embedding-cache:v1"; + + private ModelInvocationCacheKey() {} + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + private String provider; + private String model; + private final TreeMap outputConfigurations = new TreeMap<>(); + private String modality; + private String format; + private final TreeMap metadata = new TreeMap<>(); + private Object input; + + public Builder provider(String provider) { + this.provider = provider; + return this; + } + + public Builder model(String model) { + this.model = model; + return this; + } + + public Builder dimension(Integer dimension) { + if (dimension != null) { + this.outputConfigurations.put("dimension", String.valueOf(dimension)); + } + return this; + } + + public Builder outputConfiguration(String name, Object value) { + if (name != null && value != null) { + this.outputConfigurations.put(normalizeToken(name), normalizeValue(value)); + } + return this; + } + + public Builder modality(String modality) { + this.modality = modality; + return this; + } + + public Builder format(String format) { + this.format = format; + return this; + } + + public Builder metadata(String key, Object value) { + if (key != null && value != null) { + this.metadata.put(normalizeToken(key), normalizeValue(value)); + } + return this; + } + + public Builder input(Object input) { + this.input = input; + return this; + } + + public String build() { + StringBuilder key = new StringBuilder(PREFIX); + appendSection(key, "provider", normalizeToken(provider)); + appendSection(key, "model", normalizeValue(model)); + if (!outputConfigurations.isEmpty()) { + appendSection(key, "output_config", canonicalize(outputConfigurations)); + } + appendSection(key, "modality", normalizeToken(modality)); + appendSection(key, "format", normalizeToken(format)); + if (!metadata.isEmpty()) { + appendSection(key, "metadata_sha256", sha256(canonicalize(metadata))); + } + appendSection(key, "input_sha256", digestInput(input)); + return key.toString(); + } + } + + private static void appendSection(StringBuilder key, String label, String value) { + if (value == null || value.isEmpty()) { + return; + } + key.append('|').append(label).append('=').append(value); + } + + private static String canonicalize(Map values) { + StringBuilder canonical = new StringBuilder(); + boolean first = true; + for (Map.Entry entry : values.entrySet()) { + if (!first) { + canonical.append(','); + } + canonical.append(entry.getKey()).append('=').append(entry.getValue()); + first = false; + } + return canonical.toString(); + } + + private static String normalizeToken(String value) { + if (value == null) { + return null; + } + String normalized = value.trim(); + if (normalized.isEmpty()) { + return null; + } + return normalized.toLowerCase(); + } + + private static String normalizeValue(Object value) { + if (value == null) { + return null; + } + String normalized = String.valueOf(value).trim(); + return normalized.isEmpty() ? null : normalized; + } + + private static String digestInput(Object input) { + MessageDigest digest = newDigest(); + if (input == null) { + digest.update(new byte[0]); + return hex(digest.digest()); + } + if (input instanceof byte[]) { + digest.update((byte[]) input); + return hex(digest.digest()); + } + if (input instanceof ByteBuffer) { + ByteBuffer buffer = ((ByteBuffer) input).duplicate(); + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + digest.update(bytes); + return hex(digest.digest()); + } + String normalized = normalizeLineEndings(String.valueOf(input)); + digest.update(normalized.getBytes(StandardCharsets.UTF_8)); + return hex(digest.digest()); + } + + private static String normalizeLineEndings(String value) { + return value.replace("\r\n", "\n").replace('\r', '\n'); + } + + private static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 digest is required but unavailable", e); + } + } + + private static String sha256(String value) { + MessageDigest digest = newDigest(); + digest.update(value.getBytes(StandardCharsets.UTF_8)); + return hex(digest.digest()); + } + + private static String hex(byte[] bytes) { + StringBuilder hex = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + hex.append(Character.forDigit((b >>> 4) & 0x0F, 16)); + hex.append(Character.forDigit(b & 0x0F, 16)); + } + return hex.toString(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationContext.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationContext.java new file mode 100644 index 000000000000..36266260fb99 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationContext.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +public class ModelInvocationContext { + + private final String provider; + private final String model; + private final int inputCount; + private final int attempt; + private final int requestTimeoutMs; + + public ModelInvocationContext( + String provider, String model, int inputCount, int attempt, int requestTimeoutMs) { + this.provider = provider; + this.model = model; + this.inputCount = inputCount; + this.attempt = attempt; + this.requestTimeoutMs = requestTimeoutMs; + } + + public String getProvider() { + return provider; + } + + public String getModel() { + return model; + } + + public int getInputCount() { + return inputCount; + } + + public int getAttempt() { + return attempt; + } + + public int getRequestTimeoutMs() { + return requestTimeoutMs; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationErrorType.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationErrorType.java new file mode 100644 index 000000000000..d76273bb80a0 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationErrorType.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +public enum ModelInvocationErrorType { + RATE_LIMIT, + TIMEOUT, + TEMPORARY_REMOTE_ERROR, + AUTHENTICATION_ERROR, + CONFIGURATION_ERROR, + RESPONSE_PARSE_ERROR, + RESPONSE_COUNT_MISMATCH, + UNKNOWN_REMOTE_ERROR +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationException.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationException.java new file mode 100644 index 000000000000..a5126e60d3d0 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationException.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import java.io.IOException; + +public class ModelInvocationException extends IOException { + + private static final int MAX_MESSAGE_LENGTH = 256; + + private final ModelInvocationErrorType errorType; + private final boolean retryable; + private final String provider; + private final String model; + private final Integer httpStatus; + + private ModelInvocationException( + ModelInvocationErrorType errorType, + boolean retryable, + String provider, + String model, + String message, + Integer httpStatus, + Throwable cause) { + super(buildMessage(errorType, retryable, provider, model, message, httpStatus), cause); + this.errorType = errorType; + this.retryable = retryable; + this.provider = provider; + this.model = model; + this.httpStatus = httpStatus; + } + + public static ModelInvocationException retryable( + ModelInvocationErrorType errorType, + String provider, + String model, + String message, + Integer httpStatus, + Throwable cause) { + return new ModelInvocationException( + errorType, true, provider, model, message, httpStatus, cause); + } + + public static ModelInvocationException nonRetryable( + ModelInvocationErrorType errorType, + String provider, + String model, + String message, + Integer httpStatus, + Throwable cause) { + return new ModelInvocationException( + errorType, false, provider, model, message, httpStatus, cause); + } + + public static ModelInvocationException nonRetryable( + ModelInvocationErrorType errorType, + String provider, + String model, + String message, + Throwable cause) { + return nonRetryable(errorType, provider, model, message, null, cause); + } + + public static ModelInvocationException fromHttpStatus( + String provider, String model, int statusCode, String ignoredResponseBody) { + if (statusCode == 429) { + return retryable( + ModelInvocationErrorType.RATE_LIMIT, + provider, + model, + "HTTP status " + statusCode, + statusCode, + null); + } + if (statusCode >= 500) { + return retryable( + ModelInvocationErrorType.TEMPORARY_REMOTE_ERROR, + provider, + model, + "HTTP status " + statusCode, + statusCode, + null); + } + if (statusCode == 401 || statusCode == 403) { + return nonRetryable( + ModelInvocationErrorType.AUTHENTICATION_ERROR, + provider, + model, + "HTTP status " + statusCode, + statusCode, + null); + } + if (statusCode >= 400) { + return nonRetryable( + ModelInvocationErrorType.CONFIGURATION_ERROR, + provider, + model, + "HTTP status " + statusCode, + statusCode, + null); + } + return nonRetryable( + ModelInvocationErrorType.UNKNOWN_REMOTE_ERROR, + provider, + model, + "HTTP status " + statusCode, + statusCode, + null); + } + + public ModelInvocationErrorType getErrorType() { + return errorType; + } + + public boolean isRetryable() { + return retryable; + } + + public String getProvider() { + return provider; + } + + public String getModel() { + return model; + } + + public Integer getHttpStatus() { + return httpStatus; + } + + private static String buildMessage( + ModelInvocationErrorType errorType, + boolean retryable, + String provider, + String model, + String message, + Integer httpStatus) { + StringBuilder builder = new StringBuilder(); + builder.append("Model invocation failed"); + builder.append(": provider=").append(provider); + builder.append(", model=").append(model); + builder.append(", errorType=").append(errorType); + builder.append(", retryable=").append(retryable); + if (httpStatus != null) { + builder.append(", httpStatus=").append(httpStatus); + } + String sanitized = sanitize(message); + if (sanitized != null && !sanitized.isEmpty()) { + builder.append(", message=").append(sanitized); + } + return builder.toString(); + } + + private static String sanitize(String message) { + if (message == null) { + return null; + } + String sanitized = + message.replaceAll("(?i)(api[_-]?key\\s*[:=]\\s*)\\S+", "$1***") + .replaceAll("(?i)(secret[_-]?key\\s*[:=]\\s*)\\S+", "$1***") + .replaceAll("(?i)(authorization\\s*[:=]\\s*bearer\\s+)\\S+", "$1***"); + if (sanitized.length() > MAX_MESSAGE_LENGTH) { + return sanitized.substring(0, MAX_MESSAGE_LENGTH) + "..."; + } + return sanitized; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationMetrics.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationMetrics.java new file mode 100644 index 000000000000..b1534e307afa --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationMetrics.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +public interface ModelInvocationMetrics { + + ModelInvocationMetrics NOOP = new NoopModelInvocationMetrics(); + + void recordRequest(String provider, String model); + + void recordFailure(String provider, String model, ModelInvocationErrorType errorType); + + void recordRetry(String provider, String model, ModelInvocationErrorType errorType); + + void recordRetryExhausted(String provider, String model, ModelInvocationErrorType errorType); + + void recordResponseCountMismatch(String provider, String model); + + void recordGeneratedOutputs(String provider, String model, int outputCount); + + void recordLatency(String provider, String model, long elapsedMs); + + void recordCacheHit(String provider, String model, int hitCount); + + void recordCacheMiss(String provider, String model, int missCount); + + class NoopModelInvocationMetrics implements ModelInvocationMetrics { + + @Override + public void recordRequest(String provider, String model) {} + + @Override + public void recordFailure( + String provider, String model, ModelInvocationErrorType errorType) {} + + @Override + public void recordRetry( + String provider, String model, ModelInvocationErrorType errorType) {} + + @Override + public void recordRetryExhausted( + String provider, String model, ModelInvocationErrorType errorType) {} + + @Override + public void recordResponseCountMismatch(String provider, String model) {} + + @Override + public void recordGeneratedOutputs(String provider, String model, int outputCount) {} + + @Override + public void recordLatency(String provider, String model, long elapsedMs) {} + + @Override + public void recordCacheHit(String provider, String model, int hitCount) {} + + @Override + public void recordCacheMiss(String provider, String model, int missCount) {} + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationOptions.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationOptions.java new file mode 100644 index 000000000000..076418c6141b --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationOptions.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; + +public class ModelInvocationOptions { + + public static final int DEFAULT_RETRY_MAX_ATTEMPTS = 1; + public static final long DEFAULT_RETRY_BACKOFF_MS = 1000L; + public static final long DEFAULT_RETRY_MAX_BACKOFF_MS = 10000L; + public static final int DEFAULT_REQUEST_TIMEOUT_MS = 20000; + + private final int retryMaxAttempts; + private final long retryBackoffMs; + private final long retryMaxBackoffMs; + private final int requestTimeoutMs; + + public ModelInvocationOptions( + int retryMaxAttempts, + long retryBackoffMs, + long retryMaxBackoffMs, + int requestTimeoutMs) { + this.retryMaxAttempts = Math.max(1, retryMaxAttempts); + this.retryBackoffMs = Math.max(0L, retryBackoffMs); + this.retryMaxBackoffMs = Math.max(0L, retryMaxBackoffMs); + this.requestTimeoutMs = Math.max(1, requestTimeoutMs); + } + + public static ModelInvocationOptions defaults() { + return new ModelInvocationOptions( + DEFAULT_RETRY_MAX_ATTEMPTS, + DEFAULT_RETRY_BACKOFF_MS, + DEFAULT_RETRY_MAX_BACKOFF_MS, + DEFAULT_REQUEST_TIMEOUT_MS); + } + + public static ModelInvocationOptions fromConfig(ReadonlyConfig config) { + return new ModelInvocationOptions( + config.get(ModelTransformConfig.MODEL_RETRY_MAX_ATTEMPTS), + config.get(ModelTransformConfig.MODEL_RETRY_BACKOFF_MS), + config.get(ModelTransformConfig.MODEL_RETRY_MAX_BACKOFF_MS), + config.get(ModelTransformConfig.MODEL_REQUEST_TIMEOUT_MS)); + } + + public int getRetryMaxAttempts() { + return retryMaxAttempts; + } + + public long getRetryBackoffMs() { + return retryBackoffMs; + } + + public long getRetryMaxBackoffMs() { + return retryMaxBackoffMs; + } + + public int getRequestTimeoutMs() { + return requestTimeoutMs; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntime.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntime.java new file mode 100644 index 000000000000..b34c1fbbcea8 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntime.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import org.apache.http.conn.ConnectTimeoutException; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +@Slf4j +public class ModelInvocationRuntime { + + private final ModelInvocationOptions options; + private final ModelInvocationMetrics metrics; + private final ModelInvocationCache cache; + + public ModelInvocationRuntime(ModelInvocationOptions options) { + this(options, ModelInvocationMetrics.NOOP, ModelInvocationCache.NOOP); + } + + public ModelInvocationRuntime( + ModelInvocationOptions options, + ModelInvocationMetrics metrics, + ModelInvocationCache cache) { + this.options = options == null ? ModelInvocationOptions.defaults() : options; + this.metrics = metrics == null ? ModelInvocationMetrics.NOOP : metrics; + this.cache = cache == null ? ModelInvocationCache.NOOP : cache; + } + + public T invoke(Object[] inputs, ProviderAdapter adapter) throws IOException { + int inputCount = inputs == null ? 0 : inputs.length; + if (cache != ModelInvocationCache.NOOP && inputCount > 0) { + CacheLookup cacheLookup = lookupCache(inputs, adapter); + if (cacheLookup.isCompleteHit()) { + metrics.recordCacheHit(adapter.getProvider(), adapter.getModel(), inputCount); + return castOutput(cacheLookup.getCachedOutput()); + } + + int hitCount = inputCount - cacheLookup.getMissingCount(); + if (hitCount > 0) { + metrics.recordCacheHit(adapter.getProvider(), adapter.getModel(), hitCount); + } + if (cacheLookup.getMissingCount() > 0) { + metrics.recordCacheMiss( + adapter.getProvider(), adapter.getModel(), cacheLookup.getMissingCount()); + } + + T output = + invokeWithRetries( + cacheLookup.getMissingInputs(), adapter, cacheLookup.getMissingCount()); + List> remoteVectors = asVectors(output); + cacheLookup.putMissing(remoteVectors, cache); + return castOutput(cacheLookup.merge(remoteVectors)); + } + + return invokeWithRetries(inputs, adapter, inputCount); + } + + public ModelInvocationCache getCache() { + return cache; + } + + private T invokeWithRetries(Object[] inputs, ProviderAdapter adapter, int inputCount) + throws IOException { + for (int attempt = 1; attempt <= options.getRetryMaxAttempts(); attempt++) { + ModelInvocationContext context = + new ModelInvocationContext( + adapter.getProvider(), + adapter.getModel(), + inputCount, + attempt, + options.getRequestTimeoutMs()); + long start = System.currentTimeMillis(); + metrics.recordRequest(context.getProvider(), context.getModel()); + try { + T output = adapter.invoke(inputs, context); + validateOutputCount(inputCount, adapter, output, context); + metrics.recordGeneratedOutputs( + context.getProvider(), context.getModel(), adapter.getOutputCount(output)); + metrics.recordLatency( + context.getProvider(), + context.getModel(), + System.currentTimeMillis() - start); + return output; + } catch (IOException e) { + handleInvocationException(normalize(e, context), context, attempt, start); + } catch (RuntimeException e) { + handleInvocationException(normalize(e, context), context, attempt, start); + } + } + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.UNKNOWN_REMOTE_ERROR, + "UNKNOWN", + "UNKNOWN", + "No model invocation attempt was executed", + null); + } + + private CacheLookup lookupCache(Object[] inputs, ProviderAdapter adapter) { + List keys = new ArrayList<>(inputs.length); + List> cachedOutput = new ArrayList<>(inputs.length); + List missingIndices = new ArrayList<>(); + List missingInputs = new ArrayList<>(); + for (int i = 0; i < inputs.length; i++) { + Object input = inputs[i]; + String key = buildCacheKey(adapter, input); + keys.add(key); + Optional cachedValue = cache.get(key); + if (cachedValue.isPresent()) { + cachedOutput.add(asVector(cachedValue.get())); + continue; + } + cachedOutput.add(null); + missingIndices.add(i); + missingInputs.add(input); + } + return new CacheLookup( + keys, cachedOutput, missingIndices, missingInputs.toArray(new Object[0])); + } + + private String buildCacheKey(ProviderAdapter adapter, Object input) { + ModelInvocationCacheKey.Builder builder = + ModelInvocationCacheKey.builder() + .provider(adapter.getProvider()) + .model(adapter.getModel()) + .dimension(adapter.getDimension()) + .modality(adapter.getInputModality(input)) + .format(adapter.getInputFormat(input)) + .input(input); + Map metadata = adapter.getCacheMetadata(input); + if (metadata != null) { + metadata.forEach(builder::metadata); + } + return builder.build(); + } + + @SuppressWarnings("unchecked") + private List> asVectors(Object output) { + return (List>) output; + } + + @SuppressWarnings("unchecked") + private List asVector(Object value) { + return (List) value; + } + + @SuppressWarnings("unchecked") + private T castOutput(List> output) { + return (T) output; + } + + private void validateOutputCount( + int inputCount, ProviderAdapter adapter, T output, ModelInvocationContext context) + throws ModelInvocationException { + if (!adapter.validateOutputCount()) { + return; + } + int outputCount = adapter.getOutputCount(output); + if (inputCount != outputCount) { + metrics.recordResponseCountMismatch(context.getProvider(), context.getModel()); + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_COUNT_MISMATCH, + context.getProvider(), + context.getModel(), + "Expected " + inputCount + " outputs, but got " + outputCount, + null); + } + } + + private void handleInvocationException( + ModelInvocationException invocationException, + ModelInvocationContext context, + int attempt, + long start) + throws ModelInvocationException { + metrics.recordFailure( + context.getProvider(), context.getModel(), invocationException.getErrorType()); + metrics.recordLatency( + context.getProvider(), context.getModel(), System.currentTimeMillis() - start); + + boolean canRetry = + invocationException.isRetryable() && attempt < options.getRetryMaxAttempts(); + logInvocationFailure( + context, invocationException, canRetry, System.currentTimeMillis() - start); + if (!canRetry) { + if (invocationException.isRetryable()) { + metrics.recordRetryExhausted( + context.getProvider(), + context.getModel(), + invocationException.getErrorType()); + } + throw invocationException; + } + metrics.recordRetry( + context.getProvider(), context.getModel(), invocationException.getErrorType()); + sleepBeforeRetry(attempt); + } + + private ModelInvocationException normalize( + Exception exception, ModelInvocationContext context) { + if (exception instanceof ModelInvocationException) { + return (ModelInvocationException) exception; + } + if (exception instanceof SocketTimeoutException + || exception instanceof ConnectTimeoutException) { + return ModelInvocationException.retryable( + ModelInvocationErrorType.TIMEOUT, + context.getProvider(), + context.getModel(), + "Request timeout", + null, + exception); + } + return ModelInvocationException.nonRetryable( + ModelInvocationErrorType.UNKNOWN_REMOTE_ERROR, + context.getProvider(), + context.getModel(), + "Unexpected model invocation failure", + null, + exception); + } + + private void logInvocationFailure( + ModelInvocationContext context, + ModelInvocationException exception, + boolean willRetry, + long elapsedMs) { + log.warn( + "Model invocation failed: provider={}, model={}, batchSize={}, attempt={}, errorType={}, retryable={}, willRetry={}, elapsedMs={}", + context.getProvider(), + context.getModel(), + context.getInputCount(), + context.getAttempt(), + exception.getErrorType(), + exception.isRetryable(), + willRetry, + elapsedMs); + } + + private void sleepBeforeRetry(int attempt) throws ModelInvocationException { + long backoff = options.getRetryBackoffMs(); + if (backoff <= 0) { + return; + } + long cappedBackoff = + options.getRetryMaxBackoffMs() <= 0 + ? backoff + : Math.min(backoff * attempt, options.getRetryMaxBackoffMs()); + try { + Thread.sleep(cappedBackoff); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.UNKNOWN_REMOTE_ERROR, + "UNKNOWN", + "UNKNOWN", + "Interrupted while waiting to retry model invocation", + e); + } + } + + private static final class CacheLookup { + + private final List keys; + private final List> cachedOutput; + private final List missingIndices; + private final Object[] missingInputs; + + private CacheLookup( + List keys, + List> cachedOutput, + List missingIndices, + Object[] missingInputs) { + this.keys = keys; + this.cachedOutput = cachedOutput; + this.missingIndices = missingIndices; + this.missingInputs = missingInputs; + } + + private boolean isCompleteHit() { + return missingIndices.isEmpty(); + } + + private List> getCachedOutput() { + return cachedOutput; + } + + private Object[] getMissingInputs() { + return missingInputs; + } + + private int getMissingCount() { + return missingInputs.length; + } + + private void putMissing(List> remoteVectors, ModelInvocationCache cache) { + for (int i = 0; i < missingIndices.size(); i++) { + int index = missingIndices.get(i); + cache.put(keys.get(index), remoteVectors.get(i)); + } + } + + private List> merge(List> remoteVectors) { + for (int i = 0; i < missingIndices.size(); i++) { + cachedOutput.set(missingIndices.get(i), remoteVectors.get(i)); + } + return cachedOutput; + } + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelTransformConfig.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelTransformConfig.java index b1448dcde9b3..963b2bbdf9d7 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelTransformConfig.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelTransformConfig.java @@ -85,6 +85,33 @@ public class ModelTransformConfig implements Serializable { .withFallbackKeys("inference_batch_size") .withDescription("The row batch size of each process"); + public static final Option MODEL_RETRY_MAX_ATTEMPTS = + Options.key("model_retry_max_attempts") + .intType() + .defaultValue(ModelInvocationOptions.DEFAULT_RETRY_MAX_ATTEMPTS) + .withDescription( + "The maximum attempts for one remote model request. The default value 1 means no automatic retry."); + + public static final Option MODEL_RETRY_BACKOFF_MS = + Options.key("model_retry_backoff_ms") + .longType() + .defaultValue(ModelInvocationOptions.DEFAULT_RETRY_BACKOFF_MS) + .withDescription( + "The initial backoff in milliseconds before retrying a remote model request."); + + public static final Option MODEL_RETRY_MAX_BACKOFF_MS = + Options.key("model_retry_max_backoff_ms") + .longType() + .defaultValue(ModelInvocationOptions.DEFAULT_RETRY_MAX_BACKOFF_MS) + .withDescription( + "The maximum backoff in milliseconds before retrying a remote model request."); + + public static final Option MODEL_REQUEST_TIMEOUT_MS = + Options.key("model_request_timeout_ms") + .intType() + .defaultValue(ModelInvocationOptions.DEFAULT_REQUEST_TIMEOUT_MS) + .withDescription("The request timeout in milliseconds for remote model calls."); + public static final Option DIMENSION = Options.key("dimension").intType().defaultValue(2048).withDescription("dimension"); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ProviderAdapter.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ProviderAdapter.java new file mode 100644 index 000000000000..f364cfdc7c14 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ProviderAdapter.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +public interface ProviderAdapter { + + T invoke(Object[] inputs, ModelInvocationContext context) throws IOException; + + int getOutputCount(T output); + + default String getProvider() { + return "UNKNOWN"; + } + + default String getModel() { + return "UNKNOWN"; + } + + default Integer getDimension() { + return null; + } + + default String getInputModality(Object input) { + return null; + } + + default String getInputFormat(Object input) { + return null; + } + + default Map getCacheMetadata(Object input) { + return Collections.emptyMap(); + } + + default boolean validateOutputCount() { + return true; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java index 8857e6264c62..6e7ba72a3aa8 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java @@ -29,6 +29,7 @@ import org.apache.seatunnel.api.table.type.VectorType; import org.apache.seatunnel.transform.common.MultipleFieldOutputTransform; import org.apache.seatunnel.transform.exception.TransformCommonError; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; import org.apache.seatunnel.transform.nlpmodel.ModelProvider; import org.apache.seatunnel.transform.nlpmodel.ModelTransformConfig; import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.MultimodalFieldValue; @@ -88,6 +89,7 @@ private void tryOpen() { @Override public void open() { ModelProvider provider = config.get(ModelTransformConfig.MODEL_PROVIDER); + ModelInvocationOptions invocationOptions = ModelInvocationOptions.fromConfig(config); String apiPath = provider.usedEmbeddingPath( config.get(ModelTransformConfig.API_PATH), isMultimodalFields); @@ -118,7 +120,8 @@ public void open() { .CUSTOM_RESPONSE_PARSE), config.get( EmbeddingTransformConfig - .SINGLE_VECTORIZED_INPUT_NUMBER)); + .SINGLE_VECTORIZED_INPUT_NUMBER), + invocationOptions); break; case OPENAI: model = @@ -128,7 +131,8 @@ public void open() { apiPath, config.get( EmbeddingTransformConfig - .SINGLE_VECTORIZED_INPUT_NUMBER)); + .SINGLE_VECTORIZED_INPUT_NUMBER), + invocationOptions); break; case DOUBAO: model = @@ -139,7 +143,8 @@ public void open() { config.get( EmbeddingTransformConfig .SINGLE_VECTORIZED_INPUT_NUMBER), - isMultimodalFields); + isMultimodalFields, + invocationOptions); break; case QIANFAN: model = @@ -151,7 +156,8 @@ public void open() { config.get(ModelTransformConfig.OAUTH_PATH), config.get( EmbeddingTransformConfig - .SINGLE_VECTORIZED_INPUT_NUMBER)); + .SINGLE_VECTORIZED_INPUT_NUMBER), + invocationOptions); break; case ZHIPU: @@ -163,7 +169,8 @@ public void open() { config.get(ModelTransformConfig.DIMENSION), config.get( EmbeddingTransformConfig - .SINGLE_VECTORIZED_INPUT_NUMBER)); + .SINGLE_VECTORIZED_INPUT_NUMBER), + invocationOptions); break; case AMAZON: model = diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransformFactory.java index 7455a4ba4bd7..f75a1eb430bb 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransformFactory.java @@ -26,6 +26,7 @@ import org.apache.seatunnel.api.table.factory.TableTransformFactoryContext; import org.apache.seatunnel.transform.common.TransformCommonOptions; import org.apache.seatunnel.transform.nlpmodel.ModelProvider; +import org.apache.seatunnel.transform.nlpmodel.ModelTransformConfig; import org.apache.seatunnel.transform.nlpmodel.llm.LLMTransformConfig; import com.google.auto.service.AutoService; @@ -47,7 +48,11 @@ public OptionRule optionRule() { .optional( EmbeddingTransformConfig.API_PATH, EmbeddingTransformConfig.SINGLE_VECTORIZED_INPUT_NUMBER, - EmbeddingTransformConfig.PROCESS_BATCH_SIZE) + EmbeddingTransformConfig.PROCESS_BATCH_SIZE, + ModelTransformConfig.MODEL_RETRY_MAX_ATTEMPTS, + ModelTransformConfig.MODEL_RETRY_BACKOFF_MS, + ModelTransformConfig.MODEL_RETRY_MAX_BACKOFF_MS, + ModelTransformConfig.MODEL_REQUEST_TIMEOUT_MS) .conditional( EmbeddingTransformConfig.MODEL_PROVIDER, ModelProvider.AMAZON, diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/MultimodalModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/MultimodalModel.java index 57142e0308d6..676868b06815 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/MultimodalModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/MultimodalModel.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.shade.com.google.common.annotations.VisibleForTesting; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.AbstractModel; import java.io.IOException; @@ -33,6 +34,10 @@ public MultimodalModel(Integer vectorizedNumber) { super(vectorizedNumber); } + public MultimodalModel(Integer vectorizedNumber, ModelInvocationOptions invocationOptions) { + super(vectorizedNumber, invocationOptions); + } + @Override protected final List> vector(Object[] fields) throws IOException { if (isMultimodalFields(fields)) { diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/AbstractModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/AbstractModel.java index 42334609c8cb..c178ce0cbf87 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/AbstractModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/AbstractModel.java @@ -21,6 +21,8 @@ import org.apache.seatunnel.shade.org.apache.commons.lang3.ArrayUtils; import org.apache.seatunnel.common.utils.VectorUtils; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationRuntime; import java.io.IOException; import java.nio.ByteBuffer; @@ -32,9 +34,16 @@ public abstract class AbstractModel implements Model { protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); protected static final String DIMENSION_EXAMPLE = "dimension example"; protected final Integer singleVectorizedInputNumber; + protected final ModelInvocationRuntime invocationRuntime; protected AbstractModel(Integer singleVectorizedInputNumber) { + this(singleVectorizedInputNumber, ModelInvocationOptions.defaults()); + } + + protected AbstractModel( + Integer singleVectorizedInputNumber, ModelInvocationOptions invocationOptions) { this.singleVectorizedInputNumber = singleVectorizedInputNumber; + this.invocationRuntime = new ModelInvocationRuntime(invocationOptions); } @Override diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/amazon/BedrockModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/amazon/BedrockModel.java index 35ab49df22a6..85d46f4c21b1 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/amazon/BedrockModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/amazon/BedrockModel.java @@ -21,19 +21,40 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ArrayNode; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationContext; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationErrorType; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationException; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; +import org.apache.seatunnel.transform.nlpmodel.ProviderAdapter; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.AbstractModel; +import org.apache.http.conn.ConnectTimeoutException; + import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient; import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClientBuilder; +import software.amazon.awssdk.services.bedrockruntime.model.AccessDeniedException; +import software.amazon.awssdk.services.bedrockruntime.model.BedrockRuntimeException; +import software.amazon.awssdk.services.bedrockruntime.model.ConflictException; +import software.amazon.awssdk.services.bedrockruntime.model.InternalServerException; import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelRequest; import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelResponse; +import software.amazon.awssdk.services.bedrockruntime.model.ModelErrorException; +import software.amazon.awssdk.services.bedrockruntime.model.ModelNotReadyException; +import software.amazon.awssdk.services.bedrockruntime.model.ModelTimeoutException; +import software.amazon.awssdk.services.bedrockruntime.model.ResourceNotFoundException; +import software.amazon.awssdk.services.bedrockruntime.model.ServiceQuotaExceededException; +import software.amazon.awssdk.services.bedrockruntime.model.ServiceUnavailableException; +import software.amazon.awssdk.services.bedrockruntime.model.ThrottlingException; +import software.amazon.awssdk.services.bedrockruntime.model.ValidationException; import java.io.IOException; +import java.net.SocketTimeoutException; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -126,7 +147,32 @@ public BedrockModel(BedrockRuntimeClient client, String modelId, int dimension, modelId, dimension, batchSize, - modelId.startsWith("cohere.") ? "search_document" : null); + modelId.startsWith("cohere.") ? "search_document" : null, + ModelInvocationOptions.defaults()); + } + + /** + * Create a BedrockModel instance with an existing BedrockRuntimeClient and invocation options. + * + * @param client BedrockRuntimeClient instance + * @param modelId Model ID (e.g., "amazon.titan-embed-text-v1", "cohere.embed-english-v3") + * @param dimension Embedding dimension + * @param batchSize Batch size for processing + * @param invocationOptions Runtime options for retry, timeout, and backoff + */ + public BedrockModel( + BedrockRuntimeClient client, + String modelId, + int dimension, + int batchSize, + ModelInvocationOptions invocationOptions) { + this( + client, + modelId, + dimension, + batchSize, + modelId.startsWith("cohere.") ? "search_document" : null, + invocationOptions); } /** @@ -144,7 +190,28 @@ public BedrockModel( int dimension, int batchSize, String inputType) { - super(batchSize); + this(client, modelId, dimension, batchSize, inputType, ModelInvocationOptions.defaults()); + } + + /** + * Create a BedrockModel instance with an existing BedrockRuntimeClient, input type, and + * invocation options. + * + * @param client BedrockRuntimeClient instance + * @param modelId Model ID (e.g., "amazon.titan-embed-text-v1", "cohere.embed-english-v3") + * @param dimension Embedding dimension + * @param batchSize Batch size for processing + * @param inputType Input type for Cohere models (e.g., "search_document", "search_query") + * @param invocationOptions Runtime options for retry, timeout, and backoff + */ + public BedrockModel( + BedrockRuntimeClient client, + String modelId, + int dimension, + int batchSize, + String inputType, + ModelInvocationOptions invocationOptions) { + super(batchSize, invocationOptions); this.client = Objects.requireNonNull(client, "BedrockRuntimeClient cannot be null"); this.modelId = Objects.requireNonNull(modelId, "Model ID cannot be null"); this.dimension = dimension; @@ -191,6 +258,38 @@ protected List> vector(Object[] fields) throws IOException { return new ArrayList<>(); } + return invocationRuntime.invoke( + fields, + new ProviderAdapter>>() { + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + return vectorGeneration(inputs); + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return "BEDROCK"; + } + + @Override + public String getModel() { + return modelId; + } + + @Override + public Integer getDimension() { + return dimension; + } + }); + } + + private List> vectorGeneration(Object[] fields) throws IOException { if (fields.length == 1) { ObjectNode requestBody = createRequestForSingleInput(fields[0]); String responseBody = invokeModel(requestBody); @@ -274,7 +373,12 @@ private List> parseSingleResponse(String responseBody) throws IOExce return result; } catch (IOException e) { - throw new IOException("Failed to parse single response: " + responseBody, e); + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "BEDROCK", + modelId, + "Failed to parse Bedrock single response", + e); } } @@ -305,11 +409,16 @@ private List> parseBatchResponse(String responseBody) throws IOExcep } return result; } catch (IOException e) { - throw new IOException("Failed to parse batch response: " + responseBody, e); + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "BEDROCK", + modelId, + "Failed to parse Bedrock batch response", + e); } } - private String invokeModel(ObjectNode requestBody) { + private String invokeModel(ObjectNode requestBody) throws IOException { String requestString = requestBody.toString(); InvokeModelRequest request = InvokeModelRequest.builder() @@ -317,8 +426,93 @@ private String invokeModel(ObjectNode requestBody) { .body(SdkBytes.fromString(requestString, StandardCharsets.UTF_8)) .build(); - InvokeModelResponse response = client.invokeModel(request); - return response.body().asString(StandardCharsets.UTF_8); + try { + InvokeModelResponse response = client.invokeModel(request); + return response.body().asString(StandardCharsets.UTF_8); + } catch (RuntimeException e) { + throw mapAwsException(e); + } + } + + private ModelInvocationException mapAwsException(RuntimeException exception) { + if (exception instanceof ThrottlingException + || exception instanceof ServiceQuotaExceededException) { + return ModelInvocationException.retryable( + ModelInvocationErrorType.RATE_LIMIT, + "BEDROCK", + modelId, + exception.getMessage(), + statusCode(exception), + exception); + } + if (exception instanceof ModelTimeoutException || isTimeoutCause(exception)) { + return ModelInvocationException.retryable( + ModelInvocationErrorType.TIMEOUT, + "BEDROCK", + modelId, + exception.getMessage(), + statusCode(exception), + exception); + } + if (exception instanceof InternalServerException + || exception instanceof ServiceUnavailableException + || exception instanceof ModelNotReadyException + || exception instanceof ModelErrorException + || exception instanceof SdkClientException) { + return ModelInvocationException.retryable( + ModelInvocationErrorType.TEMPORARY_REMOTE_ERROR, + "BEDROCK", + modelId, + exception.getMessage(), + statusCode(exception), + exception); + } + if (exception instanceof AccessDeniedException) { + return ModelInvocationException.nonRetryable( + ModelInvocationErrorType.AUTHENTICATION_ERROR, + "BEDROCK", + modelId, + exception.getMessage(), + statusCode(exception), + exception); + } + if (exception instanceof ValidationException + || exception instanceof ResourceNotFoundException + || exception instanceof ConflictException) { + return ModelInvocationException.nonRetryable( + ModelInvocationErrorType.CONFIGURATION_ERROR, + "BEDROCK", + modelId, + exception.getMessage(), + statusCode(exception), + exception); + } + return ModelInvocationException.nonRetryable( + ModelInvocationErrorType.UNKNOWN_REMOTE_ERROR, + "BEDROCK", + modelId, + exception.getMessage(), + statusCode(exception), + exception); + } + + private Integer statusCode(RuntimeException exception) { + if (exception instanceof BedrockRuntimeException) { + return ((BedrockRuntimeException) exception).statusCode(); + } + return null; + } + + private boolean isTimeoutCause(RuntimeException exception) { + Throwable cause = exception.getCause(); + while (cause != null) { + if (cause instanceof SocketTimeoutException + || cause instanceof ConnectTimeoutException) { + return true; + } + cause = cause.getCause(); + } + return false; } @Override diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/custom/CustomModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/custom/CustomModel.java index 8f9970e9b44b..fe80b1acc8e4 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/custom/CustomModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/custom/CustomModel.java @@ -24,8 +24,14 @@ import org.apache.seatunnel.shade.com.google.common.annotations.VisibleForTesting; import org.apache.seatunnel.transform.nlpmodel.CustomConfigPlaceholder; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationContext; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationErrorType; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationException; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; +import org.apache.seatunnel.transform.nlpmodel.ProviderAdapter; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.AbstractModel; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; @@ -57,7 +63,34 @@ public CustomModel( Map body, String parse, Integer vectorizedNumber) { - this(model, apiPath, header, body, parse, vectorizedNumber, HttpClients.createDefault()); + this( + model, + apiPath, + header, + body, + parse, + vectorizedNumber, + ModelInvocationOptions.defaults(), + HttpClients.createDefault()); + } + + public CustomModel( + String model, + String apiPath, + Map header, + Map body, + String parse, + Integer vectorizedNumber, + ModelInvocationOptions invocationOptions) { + this( + model, + apiPath, + header, + body, + parse, + vectorizedNumber, + invocationOptions, + HttpClients.createDefault()); } public CustomModel( @@ -68,7 +101,27 @@ public CustomModel( String parse, Integer vectorizedNumber, CloseableHttpClient client) { - super(vectorizedNumber); + this( + model, + apiPath, + header, + body, + parse, + vectorizedNumber, + ModelInvocationOptions.defaults(), + client); + } + + public CustomModel( + String model, + String apiPath, + Map header, + Map body, + String parse, + Integer vectorizedNumber, + ModelInvocationOptions invocationOptions, + CloseableHttpClient client) { + super(vectorizedNumber, invocationOptions); this.apiPath = apiPath; this.model = model; this.header = header; @@ -79,35 +132,84 @@ public CustomModel( @Override protected List> vector(Object[] fields) throws IOException { - return vectorGeneration(fields); + return invocationRuntime.invoke(fields, vectorAdapter(true)); } @Override public Integer dimension() throws IOException { - return vectorGeneration(new Object[] {DIMENSION_EXAMPLE}).get(0).size(); + return invocationRuntime + .invoke(new Object[] {DIMENSION_EXAMPLE}, vectorAdapter(false)) + .get(0) + .size(); + } + + private ProviderAdapter>> vectorAdapter(boolean validateOutputCount) { + return new ProviderAdapter>>() { + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + return vectorGeneration(inputs, context.getRequestTimeoutMs()); + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return "CUSTOM"; + } + + @Override + public String getModel() { + return model; + } + + @Override + public boolean validateOutputCount() { + return validateOutputCount; + } + }; } - private List> vectorGeneration(Object[] fields) throws IOException { + private List> vectorGeneration(Object[] fields, int requestTimeoutMs) + throws IOException { HttpPost post = new HttpPost(apiPath); // Construct a request with custom parameters for (Map.Entry entry : header.entrySet()) { post.setHeader(entry.getKey(), entry.getValue()); } + post.setConfig( + RequestConfig.custom() + .setConnectTimeout(requestTimeoutMs) + .setSocketTimeout(requestTimeoutMs) + .build()); post.setEntity( new StringEntity( OBJECT_MAPPER.writeValueAsString(createJsonNodeFromData(fields)), "UTF-8")); - CloseableHttpResponse response = client.execute(post); + try (CloseableHttpResponse response = client.execute(post)) { + String responseStr = EntityUtils.toString(response.getEntity()); - String responseStr = EntityUtils.toString(response.getEntity()); + if (response.getStatusLine().getStatusCode() != 200) { + throw ModelInvocationException.fromHttpStatus( + "CUSTOM", model, response.getStatusLine().getStatusCode(), responseStr); + } - if (response.getStatusLine().getStatusCode() != 200) { - throw new IOException("Failed to get vector from custom, response: " + responseStr); + try { + return OBJECT_MAPPER.convertValue( + parseResponse(responseStr), new TypeReference>>() {}); + } catch (RuntimeException e) { + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "CUSTOM", + model, + "Failed to parse Custom embedding response", + e); + } } - - return OBJECT_MAPPER.convertValue( - parseResponse(responseStr), new TypeReference>>() {}); } @VisibleForTesting diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java index 46250ac829c7..e02bbe74df88 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java @@ -23,6 +23,11 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.seatunnel.shade.com.google.common.annotations.VisibleForTesting; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationContext; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationErrorType; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationException; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; +import org.apache.seatunnel.transform.nlpmodel.ProviderAdapter; import org.apache.seatunnel.transform.nlpmodel.embedding.FieldSpec; import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.ModalityType; import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.MultimodalFieldValue; @@ -53,7 +58,14 @@ public class DoubaoModel extends MultimodalModel { private final String BASE64_PARAM_TEMPLATE = "data:%s/%s;base64,%s"; public DoubaoModel(String apiKey, String model, String apiPath, Integer vectorizedNumber) { - this(apiKey, model, apiPath, vectorizedNumber, false, HttpClients.createDefault()); + this( + apiKey, + model, + apiPath, + vectorizedNumber, + false, + ModelInvocationOptions.defaults(), + HttpClients.createDefault()); } public DoubaoModel( @@ -68,6 +80,7 @@ public DoubaoModel( apiPath, vectorizedNumber, isMultimodalFields, + ModelInvocationOptions.defaults(), HttpClients.createDefault()); } @@ -77,8 +90,43 @@ public DoubaoModel( String apiPath, Integer vectorizedNumber, boolean isMultimodalFields, + ModelInvocationOptions invocationOptions) { + this( + apiKey, + model, + apiPath, + vectorizedNumber, + isMultimodalFields, + invocationOptions, + HttpClients.createDefault()); + } + + public DoubaoModel( + String apiKey, + String model, + String apiPath, + Integer vectorizedNumber, + boolean isMultimodalFields, + CloseableHttpClient client) { + this( + apiKey, + model, + apiPath, + vectorizedNumber, + isMultimodalFields, + ModelInvocationOptions.defaults(), + client); + } + + public DoubaoModel( + String apiKey, + String model, + String apiPath, + Integer vectorizedNumber, + boolean isMultimodalFields, + ModelInvocationOptions invocationOptions, CloseableHttpClient client) { - super(vectorizedNumber); + super(vectorizedNumber, invocationOptions); this.apiKey = apiKey; this.model = model; this.apiPath = apiPath; @@ -88,7 +136,7 @@ public DoubaoModel( @Override protected List> textVector(Object[] fields) throws IOException { - return textVectorGeneration(fields); + return invocationRuntime.invoke(fields, textVectorAdapter(true)); } @Override @@ -99,7 +147,37 @@ public List> multimodalVector(Object[] fields) throws IOException { } List> vectors = new ArrayList<>(); for (Object field : fields) { - vectors.add(multimodalVectorGeneration((MultimodalFieldValue) field)); + vectors.addAll( + invocationRuntime.invoke( + new Object[] {field}, + new ProviderAdapter>>() { + @Override + public List> invoke( + Object[] inputs, ModelInvocationContext context) + throws IOException { + List> result = new ArrayList<>(); + result.add( + multimodalVectorGeneration( + (MultimodalFieldValue) inputs[0], + context.getRequestTimeoutMs())); + return result; + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return "DOUBAO"; + } + + @Override + public String getModel() { + return model; + } + })); } return vectors; } @@ -107,44 +185,96 @@ public List> multimodalVector(Object[] fields) throws IOException { @Override public Integer dimension() throws IOException { return isMultimodalFields - ? multimodalVectorGeneration( - new MultimodalFieldValue( - new FieldSpec(DIMENSION_EXAMPLE), DIMENSION_EXAMPLE)) + ? multimodalVector( + new Object[] { + new MultimodalFieldValue( + new FieldSpec(DIMENSION_EXAMPLE), DIMENSION_EXAMPLE) + }) + .get(0) .size() - : textVectorGeneration(new Object[] {DIMENSION_EXAMPLE}).get(0).size(); + : invocationRuntime + .invoke(new Object[] {DIMENSION_EXAMPLE}, textVectorAdapter(false)) + .get(0) + .size(); } - private List> textVectorGeneration(Object[] fields) throws IOException { + private ProviderAdapter>> textVectorAdapter(boolean validateOutputCount) { + return new ProviderAdapter>>() { + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + return textVectorGeneration(inputs, context.getRequestTimeoutMs()); + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return "DOUBAO"; + } + + @Override + public String getModel() { + return model; + } + + @Override + public boolean validateOutputCount() { + return validateOutputCount; + } + }; + } + + private List> textVectorGeneration(Object[] fields, int requestTimeoutMs) + throws IOException { HttpPost post = new HttpPost(apiPath); post.setHeader("Authorization", "Bearer " + apiKey); post.setHeader("Content-Type", "application/json"); post.setConfig( - RequestConfig.custom().setConnectTimeout(20000).setSocketTimeout(20000).build()); + RequestConfig.custom() + .setConnectTimeout(requestTimeoutMs) + .setSocketTimeout(requestTimeoutMs) + .build()); post.setEntity( new StringEntity( OBJECT_MAPPER.writeValueAsString(createJsonNodeFromData(fields)), "UTF-8")); - CloseableHttpResponse response = client.execute(post); - String responseStr = EntityUtils.toString(response.getEntity()); + try (CloseableHttpResponse response = client.execute(post)) { + String responseStr = EntityUtils.toString(response.getEntity()); - if (response.getStatusLine().getStatusCode() != 200) { - throw new IOException("Failed to get vector from doubao, response: " + responseStr); - } - - JsonNode data = OBJECT_MAPPER.readTree(responseStr).get("data"); - List> embeddings = new ArrayList<>(); + if (response.getStatusLine().getStatusCode() != 200) { + throw ModelInvocationException.fromHttpStatus( + "DOUBAO", model, response.getStatusLine().getStatusCode(), responseStr); + } - if (data.isArray()) { - for (JsonNode node : data) { - JsonNode embeddingNode = node.get("embedding"); - List embedding = - OBJECT_MAPPER.readValue( - embeddingNode.traverse(), new TypeReference>() {}); - embeddings.add(embedding); + try { + JsonNode data = OBJECT_MAPPER.readTree(responseStr).get("data"); + List> embeddings = new ArrayList<>(); + + if (data.isArray()) { + for (JsonNode node : data) { + JsonNode embeddingNode = node.get("embedding"); + List embedding = + OBJECT_MAPPER.readValue( + embeddingNode.traverse(), + new TypeReference>() {}); + embeddings.add(embedding); + } + } + return embeddings; + } catch (IOException | RuntimeException e) { + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "DOUBAO", + model, + "Failed to parse Doubao embedding response", + e); } } - return embeddings; } @VisibleForTesting @@ -153,12 +283,17 @@ public ObjectNode createJsonNodeFromData(Object[] fields) { return OBJECT_MAPPER.createObjectNode().put("model", model).set("input", arrayNode); } - protected List multimodalVectorGeneration(MultimodalFieldValue field) - throws IOException { + protected List multimodalVectorGeneration( + MultimodalFieldValue field, int requestTimeoutMs) throws IOException { HttpPost httpPost = new HttpPost(apiPath); httpPost.setHeader("Authorization", "Bearer " + apiKey); httpPost.setHeader("Content-Type", "application/json"); + httpPost.setConfig( + RequestConfig.custom() + .setConnectTimeout(requestTimeoutMs) + .setSocketTimeout(requestTimeoutMs) + .build()); StringEntity entity = new StringEntity( @@ -171,14 +306,20 @@ protected List multimodalVectorGeneration(MultimodalFieldValue field) EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); if (response.getStatusLine().getStatusCode() != 200) { - throw new IOException( - "HTTP error " - + response.getStatusLine().getStatusCode() - + ": " - + responseBody); + throw ModelInvocationException.fromHttpStatus( + "DOUBAO", model, response.getStatusLine().getStatusCode(), responseBody); } - return parseMultimodalVectorResponse(responseBody); + try { + return parseMultimodalVectorResponse(responseBody); + } catch (IOException | RuntimeException e) { + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "DOUBAO", + model, + "Failed to parse Doubao multimodal embedding response", + e); + } } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/openai/OpenAIModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/openai/OpenAIModel.java index 932b77df92d0..2fb593c2eff1 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/openai/OpenAIModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/openai/OpenAIModel.java @@ -23,6 +23,11 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.seatunnel.shade.com.google.common.annotations.VisibleForTesting; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationContext; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationErrorType; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationException; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; +import org.apache.seatunnel.transform.nlpmodel.ProviderAdapter; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.AbstractModel; import org.apache.http.client.config.RequestConfig; @@ -45,7 +50,13 @@ public class OpenAIModel extends AbstractModel { private final String apiPath; public OpenAIModel(String apiKey, String model, String apiPath, Integer vectorizedNumber) { - this(apiKey, model, apiPath, vectorizedNumber, HttpClients.createDefault()); + this( + apiKey, + model, + apiPath, + vectorizedNumber, + ModelInvocationOptions.defaults(), + HttpClients.createDefault()); } public OpenAIModel( @@ -53,8 +64,33 @@ public OpenAIModel( String model, String apiPath, Integer vectorizedNumber, + ModelInvocationOptions invocationOptions) { + this( + apiKey, + model, + apiPath, + vectorizedNumber, + invocationOptions, + HttpClients.createDefault()); + } + + public OpenAIModel( + String apiKey, + String model, + String apiPath, + Integer vectorizedNumber, + CloseableHttpClient client) { + this(apiKey, model, apiPath, vectorizedNumber, ModelInvocationOptions.defaults(), client); + } + + public OpenAIModel( + String apiKey, + String model, + String apiPath, + Integer vectorizedNumber, + ModelInvocationOptions invocationOptions, CloseableHttpClient client) { - super(vectorizedNumber); + super(vectorizedNumber, invocationOptions); this.apiKey = apiKey; this.model = model; this.apiPath = apiPath; @@ -66,45 +102,84 @@ protected List> vector(Object[] fields) throws IOException { if (fields.length > 1) { throw new IllegalArgumentException("OpenAI model only supports single input"); } - return vectorGeneration(fields); + return invocationRuntime.invoke( + fields, + new ProviderAdapter>>() { + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + return vectorGeneration(inputs, context.getRequestTimeoutMs()); + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return "OPENAI"; + } + + @Override + public String getModel() { + return model; + } + }); } @Override public Integer dimension() throws IOException { - return vectorGeneration(new Object[] {DIMENSION_EXAMPLE}).get(0).size(); + return vector(new Object[] {DIMENSION_EXAMPLE}).get(0).size(); } - private List> vectorGeneration(Object[] fields) throws IOException { + private List> vectorGeneration(Object[] fields, int requestTimeoutMs) + throws IOException { HttpPost post = new HttpPost(apiPath); post.setHeader("Authorization", "Bearer " + apiKey); post.setHeader("Content-Type", "application/json"); post.setConfig( - RequestConfig.custom().setConnectTimeout(20000).setSocketTimeout(20000).build()); + RequestConfig.custom() + .setConnectTimeout(requestTimeoutMs) + .setSocketTimeout(requestTimeoutMs) + .build()); post.setEntity( new StringEntity( OBJECT_MAPPER.writeValueAsString(createJsonNodeFromData(fields)), "UTF-8")); - CloseableHttpResponse response = client.execute(post); - String responseStr = EntityUtils.toString(response.getEntity()); - - if (response.getStatusLine().getStatusCode() != 200) { - throw new IOException("Failed to get vector from openai, response: " + responseStr); - } + try (CloseableHttpResponse response = client.execute(post)) { + String responseStr = EntityUtils.toString(response.getEntity()); - JsonNode data = OBJECT_MAPPER.readTree(responseStr).get("data"); - List> embeddings = new ArrayList<>(); + if (response.getStatusLine().getStatusCode() != 200) { + throw ModelInvocationException.fromHttpStatus( + "OPENAI", model, response.getStatusLine().getStatusCode(), responseStr); + } - if (data.isArray()) { - for (JsonNode node : data) { - JsonNode embeddingNode = node.get("embedding"); - List embedding = - OBJECT_MAPPER.readValue( - embeddingNode.traverse(), new TypeReference>() {}); - embeddings.add(embedding); + try { + JsonNode data = OBJECT_MAPPER.readTree(responseStr).get("data"); + List> embeddings = new ArrayList<>(); + + if (data.isArray()) { + for (JsonNode node : data) { + JsonNode embeddingNode = node.get("embedding"); + List embedding = + OBJECT_MAPPER.readValue( + embeddingNode.traverse(), + new TypeReference>() {}); + embeddings.add(embedding); + } + } + return embeddings; + } catch (IOException | RuntimeException e) { + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "OPENAI", + model, + "Failed to parse OpenAI embedding response", + e); } } - return embeddings; } @VisibleForTesting diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/qianfan/QianfanModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/qianfan/QianfanModel.java index f85619eb3e64..ad3bf1ff3af2 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/qianfan/QianfanModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/qianfan/QianfanModel.java @@ -23,6 +23,11 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.seatunnel.shade.com.google.common.annotations.VisibleForTesting; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationContext; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationErrorType; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationException; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; +import org.apache.seatunnel.transform.nlpmodel.ProviderAdapter; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.AbstractModel; import org.apache.http.client.config.RequestConfig; @@ -59,14 +64,35 @@ public QianfanModel( String oauthPath, Integer vectorizedNumber) throws IOException { - super(vectorizedNumber); + this( + apiKey, + secretKey, + model, + apiPath, + oauthPath, + vectorizedNumber, + ModelInvocationOptions.defaults()); + } + + public QianfanModel( + String apiKey, + String secretKey, + String model, + String apiPath, + String oauthPath, + Integer vectorizedNumber, + ModelInvocationOptions invocationOptions) + throws IOException { + super(vectorizedNumber, invocationOptions); + ModelInvocationOptions resolvedOptions = + invocationOptions == null ? ModelInvocationOptions.defaults() : invocationOptions; this.apiKey = apiKey; this.secretKey = secretKey; this.model = model; this.apiPath = apiPath; this.oauthPath = oauthPath; this.client = HttpClients.createDefault(); - this.accessToken = getAccessToken(); + this.accessToken = getAccessToken(resolvedOptions.getRequestTimeoutMs()); } public QianfanModel( @@ -78,7 +104,28 @@ public QianfanModel( String oauthPath, String accessToken) throws IOException { - super(vectorizedNumber); + this( + apiKey, + secretKey, + model, + apiPath, + vectorizedNumber, + oauthPath, + accessToken, + ModelInvocationOptions.defaults()); + } + + public QianfanModel( + String apiKey, + String secretKey, + String model, + String apiPath, + Integer vectorizedNumber, + String oauthPath, + String accessToken, + ModelInvocationOptions invocationOptions) + throws IOException { + super(vectorizedNumber, invocationOptions); this.apiKey = apiKey; this.secretKey = secretKey; this.model = model; @@ -88,28 +135,78 @@ public QianfanModel( this.accessToken = accessToken; } - private String getAccessToken() throws IOException { + private String getAccessToken(int requestTimeoutMs) throws IOException { HttpGet get = new HttpGet(String.format(oauthPath + oauthSuffixPath, apiKey, secretKey)); - CloseableHttpResponse response = client.execute(get); - String responseStr = EntityUtils.toString(response.getEntity()); - if (response.getStatusLine().getStatusCode() != 200) { - throw new IOException("Failed to Oauth for qianfan, response: " + responseStr); + get.setConfig( + RequestConfig.custom() + .setConnectTimeout(requestTimeoutMs) + .setSocketTimeout(requestTimeoutMs) + .build()); + try (CloseableHttpResponse response = client.execute(get)) { + String responseStr = EntityUtils.toString(response.getEntity()); + if (response.getStatusLine().getStatusCode() != 200) { + throw ModelInvocationException.fromHttpStatus( + "QIANFAN", model, response.getStatusLine().getStatusCode(), responseStr); + } + try { + JsonNode result = OBJECT_MAPPER.readTree(responseStr); + return result.get("access_token").asText(); + } catch (IOException | RuntimeException e) { + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "QIANFAN", + model, + "Failed to parse Qianfan OAuth response", + e); + } } - JsonNode result = OBJECT_MAPPER.readTree(responseStr); - return result.get("access_token").asText(); } @Override public List> vector(Object[] fields) throws IOException { - return vectorGeneration(fields); + return invocationRuntime.invoke(fields, vectorAdapter(true)); } @Override public Integer dimension() throws IOException { - return vectorGeneration(new Object[] {DIMENSION_EXAMPLE}).get(0).size(); + return invocationRuntime + .invoke(new Object[] {DIMENSION_EXAMPLE}, vectorAdapter(false)) + .get(0) + .size(); + } + + private ProviderAdapter>> vectorAdapter(boolean validateOutputCount) { + return new ProviderAdapter>>() { + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + return vectorGeneration(inputs, context.getRequestTimeoutMs()); + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return "QIANFAN"; + } + + @Override + public String getModel() { + return model; + } + + @Override + public boolean validateOutputCount() { + return validateOutputCount; + } + }; } - private List> vectorGeneration(Object[] fields) throws IOException { + private List> vectorGeneration(Object[] fields, int requestTimeoutMs) + throws IOException { String formattedApiPath = String.format( (apiPath.endsWith("/") ? apiPath : apiPath + "/") + "%s?access_token=%s", @@ -118,43 +215,70 @@ private List> vectorGeneration(Object[] fields) throws IOException { HttpPost post = new HttpPost(formattedApiPath); post.setHeader("Content-Type", "application/json"); post.setConfig( - RequestConfig.custom().setConnectTimeout(20000).setSocketTimeout(20000).build()); + RequestConfig.custom() + .setConnectTimeout(requestTimeoutMs) + .setSocketTimeout(requestTimeoutMs) + .build()); post.setEntity( new StringEntity( OBJECT_MAPPER.writeValueAsString(createJsonNodeFromData(fields)), "UTF-8")); - CloseableHttpResponse response = client.execute(post); - String responseStr = EntityUtils.toString(response.getEntity()); + try (CloseableHttpResponse response = client.execute(post)) { + String responseStr = EntityUtils.toString(response.getEntity()); - if (response.getStatusLine().getStatusCode() != 200) { - throw new IOException("Failed to get vector from qianfan, response: " + responseStr); - } + if (response.getStatusLine().getStatusCode() != 200) { + throw ModelInvocationException.fromHttpStatus( + "QIANFAN", model, response.getStatusLine().getStatusCode(), responseStr); + } - JsonNode result = OBJECT_MAPPER.readTree(responseStr); - JsonNode errorCode = result.get("error_code"); + try { + JsonNode result = OBJECT_MAPPER.readTree(responseStr); + JsonNode errorCode = result.get("error_code"); - if (errorCode != null) { - // Handle access token expiration - if (errorCode.asInt() == 110) { - this.accessToken = getAccessToken(); - } - throw new IOException( - "Failed to get vector from qianfan, response: " + result.get("error_msg")); - } + if (errorCode != null) { + // Handle access token expiration and let the common runtime retry the request. + if (errorCode.asInt() == 110) { + this.accessToken = getAccessToken(requestTimeoutMs); + throw ModelInvocationException.retryable( + ModelInvocationErrorType.AUTHENTICATION_ERROR, + "QIANFAN", + model, + "Qianfan access token expired and was refreshed", + null, + null); + } + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.CONFIGURATION_ERROR, + "QIANFAN", + model, + "Qianfan returned error_code " + errorCode.asInt(), + null); + } - List> embeddings = new ArrayList<>(); - JsonNode data = result.get("data"); - if (data.isArray()) { - for (JsonNode node : data) { - List embedding = - OBJECT_MAPPER.readValue( - node.get("embedding").traverse(), - new TypeReference>() {}); - embeddings.add(embedding); + List> embeddings = new ArrayList<>(); + JsonNode data = result.get("data"); + if (data.isArray()) { + for (JsonNode node : data) { + List embedding = + OBJECT_MAPPER.readValue( + node.get("embedding").traverse(), + new TypeReference>() {}); + embeddings.add(embedding); + } + } + return embeddings; + } catch (ModelInvocationException e) { + throw e; + } catch (IOException | RuntimeException e) { + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "QIANFAN", + model, + "Failed to parse Qianfan embedding response", + e); } } - return embeddings; } @VisibleForTesting diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/zhipu/ZhipuModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/zhipu/ZhipuModel.java index a36535821ee2..7df2a4a5b013 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/zhipu/ZhipuModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/zhipu/ZhipuModel.java @@ -23,6 +23,11 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.seatunnel.shade.com.google.common.annotations.VisibleForTesting; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationContext; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationErrorType; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationException; +import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; +import org.apache.seatunnel.transform.nlpmodel.ProviderAdapter; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.AbstractModel; import org.apache.http.HttpHeaders; @@ -57,7 +62,24 @@ public ZhipuModel( Integer dimension, Integer vectorizedNumber) throws IOException { - super(vectorizedNumber); + this( + apiKey, + model, + apiPath, + dimension, + vectorizedNumber, + ModelInvocationOptions.defaults()); + } + + public ZhipuModel( + String apiKey, + String model, + String apiPath, + Integer dimension, + Integer vectorizedNumber, + ModelInvocationOptions invocationOptions) + throws IOException { + super(vectorizedNumber, invocationOptions); this.model = model; this.apiKey = apiKey; this.apiPath = apiPath; @@ -67,7 +89,35 @@ public ZhipuModel( @Override public List> vector(Object[] fields) throws IOException { - return vectorGeneration(fields); + return invocationRuntime.invoke( + fields, + new ProviderAdapter>>() { + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + return vectorGeneration(inputs, context.getRequestTimeoutMs()); + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return "ZHIPU"; + } + + @Override + public String getModel() { + return model; + } + + @Override + public Integer getDimension() { + return dimension; + } + }); } @Override @@ -75,41 +125,61 @@ public Integer dimension() throws IOException { return dimension; } - private List> vectorGeneration(Object[] fields) throws IOException { + private List> vectorGeneration(Object[] fields, int requestTimeoutMs) + throws IOException { if (fields == null || fields.length > MAX_INPUT_SIZE) { - throw new IOException( - "Zhipu input text for vectorization, with a maximum limit of 64 entries."); + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.CONFIGURATION_ERROR, + "ZHIPU", + model, + "Zhipu input text for vectorization has a maximum limit of 64 entries.", + null); } HttpPost post = new HttpPost(apiPath); post.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiKey); post.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); post.setConfig( - RequestConfig.custom().setConnectTimeout(20000).setSocketTimeout(20000).build()); + RequestConfig.custom() + .setConnectTimeout(requestTimeoutMs) + .setSocketTimeout(requestTimeoutMs) + .build()); post.setEntity( new StringEntity( OBJECT_MAPPER.writeValueAsString(createJsonNodeFromData(fields)), StandardCharsets.UTF_8.name())); - CloseableHttpResponse response = client.execute(post); - String responseStr = EntityUtils.toString(response.getEntity()); - if (response.getStatusLine().getStatusCode() != 200) { - throw new IOException("Failed to get vector from zhipu, response: " + responseStr); - } - JsonNode data = OBJECT_MAPPER.readTree(responseStr).get("data"); - List> embeddings = new ArrayList<>(); - - if (data.isArray()) { - for (JsonNode node : data) { - JsonNode embeddingNode = node.get("embedding"); - List embedding = - OBJECT_MAPPER.readValue( - embeddingNode.traverse(), new TypeReference>() {}); - embeddings.add(embedding); + try (CloseableHttpResponse response = client.execute(post)) { + String responseStr = EntityUtils.toString(response.getEntity()); + if (response.getStatusLine().getStatusCode() != 200) { + throw ModelInvocationException.fromHttpStatus( + "ZHIPU", model, response.getStatusLine().getStatusCode(), responseStr); + } + try { + JsonNode data = OBJECT_MAPPER.readTree(responseStr).get("data"); + List> embeddings = new ArrayList<>(); + + if (data.isArray()) { + for (JsonNode node : data) { + JsonNode embeddingNode = node.get("embedding"); + List embedding = + OBJECT_MAPPER.readValue( + embeddingNode.traverse(), + new TypeReference>() {}); + embeddings.add(embedding); + } + } + return embeddings; + } catch (IOException | RuntimeException e) { + throw ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "ZHIPU", + model, + "Failed to parse Zhipu embedding response", + e); } } - return embeddings; } @VisibleForTesting diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/EmbeddingModelDimensionTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/EmbeddingModelDimensionTest.java index b62aa9a06047..3c58a2ea2666 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/EmbeddingModelDimensionTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/EmbeddingModelDimensionTest.java @@ -20,10 +20,17 @@ import org.apache.seatunnel.transform.nlpmodel.embedding.remote.custom.CustomModel; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.doubao.DoubaoModel; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.openai.OpenAIModel; +import org.apache.seatunnel.transform.nlpmodel.embedding.remote.qianfan.QianfanModel; +import org.apache.http.HttpEntity; import org.apache.http.ProtocolVersion; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicStatusLine; import org.apache.http.util.EntityUtils; @@ -33,6 +40,7 @@ import org.mockito.Mockito; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -109,6 +117,76 @@ void testDoubleModelDimension() throws IOException { } } + @Test + void testDoubaoModelDimensionIgnoresBatchResponseCount() throws IOException { + CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); + DoubaoModel model = + new DoubaoModel( + "apikey", + "modelName", + "https://api.doubao.io/v1/chat/completions", + 2, + false, + client); + + CloseableHttpResponse response = okResponse(embeddingResponse(2, 3)); + Mockito.when(client.execute(Mockito.any())).thenReturn(response); + + Assertions.assertEquals(3, model.dimension()); + } + + @Test + void testCustomModelDimensionIgnoresBatchResponseCount() throws IOException { + CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); + CustomModel model = + new CustomModel( + "modelName", + "https://api.custom.com/v1/chat/completions", + new HashMap<>(), + new HashMap<>(), + "$.data[*].embedding", + 2, + client); + + CloseableHttpResponse response = okResponse(embeddingResponse(2, 3)); + Mockito.when(client.execute(Mockito.any())).thenReturn(response); + + Assertions.assertEquals(3, model.dimension()); + } + + @Test + void testQianfanModelDimensionIgnoresBatchResponseCount() throws IOException { + CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); + try (MockedStatic httpClients = Mockito.mockStatic(HttpClients.class)) { + CloseableHttpResponse tokenResponse = okResponse("{\"access_token\":\"token\"}"); + CloseableHttpResponse embeddingResponse = okResponse(embeddingResponse(2, 3)); + httpClients.when(HttpClients::createDefault).thenReturn(client); + Mockito.when( + client.execute( + Mockito.argThat( + (HttpUriRequest request) -> + request instanceof HttpGet))) + .thenReturn(tokenResponse); + Mockito.when( + client.execute( + Mockito.argThat( + (HttpUriRequest request) -> + request instanceof HttpPost))) + .thenReturn(embeddingResponse); + + QianfanModel model = + new QianfanModel( + "apikey", + "secretKey", + "modelName", + "https://api.qianfan.io/v1/embedding", + "https://api.qianfan.io/oauth", + 2); + + Assertions.assertEquals(3, model.dimension()); + } + } + @Test void testOpenAIModelDimension() throws IOException { CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); @@ -142,6 +220,39 @@ void testOpenAIModelDimension() throws IOException { } } + private static CloseableHttpResponse okResponse(String responseBody) { + CloseableHttpResponse response = Mockito.mock(CloseableHttpResponse.class); + HttpEntity entity = new StringEntity(responseBody, StandardCharsets.UTF_8); + Mockito.when(response.getStatusLine()) + .thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 1, 1), 200, "OK")); + Mockito.when(response.getEntity()).thenReturn(entity); + return response; + } + + private static String embeddingResponse(int count, int dimension) { + StringBuilder data = new StringBuilder(); + for (int i = 0; i < count; i++) { + if (i > 0) { + data.append(","); + } + data.append("{\"embedding\":").append(generateVectorJson(dimension)); + data.append(",\"index\":").append(i).append(",\"object\":\"embedding\"}"); + } + return "{\"data\":[" + data + "],\"model\":\"modelName\",\"object\":\"list\"}"; + } + + private static String generateVectorJson(int dimension) { + StringBuilder vector = new StringBuilder("["); + for (int i = 0; i < dimension; i++) { + if (i > 0) { + vector.append(","); + } + vector.append(i + 1).append(".0"); + } + vector.append("]"); + return vector.toString(); + } + private List generateVector(int dimension) { List vector = new ArrayList<>(); for (int i = 0; i < dimension; i++) { diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKeyTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKeyTest.java new file mode 100644 index 000000000000..0de16278e5ec --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKeyTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ModelInvocationCacheKeyTest { + + @Test + void keyIsStableForEquivalentInputAndMetadataOrdering() { + String first = + ModelInvocationCacheKey.builder() + .provider(" OPENAI ") + .model("text-embedding-3-small") + .dimension(1536) + .modality(" Text ") + .format("TEXT") + .input("first line\r\nsecond line") + .metadata("chunk_id", "chunk-1") + .metadata("content_hash", "hash-1") + .build(); + String second = + ModelInvocationCacheKey.builder() + .provider("OPENAI") + .model("text-embedding-3-small") + .dimension(1536) + .modality("text") + .format("text") + .input("first line\nsecond line") + .metadata("content_hash", "hash-1") + .metadata("chunk_id", "chunk-1") + .build(); + + Assertions.assertEquals(first, second); + } + + @Test + void keySeparatesProviderModelDimensionModalityAndFormat() { + String base = + ModelInvocationCacheKey.builder() + .provider("OPENAI") + .model("text-embedding-3-small") + .dimension(1536) + .modality("text") + .format("text") + .input("same chunk") + .build(); + + Assertions.assertNotEquals( + base, keyWith("DOUBAO", "text-embedding-3-small", 1536, "text", "text")); + Assertions.assertNotEquals( + base, keyWith("OPENAI", "text-embedding-3-large", 1536, "text", "text")); + Assertions.assertNotEquals( + base, keyWith("OPENAI", "text-embedding-3-small", 1024, "text", "text")); + Assertions.assertNotEquals( + base, keyWith("OPENAI", "text-embedding-3-small", 1536, "jpeg", "text")); + Assertions.assertNotEquals( + base, keyWith("OPENAI", "text-embedding-3-small", 1536, "text", "url")); + } + + @Test + void keyDoesNotExposeRawLargeOrSensitiveInputContent() { + String sensitiveInput = + "api_key=secret-token\n" + + "customer text that should only participate through a digest "; + + String key = + ModelInvocationCacheKey.builder() + .provider("OPENAI") + .model("text-embedding-3-small") + .dimension(1536) + .modality("text") + .format("text") + .input(sensitiveInput) + .build(); + + Assertions.assertFalse(key.contains("api_key")); + Assertions.assertFalse(key.contains("secret-token")); + Assertions.assertFalse(key.contains("customer text")); + Assertions.assertTrue(key.contains("input_sha256=")); + } + + @Test + void binaryInputDigestUsesContentInsteadOfArrayIdentity() { + String first = + ModelInvocationCacheKey.builder() + .provider("BEDROCK") + .model("amazon.titan-embed-image-v1") + .dimension(1024) + .modality("jpeg") + .format("binary") + .input(new byte[] {1, 2, 3}) + .build(); + String second = + ModelInvocationCacheKey.builder() + .provider("BEDROCK") + .model("amazon.titan-embed-image-v1") + .dimension(1024) + .modality("jpeg") + .format("binary") + .input(new byte[] {1, 2, 3}) + .build(); + String different = + ModelInvocationCacheKey.builder() + .provider("BEDROCK") + .model("amazon.titan-embed-image-v1") + .dimension(1024) + .modality("jpeg") + .format("binary") + .input(new byte[] {1, 2, 4}) + .build(); + + Assertions.assertEquals(first, second); + Assertions.assertNotEquals(first, different); + } + + private static String keyWith( + String provider, String model, int dimension, String modality, String format) { + return ModelInvocationCacheKey.builder() + .provider(provider) + .model(model) + .dimension(dimension) + .modality(modality) + .format(format) + .input("same chunk") + .build(); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeCacheTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeCacheTest.java new file mode 100644 index 000000000000..ba2bfe9c9040 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeCacheTest.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class ModelInvocationRuntimeCacheTest { + + private static final String PROVIDER = "OPENAI"; + private static final String MODEL = "text-embedding-3-small"; + private static final int DIMENSION = 1536; + private static final String MODALITY = "text"; + private static final String FORMAT = "text"; + + @Test + void fullCacheHitSkipsRemoteInvocation() throws IOException { + RecordingModelInvocationCache cache = new RecordingModelInvocationCache(); + cache.put(cacheKey("alpha"), vector(1.0f, 2.0f)); + cache.put(cacheKey("beta"), vector(3.0f, 4.0f)); + + CountingVectorAdapter adapter = new CountingVectorAdapter(); + ModelInvocationRuntime runtime = + new ModelInvocationRuntime( + ModelInvocationOptions.defaults(), ModelInvocationMetrics.NOOP, cache); + + List> result = runtime.invoke(new Object[] {"alpha", "beta"}, adapter); + + Assertions.assertEquals(0, adapter.getInvocationCount()); + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(vector(1.0f, 2.0f), result.get(0)); + Assertions.assertEquals(vector(3.0f, 4.0f), result.get(1)); + } + + @Test + void partialCacheHitReassemblesCachedAndGeneratedVectorsInOrder() throws IOException { + RecordingModelInvocationCache cache = new RecordingModelInvocationCache(); + cache.put(cacheKey("alpha"), vector(1.0f, 2.0f)); + + CountingVectorAdapter adapter = + new CountingVectorAdapter(new Object[] {"beta"}, vectors(vector(3.0f, 4.0f))); + ModelInvocationRuntime runtime = + new ModelInvocationRuntime( + ModelInvocationOptions.defaults(), ModelInvocationMetrics.NOOP, cache); + + List> result = runtime.invoke(new Object[] {"alpha", "beta"}, adapter); + + Assertions.assertEquals(1, adapter.getInvocationCount()); + Assertions.assertEquals(1, adapter.getLastInvocationInputCount()); + Assertions.assertEquals("beta", adapter.getLastInvocationInputs()[0]); + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(vector(1.0f, 2.0f), result.get(0)); + Assertions.assertEquals(vector(3.0f, 4.0f), result.get(1)); + } + + @Test + void successfulInvocationStoresGeneratedVectorsInCache() throws IOException { + RecordingModelInvocationCache cache = new RecordingModelInvocationCache(); + CountingVectorAdapter adapter = + new CountingVectorAdapter( + new Object[] {"alpha", "beta"}, + vectors(vector(1.0f, 2.0f), vector(3.0f, 4.0f))); + ModelInvocationRuntime runtime = + new ModelInvocationRuntime( + ModelInvocationOptions.defaults(), ModelInvocationMetrics.NOOP, cache); + + List> result = runtime.invoke(new Object[] {"alpha", "beta"}, adapter); + + Assertions.assertEquals(1, adapter.getInvocationCount()); + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(vector(1.0f, 2.0f), result.get(0)); + Assertions.assertEquals(vector(3.0f, 4.0f), result.get(1)); + Assertions.assertEquals(2, cache.size()); + Assertions.assertEquals( + vector(1.0f, 2.0f), + cache.get(cacheKey("alpha")) + .orElseThrow(() -> new AssertionError("missing alpha"))); + Assertions.assertEquals( + vector(3.0f, 4.0f), + cache.get(cacheKey("beta")).orElseThrow(() -> new AssertionError("missing beta"))); + } + + private static String cacheKey(String input) { + return ModelInvocationCacheKey.builder() + .provider(PROVIDER) + .model(MODEL) + .dimension(DIMENSION) + .modality(MODALITY) + .format(FORMAT) + .input(input) + .build(); + } + + private static List vector(float... values) { + List result = new ArrayList<>(); + for (float value : values) { + result.add(value); + } + return result; + } + + @SafeVarargs + private static List> vectors(List... vectors) { + List> result = new ArrayList<>(); + for (List vector : vectors) { + result.add(vector); + } + return result; + } + + private static class RecordingModelInvocationCache implements ModelInvocationCache { + + private final Map values = new HashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public Optional get(String key) { + return Optional.ofNullable((T) values.get(key)); + } + + @Override + public void put(String key, T value) { + values.put(key, value); + } + + private int size() { + return values.size(); + } + } + + private static class CountingVectorAdapter implements ProviderAdapter>> { + + private final Object[] expectedInputs; + private final List> response; + private int invocationCount; + private Object[] lastInvocationInputs = new Object[0]; + + private CountingVectorAdapter() { + this.expectedInputs = null; + this.response = null; + } + + private CountingVectorAdapter(Object[] expectedInputs, List> response) { + this.expectedInputs = expectedInputs; + this.response = response; + } + + private int getInvocationCount() { + return invocationCount; + } + + private int getLastInvocationInputCount() { + return lastInvocationInputs.length; + } + + private Object[] getLastInvocationInputs() { + return lastInvocationInputs; + } + + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + invocationCount++; + lastInvocationInputs = inputs == null ? new Object[0] : inputs.clone(); + if (expectedInputs != null) { + Assertions.assertArrayEquals(expectedInputs, lastInvocationInputs); + } else { + throw new AssertionError( + "Remote invocation should not be called on a full cache hit"); + } + return response; + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return PROVIDER; + } + + @Override + public String getModel() { + return MODEL; + } + + @Override + public Integer getDimension() { + return DIMENSION; + } + + @Override + public String getInputModality(Object input) { + return MODALITY; + } + + @Override + public String getInputFormat(Object input) { + return FORMAT; + } + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeMetricsTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeMetricsTest.java new file mode 100644 index 000000000000..b78b7bd9a799 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeMetricsTest.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Queue; + +public class ModelInvocationRuntimeMetricsTest { + + private static final String PROVIDER = "OPENAI"; + private static final String MODEL = "text-embedding-3-small"; + + @Test + void successfulInvocationRecordsRequestGeneratedOutputsAndLatency() throws IOException { + RecordingMetrics metrics = new RecordingMetrics(); + MetricVectorAdapter adapter = new MetricVectorAdapter(vectors(vector(1.0f, 2.0f))); + ModelInvocationRuntime runtime = + new ModelInvocationRuntime( + ModelInvocationOptions.defaults(), metrics, ModelInvocationCache.NOOP); + + List> result = runtime.invoke(new Object[] {"chunk"}, adapter); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals( + Arrays.asList( + "request:OPENAI:text-embedding-3-small", + "generated:OPENAI:text-embedding-3-small:1", + "latency:OPENAI:text-embedding-3-small"), + metrics.eventPrefixes()); + } + + @Test + void retryableFailureThenSuccessRecordsFailureRetryAndLatency() throws IOException { + RecordingMetrics metrics = new RecordingMetrics(); + MetricVectorAdapter adapter = + new MetricVectorAdapter( + ModelInvocationException.fromHttpStatus( + PROVIDER, MODEL, 429, "rate limited")); + adapter.setSuccessResponse(vectors(vector(1.0f, 2.0f))); + ModelInvocationRuntime runtime = + new ModelInvocationRuntime(testRetryOptions(), metrics, ModelInvocationCache.NOOP); + + List> result = runtime.invoke(new Object[] {"chunk"}, adapter); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(2, metrics.countEvents("request:OPENAI:text-embedding-3-small")); + Assertions.assertEquals( + 1, metrics.countEvents("failure:OPENAI:text-embedding-3-small:RATE_LIMIT")); + Assertions.assertEquals( + 1, metrics.countEvents("retry:OPENAI:text-embedding-3-small:RATE_LIMIT")); + Assertions.assertEquals( + 1, metrics.countEvents("generated:OPENAI:text-embedding-3-small:1")); + Assertions.assertEquals(2, metrics.countEvents("latency:OPENAI:text-embedding-3-small")); + } + + @Test + void retryExhaustionRecordsRetryExhaustedMetric() { + RecordingMetrics metrics = new RecordingMetrics(); + MetricVectorAdapter adapter = + new MetricVectorAdapter( + ModelInvocationException.fromHttpStatus( + PROVIDER, MODEL, 429, "rate limited"), + ModelInvocationException.fromHttpStatus( + PROVIDER, MODEL, 429, "rate limited")); + ModelInvocationRuntime runtime = + new ModelInvocationRuntime(testRetryOptions(), metrics, ModelInvocationCache.NOOP); + + ModelInvocationException exception = + Assertions.assertThrows( + ModelInvocationException.class, + () -> runtime.invoke(new Object[] {"chunk"}, adapter)); + + Assertions.assertEquals(ModelInvocationErrorType.RATE_LIMIT, exception.getErrorType()); + Assertions.assertEquals(2, metrics.countEvents("request:OPENAI:text-embedding-3-small")); + Assertions.assertEquals( + 2, metrics.countEvents("failure:OPENAI:text-embedding-3-small:RATE_LIMIT")); + Assertions.assertEquals( + 1, metrics.countEvents("retry:OPENAI:text-embedding-3-small:RATE_LIMIT")); + Assertions.assertEquals( + 1, metrics.countEvents("retryExhausted:OPENAI:text-embedding-3-small:RATE_LIMIT")); + } + + @Test + void responseCountMismatchRecordsMismatchMetric() { + RecordingMetrics metrics = new RecordingMetrics(); + MetricVectorAdapter adapter = new MetricVectorAdapter(vectors(vector(1.0f, 2.0f))); + ModelInvocationRuntime runtime = + new ModelInvocationRuntime(testRetryOptions(), metrics, ModelInvocationCache.NOOP); + + ModelInvocationException exception = + Assertions.assertThrows( + ModelInvocationException.class, + () -> runtime.invoke(new Object[] {"chunk-1", "chunk-2"}, adapter)); + + Assertions.assertEquals( + ModelInvocationErrorType.RESPONSE_COUNT_MISMATCH, exception.getErrorType()); + Assertions.assertEquals( + 1, metrics.countEvents("responseCountMismatch:OPENAI:text-embedding-3-small")); + Assertions.assertEquals( + 1, + metrics.countEvents( + "failure:OPENAI:text-embedding-3-small:RESPONSE_COUNT_MISMATCH")); + } + + @Test + void cacheMetricsRecordHitAndMissCounts() throws IOException { + RecordingMetrics metrics = new RecordingMetrics(); + RecordingModelInvocationCache cache = new RecordingModelInvocationCache(); + cache.put(cacheKey("alpha"), vector(1.0f, 2.0f)); + MetricVectorAdapter adapter = new MetricVectorAdapter(vectors(vector(3.0f, 4.0f))); + ModelInvocationRuntime runtime = + new ModelInvocationRuntime(testRetryOptions(), metrics, cache); + + List> result = runtime.invoke(new Object[] {"alpha", "beta"}, adapter); + + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(1, metrics.countEvents("cacheHit:OPENAI:text-embedding-3-small:1")); + Assertions.assertEquals( + 1, metrics.countEvents("cacheMiss:OPENAI:text-embedding-3-small:1")); + } + + private static ModelInvocationOptions testRetryOptions() { + return new ModelInvocationOptions(2, 0L, 0L, 20000); + } + + private static String cacheKey(String input) { + return ModelInvocationCacheKey.builder() + .provider(PROVIDER) + .model(MODEL) + .dimension(1536) + .modality("text") + .format("text") + .input(input) + .build(); + } + + private static List vector(float... values) { + List result = new ArrayList<>(); + for (float value : values) { + result.add(value); + } + return result; + } + + @SafeVarargs + private static List> vectors(List... vectors) { + List> result = new ArrayList<>(); + for (List vector : vectors) { + result.add(vector); + } + return result; + } + + private static class RecordingMetrics implements ModelInvocationMetrics { + + private final List events = new ArrayList<>(); + + @Override + public void recordRequest(String provider, String model) { + events.add("request:" + provider + ":" + model); + } + + @Override + public void recordFailure( + String provider, String model, ModelInvocationErrorType errorType) { + events.add("failure:" + provider + ":" + model + ":" + errorType); + } + + @Override + public void recordRetry(String provider, String model, ModelInvocationErrorType errorType) { + events.add("retry:" + provider + ":" + model + ":" + errorType); + } + + @Override + public void recordRetryExhausted( + String provider, String model, ModelInvocationErrorType errorType) { + events.add("retryExhausted:" + provider + ":" + model + ":" + errorType); + } + + @Override + public void recordResponseCountMismatch(String provider, String model) { + events.add("responseCountMismatch:" + provider + ":" + model); + } + + @Override + public void recordGeneratedOutputs(String provider, String model, int outputCount) { + events.add("generated:" + provider + ":" + model + ":" + outputCount); + } + + @Override + public void recordLatency(String provider, String model, long elapsedMs) { + events.add("latency:" + provider + ":" + model); + } + + @Override + public void recordCacheHit(String provider, String model, int hitCount) { + events.add("cacheHit:" + provider + ":" + model + ":" + hitCount); + } + + @Override + public void recordCacheMiss(String provider, String model, int missCount) { + events.add("cacheMiss:" + provider + ":" + model + ":" + missCount); + } + + private int countEvents(String prefix) { + int count = 0; + for (String event : events) { + if (event.startsWith(prefix)) { + count++; + } + } + return count; + } + + private List eventPrefixes() { + return events; + } + } + + private static class RecordingModelInvocationCache implements ModelInvocationCache { + + private final java.util.Map values = new java.util.HashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public Optional get(String key) { + return Optional.ofNullable((T) values.get(key)); + } + + @Override + public void put(String key, T value) { + values.put(key, value); + } + } + + private static class MetricVectorAdapter implements ProviderAdapter>> { + + private final Queue failures = new LinkedList<>(); + private List> successResponse = vectors(vector(1.0f, 2.0f)); + + private MetricVectorAdapter(List> successResponse) { + this.successResponse = successResponse; + } + + private MetricVectorAdapter(IOException... failures) { + for (IOException failure : failures) { + this.failures.add(failure); + } + } + + private void setSuccessResponse(List> successResponse) { + this.successResponse = successResponse; + } + + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + if (!failures.isEmpty()) { + throw failures.remove(); + } + return successResponse; + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + + @Override + public String getProvider() { + return PROVIDER; + } + + @Override + public String getModel() { + return MODEL; + } + + @Override + public Integer getDimension() { + return 1536; + } + + @Override + public String getInputModality(Object input) { + return "text"; + } + + @Override + public String getInputFormat(Object input) { + return "text"; + } + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeTest.java new file mode 100644 index 000000000000..eec7456f2390 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationRuntimeTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +public class ModelInvocationRuntimeTest { + + @Test + void defaultRetryPolicyAttemptsRequestOnce() { + ModelInvocationRuntime runtime = + new ModelInvocationRuntime(ModelInvocationOptions.defaults()); + CountingVectorAdapter adapter = + new CountingVectorAdapter( + ModelInvocationException.fromHttpStatus( + "OPENAI", "text-embedding-3-small", 429, "rate limited")); + + ModelInvocationException exception = + Assertions.assertThrows( + ModelInvocationException.class, + () -> runtime.invoke(new Object[] {"chunk"}, adapter)); + + Assertions.assertEquals(ModelInvocationErrorType.RATE_LIMIT, exception.getErrorType()); + Assertions.assertEquals(1, adapter.getAttempts()); + } + + @Test + void retryableRateLimitFailureSucceedsAfterRetry() throws IOException { + ModelInvocationRuntime runtime = new ModelInvocationRuntime(testRetryOptions()); + CountingVectorAdapter adapter = + new CountingVectorAdapter( + ModelInvocationException.fromHttpStatus( + "OPENAI", "text-embedding-3-small", 429, "rate limited")); + adapter.setSuccessResponse(vectors(1)); + + List> result = runtime.invoke(new Object[] {"chunk"}, adapter); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(2, adapter.getAttempts()); + } + + @Test + void retryableServerFailureSucceedsAfterRetry() throws IOException { + ModelInvocationRuntime runtime = new ModelInvocationRuntime(testRetryOptions()); + CountingVectorAdapter adapter = + new CountingVectorAdapter( + ModelInvocationException.fromHttpStatus( + "DOUBAO", "doubao-embedding", 500, "temporary failure")); + adapter.setSuccessResponse(vectors(1)); + + List> result = runtime.invoke(new Object[] {"chunk"}, adapter); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(2, adapter.getAttempts()); + } + + @Test + void retryableFailureIsRetriedUntilAttemptsAreExhausted() { + ModelInvocationRuntime runtime = new ModelInvocationRuntime(testRetryOptions()); + CountingVectorAdapter adapter = + new CountingVectorAdapter( + ModelInvocationException.fromHttpStatus( + "OPENAI", "text-embedding-3-small", 429, "rate limited"), + ModelInvocationException.fromHttpStatus( + "OPENAI", "text-embedding-3-small", 429, "rate limited"), + ModelInvocationException.fromHttpStatus( + "OPENAI", "text-embedding-3-small", 429, "rate limited")); + + ModelInvocationException exception = + Assertions.assertThrows( + ModelInvocationException.class, + () -> runtime.invoke(new Object[] {"chunk"}, adapter)); + + Assertions.assertEquals(ModelInvocationErrorType.RATE_LIMIT, exception.getErrorType()); + Assertions.assertEquals(3, adapter.getAttempts()); + } + + @Test + void socketTimeoutIsNormalizedAsRetryableTimeout() throws IOException { + ModelInvocationRuntime runtime = new ModelInvocationRuntime(testRetryOptions()); + CountingVectorAdapter adapter = new CountingVectorAdapter(new SocketTimeoutException()); + adapter.setSuccessResponse(vectors(1)); + + List> result = runtime.invoke(new Object[] {"chunk"}, adapter); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(2, adapter.getAttempts()); + } + + @Test + void authenticationFailureIsNotRetried() { + ModelInvocationRuntime runtime = new ModelInvocationRuntime(testRetryOptions()); + CountingVectorAdapter adapter = + new CountingVectorAdapter( + ModelInvocationException.fromHttpStatus( + "OPENAI", "text-embedding-3-small", 401, "bad key")); + + ModelInvocationException exception = + Assertions.assertThrows( + ModelInvocationException.class, + () -> runtime.invoke(new Object[] {"chunk"}, adapter)); + + Assertions.assertEquals( + ModelInvocationErrorType.AUTHENTICATION_ERROR, exception.getErrorType()); + Assertions.assertEquals(1, adapter.getAttempts()); + } + + @Test + void responseCountMismatchFailsWithoutEmittingMisalignedVectors() { + ModelInvocationRuntime runtime = new ModelInvocationRuntime(testRetryOptions()); + CountingVectorAdapter adapter = new CountingVectorAdapter(); + adapter.setSuccessResponse(vectors(1)); + + ModelInvocationException exception = + Assertions.assertThrows( + ModelInvocationException.class, + () -> runtime.invoke(new Object[] {"chunk-1", "chunk-2"}, adapter)); + + Assertions.assertEquals( + ModelInvocationErrorType.RESPONSE_COUNT_MISMATCH, exception.getErrorType()); + Assertions.assertEquals(1, adapter.getAttempts()); + } + + @Test + void responseParseFailureIsNotRetried() { + ModelInvocationRuntime runtime = new ModelInvocationRuntime(testRetryOptions()); + CountingVectorAdapter adapter = + new CountingVectorAdapter( + ModelInvocationException.nonRetryable( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, + "CUSTOM", + "custom-model", + "custom_response_parse did not match", + null)); + + ModelInvocationException exception = + Assertions.assertThrows( + ModelInvocationException.class, + () -> runtime.invoke(new Object[] {"chunk"}, adapter)); + + Assertions.assertEquals( + ModelInvocationErrorType.RESPONSE_PARSE_ERROR, exception.getErrorType()); + Assertions.assertEquals(1, adapter.getAttempts()); + } + + @Test + void httpStatusExceptionDoesNotExposeProviderResponseBody() { + ModelInvocationException exception = + ModelInvocationException.fromHttpStatus( + "OPENAI", + "text-embedding-3-small", + 401, + "api_key=secret original text chunk"); + + Assertions.assertFalse(exception.getMessage().contains("secret")); + Assertions.assertFalse(exception.getMessage().contains("original text chunk")); + Assertions.assertTrue(exception.getMessage().contains("HTTP status 401")); + } + + private static ModelInvocationOptions testRetryOptions() { + return new ModelInvocationOptions(3, 0L, 0L, 20000); + } + + private static List> vectors(int count) { + List> result = new ArrayList<>(); + for (int i = 0; i < count; i++) { + List vector = new ArrayList<>(); + vector.add(1.0f); + vector.add(2.0f); + result.add(vector); + } + return result; + } + + private static class CountingVectorAdapter implements ProviderAdapter>> { + + private final Queue failures = new LinkedList<>(); + private List> successResponse = vectors(1); + private int attempts; + + private CountingVectorAdapter(IOException... failures) { + for (IOException failure : failures) { + this.failures.add(failure); + } + } + + private void setSuccessResponse(List> successResponse) { + this.successResponse = successResponse; + } + + private int getAttempts() { + return attempts; + } + + @Override + public List> invoke(Object[] inputs, ModelInvocationContext context) + throws IOException { + attempts++; + if (!failures.isEmpty()) { + throw failures.remove(); + } + return successResponse; + } + + @Override + public int getOutputCount(List> output) { + return output == null ? 0 : output.size(); + } + } +} From 6d068adaa7724b4380e16bd0f251e1d820d03f96 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Sat, 20 Jun 2026 19:04:50 +0800 Subject: [PATCH 038/375] [Feature][Transform-V2] Add Calcite SQL transform plugin (#11062) --- docs/en/transforms/calcite-udf.md | 270 ++ docs/en/transforms/calcite.md | 507 ++ docs/sidebars.js | 2 + docs/zh/transforms/calcite-udf.md | 270 ++ docs/zh/transforms/calcite.md | 507 ++ seatunnel-dist/release-docs/NOTICE | 17 + .../TransformSpecificationCheckTest.java | 2 +- .../e2e/connector/file/local/LocalFileIT.java | 75 +- .../file/local/LocalFileWithMetadataIT.java | 7 +- .../e2e/transform/TestCalciteIT.java | 67 + .../calcite_builtin_functions.conf | 76 + .../calcite_transform/calcite_des_udf.conf | 84 + .../calcite_transform/calcite_json.conf | 69 + .../calcite_transform/calcite_mask_udf.conf | 82 + .../calcite_select_where.conf | 76 + seatunnel-shade/pom.xml | 1 + seatunnel-shade/seatunnel-calcite/pom.xml | 153 + seatunnel-shade/seatunnel-janino/pom.xml | 2 +- seatunnel-transforms-v2/pom.xml | 6 + .../calcite/CalciteMultiCatalogTransform.java | 52 + .../transform/calcite/CalciteTransform.java | 203 + .../calcite/CalciteTransformConfig.java | 30 + .../calcite/CalciteTransformFactory.java | 53 + .../calcite/adapter/SeaTunnelDataContext.java | 109 + .../adapter/SeaTunnelScannableTable.java | 59 + .../calcite/engine/CalciteSQLEngine.java | 222 + .../calcite/engine/CalciteSchemaFactory.java | 67 + .../calcite/type/CalciteTypeConverter.java | 262 + .../calcite/type/CalciteValueConverter.java | 174 + .../calcite/type/OutputRowTypeDeriver.java | 131 + .../udf/BinaryAwareScalarFunction.java | 105 + .../calcite/udf/BuiltinFunctions.java | 112 + .../transform/calcite/udf/CalciteUdf.java | 38 + .../calcite/udf/CalciteUdfContext.java | 150 + .../calcite/udf/CosineDistanceFunction.java | 64 + .../calcite/udf/DesDecryptFunction.java | 44 + .../calcite/udf/DesEncryptFunction.java | 43 + .../calcite/udf/InnerProductFunction.java | 54 + .../calcite/udf/L1DistanceFunction.java | 56 + .../calcite/udf/L2DistanceFunction.java | 62 + .../transform/calcite/udf/MaskFunction.java | 51 + .../calcite/udf/MaskHashFunction.java | 60 + .../calcite/udf/VectorDimsFunction.java | 46 + .../calcite/udf/VectorNormFunction.java | 47 + .../calcite/udf/VectorNormalizeFunction.java | 65 + .../calcite/udf/VectorReduceFunction.java | 151 + .../calcite/CalciteSQLEngineTest.java | 4252 +++++++++++++++++ .../calcite/CalciteTransformFactoryTest.java | 40 + .../calcite/CalciteTypeConverterTest.java | 681 +++ .../udf/DesEncryptDecryptFunctionTest.java | 90 + .../calcite/udf/MaskFunctionTest.java | 66 + .../calcite/udf/MaskHashFunctionTest.java | 72 + .../transform/calcite/udf/VectorUdfTest.java | 241 + tools/dependencies/known-dependencies.txt | 1 + 54 files changed, 10175 insertions(+), 51 deletions(-) create mode 100644 docs/en/transforms/calcite-udf.md create mode 100644 docs/en/transforms/calcite.md create mode 100644 docs/zh/transforms/calcite-udf.md create mode 100644 docs/zh/transforms/calcite.md create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/java/org/apache/seatunnel/e2e/transform/TestCalciteIT.java create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_builtin_functions.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_des_udf.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_json.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_mask_udf.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_select_where.conf create mode 100644 seatunnel-shade/seatunnel-calcite/pom.xml create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteMultiCatalogTransform.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransform.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransformConfig.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransformFactory.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/adapter/SeaTunnelDataContext.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/adapter/SeaTunnelScannableTable.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/engine/CalciteSQLEngine.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/engine/CalciteSchemaFactory.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/CalciteTypeConverter.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/CalciteValueConverter.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/OutputRowTypeDeriver.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BinaryAwareScalarFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BuiltinFunctions.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CalciteUdf.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CalciteUdfContext.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CosineDistanceFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/DesDecryptFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/DesEncryptFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/InnerProductFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/L1DistanceFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/L2DistanceFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/MaskFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/MaskHashFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorDimsFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorNormFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorNormalizeFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorReduceFunction.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteSQLEngineTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteTypeConverterTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/DesEncryptDecryptFunctionTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/MaskFunctionTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/MaskHashFunctionTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/VectorUdfTest.java diff --git a/docs/en/transforms/calcite-udf.md b/docs/en/transforms/calcite-udf.md new file mode 100644 index 000000000000..114257355947 --- /dev/null +++ b/docs/en/transforms/calcite-udf.md @@ -0,0 +1,270 @@ +# Calcite UDF + +> User-Defined Functions for the Calcite Transform plugin + +## Description + +Use the `CalciteUdf` SPI to extend the [Calcite Transform](calcite.md) with custom scalar functions. Implementations are discovered at runtime via Java `ServiceLoader`. + +## UDF API + +```java +package org.apache.seatunnel.transform.calcite.udf; + +public interface CalciteUdf extends AutoCloseable { + + /** + * SQL function name used in queries, e.g. "MY_UPPER". + * Case-insensitive at query time. + */ + String functionName(); + + /** Open UDF resources. Called once before first eval. */ + default void open() {} + + /** Release UDF resources. */ + @Override + default void close() throws Exception {} +} +``` + +## UDF Implementation Example + +### Step 1. Add Maven dependencies + +```xml + + + org.apache.seatunnel + seatunnel-transforms-v2 + ${seatunnel.version} + provided + + + org.apache.seatunnel + seatunnel-api + ${seatunnel.version} + provided + + + com.google.auto.service + auto-service + 1.1.1 + provided + + +``` + +### Step 2. Implement CalciteUdf + +Create a class that implements `CalciteUdf` and add a **public static `eval`** method: + +```java +package com.example; + +import org.apache.seatunnel.transform.calcite.udf.CalciteUdf; +import com.google.auto.service.AutoService; +import java.util.Locale; + +@AutoService(CalciteUdf.class) +public class MyUpperUdf implements CalciteUdf { + + @Override + public String functionName() { + return "MY_UPPER"; + } + + public static String eval(String input) { + return input == null ? null : input.toUpperCase(Locale.ROOT); + } +} +``` + +Key requirements: + +- `eval` **must be `public static`** -- Calcite's code generation calls it directly without creating an instance. An instance method would cause Calcite to create a new object for each call, bypassing any initialization done in `open()`. +- The `eval` method signature defines the SQL function's input/output types. For example, `String eval(String, int)` means the SQL function takes a VARCHAR and an INTEGER and returns a VARCHAR. +- `@AutoService(CalciteUdf.class)` generates the `META-INF/services` file for SPI discovery. +- `functionName()` returns the SQL function name. Function names are **case-insensitive** at query time -- `MY_UPPER(...)`, `my_upper(...)`, and `My_Upper(...)` all work. + +### Step 3. Deploy + +Build the JAR and place it in `${SEATUNNEL_HOME}/lib/`. If your UDF uses third-party libraries, include them in the same directory. + +If you use cluster mode, you need to place the JAR on all nodes' `${SEATUNNEL_HOME}/lib/` and restart the cluster. + +### Step 4. Use in SQL + +```sql +SELECT MY_UPPER(name) AS upper_name FROM source_table +``` + +## Lifecycle UDF Example + +If your UDF needs to initialize or release resources (e.g., database connections, caches), override `open()` and `close()`: + +```java +package com.example; + +import org.apache.seatunnel.transform.calcite.udf.CalciteUdf; +import com.google.auto.service.AutoService; + +@AutoService(CalciteUdf.class) +public class PrefixUdf implements CalciteUdf { + + private static volatile String prefix; + + @Override + public String functionName() { + return "WITH_PREFIX"; + } + + @Override + public void open() { + prefix = "HELLO"; + } + + public static String eval(String input) { + if (input == null) return null; + String p = prefix; + return (p != null ? p : "") + ": " + input; + } + + @Override + public void close() { + prefix = null; + } +} +``` + +:::caution + +Since `eval` must be static, shared state (like `prefix` above) must also be stored in a static field. Use `volatile` for simple references and proper synchronization for complex mutable state to ensure visibility across threads. + +::: + +## Context-aware UDF Example + +If your UDF needs access to row-level metadata (e.g., RowKind for CDC, table path), use `CalciteUdfContext.current()` inside the static `eval` method: + +```java +package com.example; + +import org.apache.seatunnel.transform.calcite.udf.CalciteUdf; +import org.apache.seatunnel.transform.calcite.udf.CalciteUdfContext; +import com.google.auto.service.AutoService; + +@AutoService(CalciteUdf.class) +public class RowKindUdf implements CalciteUdf { + + @Override + public String functionName() { + return "ROW_KIND"; + } + + public static String eval(String input) { + CalciteUdfContext ctx = CalciteUdfContext.current(); + if (ctx == null || input == null) return null; + return ctx.getRowKind().shortString() + ":" + input; + } +} +``` + +The `CalciteUdfContext` provides the following methods: + +| Method | Return Type | Description | +|--------|-------------|-------------| +| `getRawTableId()` | String | Raw table identifier (e.g., `db.schema.table`) | +| `getDatabase()` | String | Parsed database name | +| `getSchema()` | String | Parsed schema name | +| `getTable()` | String | Parsed table name | +| `getRowKind()` | RowKind | Row change type: `INSERT`, `UPDATE_BEFORE`, `UPDATE_AFTER`, `DELETE` | + +Usage: + +```sql +SELECT ROW_KIND(name) AS kind_name FROM source_table +``` + +## Multi-parameter UDF Example + +The `eval` method can accept multiple parameters of different types: + +```java +package com.example; + +import org.apache.seatunnel.transform.calcite.udf.CalciteUdf; +import com.google.auto.service.AutoService; + +@AutoService(CalciteUdf.class) +public class SubstringUdf implements CalciteUdf { + + @Override + public String functionName() { + return "MY_SUBSTR"; + } + + public static String eval(String input, int start, int length) { + if (input == null) return null; + int end = Math.min(start + length, input.length()); + return input.substring(Math.max(0, start), end); + } +} +``` + +Usage: + +```sql +SELECT MY_SUBSTR(name, 0, 3) AS short_name FROM source_table +``` + +## Complete Job Example + +Input: + +| id | name | age | +|----|------|-----| +| 1 | Joy Ding | 20 | +| 2 | May Ding | 21 | +| 3 | Kin Dom | 24 | +| 4 | Joy Dom | 22 | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, MY_UPPER(name) AS name, age FROM fake" + } +} +``` + +Output: + +| id | name | age | +|----|------|-----| +| 1 | JOY DING | 20 | +| 2 | MAY DING | 21 | +| 3 | KIN DOM | 24 | +| 4 | JOY DOM | 22 | + +## Type Mapping + +The `eval` method's Java types map to SQL types as follows: + +| Java Type | SQL Type | +|-----------|----------| +| `String` | VARCHAR | +| `int` / `Integer` | INTEGER | +| `long` / `Long` | BIGINT | +| `float` / `Float` | REAL | +| `double` / `Double` | DOUBLE | +| `boolean` / `Boolean` | BOOLEAN | +| `java.math.BigDecimal` | DECIMAL | +| `byte[]` | VARBINARY | + +## Changelog + +### next-release + +- Add Calcite UDF documentation diff --git a/docs/en/transforms/calcite.md b/docs/en/transforms/calcite.md new file mode 100644 index 000000000000..c07dcc4c6f44 --- /dev/null +++ b/docs/en/transforms/calcite.md @@ -0,0 +1,507 @@ +# Calcite + +> Calcite SQL transform plugin + +## Description + +SQL transform plugin powered by [Apache Calcite](https://calcite.apache.org/). Use standard SQL to transform data rows. The SQL plan is compiled once at job startup and applied to each row at runtime. + +:::tip + +- Each row is processed independently -- `JOIN` and cross-row aggregation (`GROUP BY`, `SUM`, `COUNT`) are **not** supported. +- Vector types (FLOAT_VECTOR, BINARY_VECTOR, etc.) are mapped to VARBINARY internally. Use the built-in vector UDFs (e.g., `COSINE_DISTANCE`, `VECTOR_REDUCE`) for vector operations. + +::: + +## Options + +| Name | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| sql | string | yes | - | SQL statement to execute | +| table_transform | list | no | [] | Per-table SQL overrides for multi-table CDC scenarios | +| table_match_regex | string | no | .* | Regex to match table paths. Unmatched tables pass through unchanged | +| row_error_handle_way | enum | no | FAIL | How to handle row-level errors: `FAIL`, `SKIP`, or `ROUTE_TO_TABLE` | + +### sql [string] + +```hocon +sql = "SELECT id, UPPER(name) AS name, age + 1 AS next_age FROM source_table WHERE age > 18" +``` + +### table_transform [list] + +Per-table SQL overrides for multi-table CDC scenarios. Each entry specifies a `table_path` and a `sql` statement. Tables not listed fall back to the global `sql` (if their path matches `table_match_regex`) or pass through unchanged. + +```hocon +table_transform = [ + { + table_path = "db.users" + sql = "SELECT id, name, UPPER(email) AS email FROM users" + }, + { + table_path = "db.orders" + sql = "SELECT order_id, amount * 100 AS amount_cents FROM orders" + } +] +``` + +### table_match_regex [string] + +A regex pattern to filter which tables should be transformed. Only tables whose path matches this regex will have the global `sql` applied. Tables that do not match pass through unchanged. Default is `.*` (match all). + +### row_error_handle_way [enum] + +How to handle errors during SQL execution for a row: + +- `FAIL` (default) -- fail the job immediately +- `SKIP` -- skip the problematic row and continue +- `ROUTE_TO_TABLE` -- route the error row to a separate error table + +### common options [string] + +Transform plugin common parameters, please refer to [Transform Plugin](common-options/common-options.md) for details. + +## Built-in UDFs + +All built-in UDFs return `NULL` when any required argument is `NULL`. +Function identifiers are case-insensitive. For example, `MASK(...)`, `mask(...)`, and `Mask(...)` are equivalent. + +### Data Masking Functions + +#### MASK + +```MASK(value, start, end, maskChar) -> STRING``` + +Replaces characters in range `[start, end)` with `maskChar`. Returns the original string if the range is invalid. Default mask char is `*` when null or empty. + +Example: + +```sql +SELECT MASK(phone, 3, 7, '*') AS masked_phone FROM t +``` + +#### MASK_HASH + +```MASK_HASH(value) -> STRING``` + +Returns the SHA-256 hex hash (64 characters) of the input. Deterministic -- same input always produces the same hash. + +Example: + +```sql +SELECT MASK_HASH(phone) AS phone_hash FROM t +``` + +#### DES_ENCRYPT + +```DES_ENCRYPT(password, data) -> STRING``` + +Encrypts `data` with DES (CBC/PKCS5Padding) using `password` (must be >= 8 chars). Returns Base64-encoded ciphertext. + +Example: + +```sql +SELECT DES_ENCRYPT('12345678', secret) AS encrypted FROM t +``` + +#### DES_DECRYPT + +```DES_DECRYPT(password, data) -> STRING``` + +Decrypts Base64-encoded `data` with the same password used for encryption. + +Example: + +```sql +SELECT DES_DECRYPT('12345678', encrypted_secret) AS original FROM t +``` + +### Vector Functions + +#### COSINE_DISTANCE + +```COSINE_DISTANCE(vector1, vector2) -> DOUBLE``` + +Returns a DOUBLE value between 0 and 1: 0 means identical vectors (completely similar), 1 means orthogonal vectors (completely dissimilar). + +Example: + +```sql +SELECT COSINE_DISTANCE(vec1, vec2) AS distance FROM t +``` + +#### L1_DISTANCE + +```L1_DISTANCE(vector1, vector2) -> DOUBLE``` + +Calculates the Manhattan (L1) distance between two vectors. + +Example: + +```sql +SELECT L1_DISTANCE(vec1, vec2) AS dist FROM t +``` + +#### L2_DISTANCE + +```L2_DISTANCE(vector1, vector2) -> DOUBLE``` + +Calculates the Euclidean (L2) distance between two vectors. + +Example: + +```sql +SELECT L2_DISTANCE(vec1, vec2) AS dist FROM t +``` + +#### VECTOR_DIMS + +```VECTOR_DIMS(vector) -> INT``` + +Returns an INT value representing the number of dimensions (elements) in the vector. + +Example: + +```sql +SELECT VECTOR_DIMS(embedding) AS dims FROM t +``` + +#### VECTOR_NORM + +```VECTOR_NORM(vector) -> DOUBLE``` + +Calculates the L2 norm (Euclidean norm) of a vector, which represents the length or magnitude of the vector. + +Example: + +```sql +SELECT VECTOR_NORM(embedding) AS norm FROM t +``` + +#### INNER_PRODUCT + +```INNER_PRODUCT(vector1, vector2) -> DOUBLE``` + +Calculates the inner product (dot product) of two vectors, which is used to measure the similarity and projection between the vectors. + +Example: + +```sql +SELECT INNER_PRODUCT(vec1, vec2) AS ip FROM t +``` + +#### VECTOR_REDUCE + +```VECTOR_REDUCE(vector_field, target_dimension, method)``` + +Generic vector dimension reduction function that supports multiple reduction methods. + +**Parameters:** +- `vector_field`: The vector field to reduce (VECTOR type) +- `target_dimension`: The target dimension (INTEGER, must be smaller than source dimension) +- `method`: The reduction method (STRING): + - **'TRUNCATE'**: Truncates the vector by keeping only the first N elements. Simplest and fastest, but may lose information in truncated dimensions. + - **'RANDOM_PROJECTION'**: Uses Gaussian random projection. Preserves relative distances between vectors following the Johnson-Lindenstrauss lemma. + - **'SPARSE_RANDOM_PROJECTION'**: Uses sparse random projection where matrix elements are mostly zero. More computationally efficient than regular random projection. + +**Returns:** VARBINARY -- the reduced vector + +**Example:** + +```sql +SELECT id, VECTOR_REDUCE(embedding, 256, 'TRUNCATE') AS reduced FROM t +SELECT id, VECTOR_REDUCE(embedding, 128, 'RANDOM_PROJECTION') AS reduced FROM t +SELECT id, VECTOR_REDUCE(embedding, 64, 'SPARSE_RANDOM_PROJECTION') AS reduced FROM t +``` + +#### VECTOR_NORMALIZE + +```VECTOR_NORMALIZE(vector_field)``` + +Normalizes a vector to unit length (magnitude = 1). Useful for computing cosine similarity. + +**Parameters:** +- `vector_field`: The vector field to normalize (VECTOR type) + +**Returns:** VARBINARY -- the normalized vector + +**Example:** + +```sql +SELECT id, VECTOR_NORMALIZE(embedding) AS unit_vec FROM t +``` + +In addition to the UDFs listed above, all standard SQL functions provided by Apache Calcite are available (string, math, date/time, JSON, conditional, etc.). For the full function reference, see the [Apache Calcite SQL Reference](https://calcite.apache.org/docs/reference.html). + +## Examples + +### Basic SELECT + WHERE + +The data read from source is a table like this: + +| id | name | age | +|----|------|-----| +| 1 | Joy Ding | 20 | +| 2 | May Ding | 21 | +| 3 | Kin Dom | 24 | +| 4 | Joy Dom | 15 | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, name, age FROM fake WHERE age >= 18" + } +} +``` + +Then the data in result table `result` will be: + +| id | name | age | +|----|------|-----| +| 1 | Joy Ding | 20 | +| 2 | May Ding | 21 | +| 3 | Kin Dom | 24 | + +Row with `age = 15` is filtered out. + +### String and Math Functions + +Input: + +| id | name | salary | +|----|------|--------| +| 1 | Joy Ding | 5000 | +| 2 | May Ding | 8000 | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, UPPER(name) AS name_upper, CHAR_LENGTH(name) AS name_len, salary * 1.1 AS new_salary FROM fake" + } +} +``` + +Output: + +| id | name_upper | name_len | new_salary | +|----|------------|----------|------------| +| 1 | JOY DING | 8 | 5500.0 | +| 2 | MAY DING | 8 | 8800.0 | + +### CASE WHEN Conditional + +Input: + +| id | name | age | +|----|------|-----| +| 1 | Alice | 8 | +| 2 | Bob | 15 | +| 3 | Carol | 30 | +| 4 | Dave | 70 | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, name, CASE WHEN age < 13 THEN 'child' WHEN age < 18 THEN 'teen' WHEN age < 65 THEN 'adult' ELSE 'senior' END AS age_group FROM fake" + } +} +``` + +Output: + +| id | name | age_group | +|----|------|-----------| +| 1 | Alice | child | +| 2 | Bob | teen | +| 3 | Carol | adult | +| 4 | Dave | senior | + +### JSON Extraction + +Input: + +| id | payload | +|----|---------| +| 1 | {"user": {"name": "Joy Ding", "email": "joy@example.com"}} | +| 2 | {"user": {"name": "May Ding", "email": "may@example.com"}} | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, JSON_VALUE(payload, '$.user.name') AS user_name, JSON_VALUE(payload, '$.user.email') AS email FROM fake" + } +} +``` + +Output: + +| id | user_name | email | +|----|-----------|-------| +| 1 | Joy Ding | joy@example.com | +| 2 | May Ding | may@example.com | + +### Data Masking (MASK + MASK_HASH + DES) + +Input: + +| id | phone | secret | +|----|-------|--------| +| 1 | 13812345678 | seatunnel-password | +| 2 | 13987654321 | connector-api-key | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, MASK(phone, 3, 7, '*') AS masked_phone, MASK_HASH(phone) AS phone_hash, DES_ENCRYPT('12345678', secret) AS encrypted_secret FROM fake" + } +} +``` + +Output: + +| id | masked_phone | phone_hash | encrypted_secret | +|----|--------------|------------|------------------| +| 1 | 138\*\*\*\*5678 | a1b2c3...(64-char SHA-256 hex) | Base64-encoded ciphertext | +| 2 | 139\*\*\*\*4321 | d4e5f6...(64-char SHA-256 hex) | Base64-encoded ciphertext | + +To decrypt later in the pipeline: + +```hocon +transform { + Calcite { + plugin_input = "result" + plugin_output = "decrypted" + sql = "SELECT id, DES_DECRYPT('12345678', encrypted_secret) AS original_secret FROM result" + } +} +``` + +### Vector Operations + +Use built-in vector UDFs to compute distances, reduce dimensions, or normalize vectors in a data pipeline (e.g., between Milvus/Qdrant source and sink). + +```hocon +transform { + Calcite { + plugin_input = "vector_source" + plugin_output = "result" + sql = "SELECT id, COSINE_DISTANCE(query_vec, doc_vec) AS distance, VECTOR_DIMS(doc_vec) AS dims, VECTOR_REDUCE(doc_vec, 128, 'TRUNCATE') AS reduced_vec FROM vector_source" + } +} +``` + +Given two FLOAT_VECTOR columns `query_vec` and `doc_vec`, this computes the cosine distance, extracts dimensions, and reduces `doc_vec` from its original dimension to 128. + +### Multi-table CDC (table_transform) + +```hocon +transform { + Calcite { + plugin_input = "cdc_source" + plugin_output = "result" + table_transform = [ + { + table_path = "db.users" + sql = "SELECT id, name, UPPER(email) AS email FROM users" + }, + { + table_path = "db.orders" + sql = "SELECT order_id, amount * 100 AS amount_cents FROM orders" + } + ] + } +} +``` + +Tables not listed in `table_transform` but matching `table_match_regex` (default `.*`) will have the global `sql` applied. Tables not matching any rule pass through unchanged. + +### Error Handling (row_error_handle_way) + +```hocon +transform { + Calcite { + plugin_input = "source_table" + plugin_output = "result" + sql = "SELECT id, CAST(age AS VARCHAR) AS age_str FROM source_table" + row_error_handle_way = "SKIP" + } +} +``` + +When a row causes a SQL execution error: + +- `FAIL` -- the job fails immediately (default, recommended for data quality) +- `SKIP` -- the problematic row is silently dropped +- `ROUTE_TO_TABLE` -- the row is sent to a separate error table for later inspection + +## Custom UDF + +Custom scalar functions can be added via the `CalciteUdf` SPI. For the full development guide, API reference, examples, and type mapping, see [Calcite UDF](calcite-udf.md). + +## Limitations + +| Limitation | Detail | +|------------|--------| +| Single input table | Only one table is registered in the Calcite schema per transform. Multi-table `JOIN` is not supported | +| Row-at-a-time processing | Each row is processed independently. `GROUP BY` / `SUM()` / `COUNT()` operate on a single row and are generally not useful for batch aggregation | +| WHERE filtering | `WHERE` conditions that evaluate to `false` cause the row to be dropped (not passed through) | +| Table name matching | The `FROM` table name in SQL must exactly match the `plugin_input` value | +| Scalar UDFs only | Only scalar functions are supported. Table-valued functions and aggregate UDFs are not available | +| Vector type mapping | Vector types are mapped to VARBINARY internally. Use built-in vector UDFs (COSINE_DISTANCE, L1_DISTANCE, etc.) for vector operations | + +:::tip CDC schema changes +When an `AlterTableEvent` is received (for example, add/drop columns), the engine automatically rebuilds the SQL plan and re-infers the output schema. No manual intervention is needed. +::: + +## Job Config Example + +```hocon +env { + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + row.num = 100 + schema = { + fields { + id = "int" + name = "string" + age = "int" + phone = "string" + } + } + } +} + +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, UPPER(name) AS name, age + 1 AS age, MASK(phone, 3, 7, '*') AS phone FROM fake WHERE age >= 0" + } +} + +sink { + Console { + plugin_input = "result" + } +} +``` + +## Changelog + +### next-release + +- Add Calcite Transform plugin diff --git a/docs/sidebars.js b/docs/sidebars.js index d5980484d303..28bcf937f97d 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -243,6 +243,8 @@ const sidebars = { "transforms/common-options/common-options" ] }, + "transforms/calcite", + "transforms/calcite-udf", "transforms/copy", "transforms/data-validator", "transforms/define-sink-type", diff --git a/docs/zh/transforms/calcite-udf.md b/docs/zh/transforms/calcite-udf.md new file mode 100644 index 000000000000..e8e02ba34138 --- /dev/null +++ b/docs/zh/transforms/calcite-udf.md @@ -0,0 +1,270 @@ +# Calcite 用户自定义函数 + +> Calcite Transform 插件的用户自定义函数 (UDF) + +## 描述 + +使用 `CalciteUdf` SPI 扩展 [Calcite Transform](calcite.md) 的自定义标量函数。实现类通过 Java `ServiceLoader` 在运行时自动发现。 + +## UDF API + +```java +package org.apache.seatunnel.transform.calcite.udf; + +public interface CalciteUdf extends AutoCloseable { + + /** + * SQL 函数名，如 "MY_UPPER"。 + * 查询时大小写不敏感。 + */ + String functionName(); + + /** 初始化 UDF 资源。在第一次 eval 之前调用一次。 */ + default void open() {} + + /** 释放 UDF 资源。 */ + @Override + default void close() throws Exception {} +} +``` + +## UDF 实现示例 + +### 第一步：添加 Maven 依赖 + +```xml + + + org.apache.seatunnel + seatunnel-transforms-v2 + ${seatunnel.version} + provided + + + org.apache.seatunnel + seatunnel-api + ${seatunnel.version} + provided + + + com.google.auto.service + auto-service + 1.1.1 + provided + + +``` + +### 第二步：实现 CalciteUdf + +创建一个实现 `CalciteUdf` 接口的类，并添加 **public static `eval`** 方法： + +```java +package com.example; + +import org.apache.seatunnel.transform.calcite.udf.CalciteUdf; +import com.google.auto.service.AutoService; +import java.util.Locale; + +@AutoService(CalciteUdf.class) +public class MyUpperUdf implements CalciteUdf { + + @Override + public String functionName() { + return "MY_UPPER"; + } + + public static String eval(String input) { + return input == null ? null : input.toUpperCase(Locale.ROOT); + } +} +``` + +关键要求： + +- `eval` **必须是 `public static`** -- Calcite 代码生成直接调用静态方法，不创建实例。实例方法会导致 Calcite 每次创建新对象，绕过 `open()` 中的初始化。 +- `eval` 方法签名决定 SQL 函数的输入/输出类型。例如 `String eval(String, int)` 表示 SQL 函数接受 VARCHAR 和 INTEGER 参数，返回 VARCHAR。 +- `@AutoService(CalciteUdf.class)` 自动生成 `META-INF/services` 文件用于 SPI 发现。 +- `functionName()` 返回 SQL 函数名。函数名**大小写不敏感** -- `MY_UPPER(...)`、`my_upper(...)`、`My_Upper(...)` 均可使用。 + +### 第三步：部署 + +构建 JAR 并放入 `${SEATUNNEL_HOME}/lib/`。如果 UDF 依赖第三方库，也需要一并放入该目录。 + +如果使用集群模式，需要将 JAR 放到所有节点的 `${SEATUNNEL_HOME}/lib/` 并重启集群。 + +### 第四步：在 SQL 中使用 + +```sql +SELECT MY_UPPER(name) AS upper_name FROM source_table +``` + +## 带生命周期的 UDF 示例 + +如果 UDF 需要初始化或释放资源（如数据库连接、缓存），可以覆写 `open()` 和 `close()`： + +```java +package com.example; + +import org.apache.seatunnel.transform.calcite.udf.CalciteUdf; +import com.google.auto.service.AutoService; + +@AutoService(CalciteUdf.class) +public class PrefixUdf implements CalciteUdf { + + private static volatile String prefix; + + @Override + public String functionName() { + return "WITH_PREFIX"; + } + + @Override + public void open() { + prefix = "HELLO"; + } + + public static String eval(String input) { + if (input == null) return null; + String p = prefix; + return (p != null ? p : "") + ": " + input; + } + + @Override + public void close() { + prefix = null; + } +} +``` + +:::caution + +由于 `eval` 必须是静态方法，共享状态（如上面的 `prefix`）必须存储在静态字段中。对于简单引用类型请使用 `volatile`，对于复杂可变状态请使用适当的同步机制，以确保跨线程的可见性。 + +::: + +## 上下文感知 UDF 示例 + +如果 UDF 需要访问行级元数据（如 CDC 场景的 RowKind、表路径等），可以在静态 `eval` 方法中通过 `CalciteUdfContext.current()` 获取： + +```java +package com.example; + +import org.apache.seatunnel.transform.calcite.udf.CalciteUdf; +import org.apache.seatunnel.transform.calcite.udf.CalciteUdfContext; +import com.google.auto.service.AutoService; + +@AutoService(CalciteUdf.class) +public class RowKindUdf implements CalciteUdf { + + @Override + public String functionName() { + return "ROW_KIND"; + } + + public static String eval(String input) { + CalciteUdfContext ctx = CalciteUdfContext.current(); + if (ctx == null || input == null) return null; + return ctx.getRowKind().shortString() + ":" + input; + } +} +``` + +`CalciteUdfContext` 提供以下方法： + +| 方法 | 返回类型 | 说明 | +|------|---------|------| +| `getRawTableId()` | String | 原始表标识符（如 `db.schema.table`） | +| `getDatabase()` | String | 解析后的数据库名 | +| `getSchema()` | String | 解析后的 Schema 名 | +| `getTable()` | String | 解析后的表名 | +| `getRowKind()` | RowKind | 行变更类型：`INSERT`、`UPDATE_BEFORE`、`UPDATE_AFTER`、`DELETE` | + +使用： + +```sql +SELECT ROW_KIND(name) AS kind_name FROM source_table +``` + +## 多参数 UDF 示例 + +`eval` 方法可以接受多个不同类型的参数： + +```java +package com.example; + +import org.apache.seatunnel.transform.calcite.udf.CalciteUdf; +import com.google.auto.service.AutoService; + +@AutoService(CalciteUdf.class) +public class SubstringUdf implements CalciteUdf { + + @Override + public String functionName() { + return "MY_SUBSTR"; + } + + public static String eval(String input, int start, int length) { + if (input == null) return null; + int end = Math.min(start + length, input.length()); + return input.substring(Math.max(0, start), end); + } +} +``` + +使用： + +```sql +SELECT MY_SUBSTR(name, 0, 3) AS short_name FROM source_table +``` + +## 完整作业示例 + +输入： + +| id | name | age | +|----|------|-----| +| 1 | Joy Ding | 20 | +| 2 | May Ding | 21 | +| 3 | Kin Dom | 24 | +| 4 | Joy Dom | 22 | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, MY_UPPER(name) AS name, age FROM fake" + } +} +``` + +输出： + +| id | name | age | +|----|------|-----| +| 1 | JOY DING | 20 | +| 2 | MAY DING | 21 | +| 3 | KIN DOM | 24 | +| 4 | JOY DOM | 22 | + +## 类型映射 + +`eval` 方法的 Java 类型与 SQL 类型的对应关系： + +| Java 类型 | SQL 类型 | +|-----------|----------| +| `String` | VARCHAR | +| `int` / `Integer` | INTEGER | +| `long` / `Long` | BIGINT | +| `float` / `Float` | REAL | +| `double` / `Double` | DOUBLE | +| `boolean` / `Boolean` | BOOLEAN | +| `java.math.BigDecimal` | DECIMAL | +| `byte[]` | VARBINARY | + +## 更新日志 + +### next-release + +- 添加 Calcite UDF 文档 diff --git a/docs/zh/transforms/calcite.md b/docs/zh/transforms/calcite.md new file mode 100644 index 000000000000..a3b178b2da3d --- /dev/null +++ b/docs/zh/transforms/calcite.md @@ -0,0 +1,507 @@ +# Calcite + +> Calcite SQL Transform 插件 + +## 描述 + +基于 [Apache Calcite](https://calcite.apache.org/) 的 SQL Transform 插件。使用标准 SQL 对数据行进行转换，在作业启动时编译 SQL 执行计划，运行时逐行应用。 + +:::tip + +- 每行独立处理——不支持 `JOIN` 和跨行聚合（`GROUP BY`、`SUM`、`COUNT`）。 +- 向量类型（FLOAT_VECTOR、BINARY_VECTOR 等）内部映射为 VARBINARY。请使用内置向量 UDF（如 `COSINE_DISTANCE`、`VECTOR_REDUCE`）进行向量运算。 + +::: + +## 属性 + +| 名称 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| sql | string | 是 | - | 要执行的 SQL 语句 | +| table_transform | list | 否 | [] | 多表 CDC 场景下的逐表 SQL 覆盖 | +| table_match_regex | string | 否 | .* | 表路径匹配正则。不匹配的表直接透传 | +| row_error_handle_way | enum | 否 | FAIL | 行级错误处理方式：`FAIL`、`SKIP`、`ROUTE_TO_TABLE` | + +### sql [string] + +```hocon +sql = "SELECT id, UPPER(name) AS name, age + 1 AS next_age FROM source_table WHERE age > 18" +``` + +### table_transform [list] + +多表 CDC 场景下的逐表 SQL 覆盖。每项指定 `table_path` 和 `sql`。未列出的表会回退到全局 `sql`（如果路径匹配 `table_match_regex`），否则直接透传。 + +```hocon +table_transform = [ + { + table_path = "db.users" + sql = "SELECT id, name, UPPER(email) AS email FROM users" + }, + { + table_path = "db.orders" + sql = "SELECT order_id, amount * 100 AS amount_cents FROM orders" + } +] +``` + +### table_match_regex [string] + +用于过滤需要转换的表的正则表达式。只有路径匹配此正则的表才会应用全局 `sql`。不匹配的表直接透传。默认 `.*`（匹配所有）。 + +### row_error_handle_way [enum] + +行级 SQL 执行错误的处理方式： + +- `FAIL`（默认）-- 立即终止作业 +- `SKIP` -- 跳过错误行，继续处理 +- `ROUTE_TO_TABLE` -- 将错误行路由到独立的错误表 + +### 公共参数 [string] + +Transform 公共参数，请参考 [Transform 插件公共参数](common-options/common-options.md)。 + +## 内置 UDF + +所有内置 UDF 在任意必需参数为 `NULL` 时返回 `NULL`。 +函数名大小写不敏感。例如 `MASK(...)`、`mask(...)`、`Mask(...)` 等价。 + +### 数据脱敏函数 + +#### MASK + +```MASK(value, start, end, maskChar) -> STRING``` + +将 `[start, end)` 范围内的字符替换为 `maskChar`。范围无效时返回原值。maskChar 为 null 或空时默认 `*`。 + +示例： + +```sql +SELECT MASK(phone, 3, 7, '*') AS masked_phone FROM t +``` + +#### MASK_HASH + +```MASK_HASH(value) -> STRING``` + +返回输入的 SHA-256 十六进制哈希（64 字符）。确定性——相同输入总是产生相同哈希。 + +示例： + +```sql +SELECT MASK_HASH(phone) AS phone_hash FROM t +``` + +#### DES_ENCRYPT + +```DES_ENCRYPT(password, data) -> STRING``` + +使用 `password`（不少于 8 字符）对 `data` 进行 DES 加密（CBC/PKCS5Padding），返回 Base64 编码密文。 + +示例： + +```sql +SELECT DES_ENCRYPT('12345678', secret) AS encrypted FROM t +``` + +#### DES_DECRYPT + +```DES_DECRYPT(password, data) -> STRING``` + +使用相同密码解密 Base64 编码的密文。 + +示例： + +```sql +SELECT DES_DECRYPT('12345678', encrypted_secret) AS original FROM t +``` + +### 向量函数 + +#### COSINE_DISTANCE + +```COSINE_DISTANCE(vector1, vector2) -> DOUBLE``` + +返回介于 0 和 1 之间的 DOUBLE 值：0 表示完全相同的向量，1 表示完全正交的向量。 + +示例： + +```sql +SELECT COSINE_DISTANCE(vec1, vec2) AS distance FROM t +``` + +#### L1_DISTANCE + +```L1_DISTANCE(vector1, vector2) -> DOUBLE``` + +计算两个向量之间的曼哈顿（L1）距离。 + +示例： + +```sql +SELECT L1_DISTANCE(vec1, vec2) AS dist FROM t +``` + +#### L2_DISTANCE + +```L2_DISTANCE(vector1, vector2) -> DOUBLE``` + +计算两个向量之间的欧几里得（L2）距离。 + +示例： + +```sql +SELECT L2_DISTANCE(vec1, vec2) AS dist FROM t +``` + +#### VECTOR_DIMS + +```VECTOR_DIMS(vector) -> INT``` + +返回一个 INT 值，表示向量中的维数（元素数量）。 + +示例： + +```sql +SELECT VECTOR_DIMS(embedding) AS dims FROM t +``` + +#### VECTOR_NORM + +```VECTOR_NORM(vector) -> DOUBLE``` + +计算向量的 L2 范数（欧几里得范数），表示向量的长度或大小。 + +示例： + +```sql +SELECT VECTOR_NORM(embedding) AS norm FROM t +``` + +#### INNER_PRODUCT + +```INNER_PRODUCT(vector1, vector2) -> DOUBLE``` + +计算两个向量的内积（点积），用于测量向量之间的相似性和投影。 + +示例： + +```sql +SELECT INNER_PRODUCT(vec1, vec2) AS ip FROM t +``` + +#### VECTOR_REDUCE + +```VECTOR_REDUCE(vector_field, target_dimension, method)``` + +通用向量降维函数，支持多种降维方法。 + +**参数：** +- `vector_field`：要降维的向量字段（VECTOR 类型） +- `target_dimension`：目标维度（INTEGER，必须小于源维度） +- `method`：降维方法（STRING）： + - **'TRUNCATE'**：截断法，保留前 N 个元素。最简单快速，但可能丢失被截断维度的信息。 + - **'RANDOM_PROJECTION'**：高斯随机投影法。在降维的同时保持向量间的相对距离，遵循 Johnson-Lindenstrauss 引理。 + - **'SPARSE_RANDOM_PROJECTION'**：稀疏随机投影法，矩阵元素大多为零。比常规随机投影更高效。 + +**返回值：** VARBINARY——降维后的向量 + +**示例：** + +```sql +SELECT id, VECTOR_REDUCE(embedding, 256, 'TRUNCATE') AS reduced FROM t +SELECT id, VECTOR_REDUCE(embedding, 128, 'RANDOM_PROJECTION') AS reduced FROM t +SELECT id, VECTOR_REDUCE(embedding, 64, 'SPARSE_RANDOM_PROJECTION') AS reduced FROM t +``` + +#### VECTOR_NORMALIZE + +```VECTOR_NORMALIZE(vector_field)``` + +将向量归一化为单位长度（模长 = 1）。对于计算余弦相似度很有用。 + +**参数：** +- `vector_field`：要归一化的向量字段（VECTOR 类型） + +**返回值：** VARBINARY——归一化后的向量 + +**示例：** + +```sql +SELECT id, VECTOR_NORMALIZE(embedding) AS unit_vec FROM t +``` + +除上述 UDF 外，Apache Calcite 提供的所有标准 SQL 函数均可使用（字符串、数学、日期/时间、JSON、条件表达式等）。完整函数参考请见 [Apache Calcite SQL 参考文档](https://calcite.apache.org/docs/reference.html)。 + +## 示例 + +### 基础 SELECT + WHERE + +从 Source 读取的数据如下： + +| id | name | age | +|----|------|-----| +| 1 | Joy Ding | 20 | +| 2 | May Ding | 21 | +| 3 | Kin Dom | 24 | +| 4 | Joy Dom | 15 | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, name, age FROM fake WHERE age >= 18" + } +} +``` + +结果表 `result` 中的数据为： + +| id | name | age | +|----|------|-----| +| 1 | Joy Ding | 20 | +| 2 | May Ding | 21 | +| 3 | Kin Dom | 24 | + +`age = 15` 的行被过滤。 + +### 字符串和数学函数 + +输入： + +| id | name | salary | +|----|------|--------| +| 1 | Joy Ding | 5000 | +| 2 | May Ding | 8000 | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, UPPER(name) AS name_upper, CHAR_LENGTH(name) AS name_len, salary * 1.1 AS new_salary FROM fake" + } +} +``` + +输出： + +| id | name_upper | name_len | new_salary | +|----|------------|----------|------------| +| 1 | JOY DING | 8 | 5500.0 | +| 2 | MAY DING | 8 | 8800.0 | + +### CASE WHEN 条件表达式 + +输入： + +| id | name | age | +|----|------|-----| +| 1 | Alice | 8 | +| 2 | Bob | 15 | +| 3 | Carol | 30 | +| 4 | Dave | 70 | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, name, CASE WHEN age < 13 THEN 'child' WHEN age < 18 THEN 'teen' WHEN age < 65 THEN 'adult' ELSE 'senior' END AS age_group FROM fake" + } +} +``` + +输出： + +| id | name | age_group | +|----|------|-----------| +| 1 | Alice | child | +| 2 | Bob | teen | +| 3 | Carol | adult | +| 4 | Dave | senior | + +### JSON 提取 + +输入： + +| id | payload | +|----|---------| +| 1 | {"user": {"name": "Joy Ding", "email": "joy@example.com"}} | +| 2 | {"user": {"name": "May Ding", "email": "may@example.com"}} | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, JSON_VALUE(payload, '$.user.name') AS user_name, JSON_VALUE(payload, '$.user.email') AS email FROM fake" + } +} +``` + +输出： + +| id | user_name | email | +|----|-----------|-------| +| 1 | Joy Ding | joy@example.com | +| 2 | May Ding | may@example.com | + +### 数据脱敏（MASK + MASK_HASH + DES） + +输入： + +| id | phone | secret | +|----|-------|--------| +| 1 | 13812345678 | seatunnel-password | +| 2 | 13987654321 | connector-api-key | + +```hocon +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, MASK(phone, 3, 7, '*') AS masked_phone, MASK_HASH(phone) AS phone_hash, DES_ENCRYPT('12345678', secret) AS encrypted_secret FROM fake" + } +} +``` + +输出： + +| id | masked_phone | phone_hash | encrypted_secret | +|----|--------------|------------|------------------| +| 1 | 138\*\*\*\*5678 | a1b2c3...（64 字符 SHA-256 hex） | Base64 编码密文 | +| 2 | 139\*\*\*\*4321 | d4e5f6...（64 字符 SHA-256 hex） | Base64 编码密文 | + +后续解密： + +```hocon +transform { + Calcite { + plugin_input = "result" + plugin_output = "decrypted" + sql = "SELECT id, DES_DECRYPT('12345678', encrypted_secret) AS original_secret FROM result" + } +} +``` + +### 向量运算 + +使用内置向量 UDF 在数据管线中计算距离、降维或归一化（例如 Milvus/Qdrant 源与目标之间的处理）。 + +```hocon +transform { + Calcite { + plugin_input = "vector_source" + plugin_output = "result" + sql = "SELECT id, COSINE_DISTANCE(query_vec, doc_vec) AS distance, VECTOR_DIMS(doc_vec) AS dims, VECTOR_REDUCE(doc_vec, 128, 'TRUNCATE') AS reduced_vec FROM vector_source" + } +} +``` + +给定两个 FLOAT_VECTOR 列 `query_vec` 和 `doc_vec`，此配置计算余弦距离、提取维度，并将 `doc_vec` 从原始维度降至 128 维。 + +### 多表 CDC（table_transform） + +```hocon +transform { + Calcite { + plugin_input = "cdc_source" + plugin_output = "result" + table_transform = [ + { + table_path = "db.users" + sql = "SELECT id, name, UPPER(email) AS email FROM users" + }, + { + table_path = "db.orders" + sql = "SELECT order_id, amount * 100 AS amount_cents FROM orders" + } + ] + } +} +``` + +未列入 `table_transform` 但匹配 `table_match_regex`（默认 `.*`）的表会应用全局 `sql`。不匹配任何规则的表直接透传。 + +### 错误处理（row_error_handle_way） + +```hocon +transform { + Calcite { + plugin_input = "source_table" + plugin_output = "result" + sql = "SELECT id, CAST(age AS VARCHAR) AS age_str FROM source_table" + row_error_handle_way = "SKIP" + } +} +``` + +行级 SQL 执行出错时： + +- `FAIL` -- 立即终止作业（默认，推荐用于数据质量要求高的场景） +- `SKIP` -- 静默跳过错误行 +- `ROUTE_TO_TABLE` -- 将错误行路由到独立错误表，便于后续排查 + +## 自定义 UDF + +通过 `CalciteUdf` SPI 添加自定义标量函数。完整的开发指南、API 参考、示例和类型映射请参阅 [Calcite 用户自定义函数](calcite-udf.md)。 + +## 限制 + +| 限制 | 说明 | +|------|------| +| 单表输入 | 每个 Transform 只注册一张表到 Calcite Schema，不支持多表 `JOIN` | +| 逐行处理 | 每行独立处理。`GROUP BY` / `SUM()` / `COUNT()` 作用于单行，通常无实际聚合意义 | +| WHERE 过滤 | `WHERE` 条件为 `false` 的行会被丢弃（不透传） | +| 表名匹配 | SQL `FROM` 中的表名必须与 `plugin_input` 值完全一致 | +| 仅标量 UDF | 仅支持标量函数，不支持表值函数和聚合 UDF | +| 向量类型映射 | 向量类型内部映射为 VARBINARY。可使用内置向量 UDF（COSINE_DISTANCE、L1_DISTANCE 等）进行向量运算 | + +:::tip CDC Schema 变更 +收到 `AlterTableEvent`（例如加列、删列）时，引擎会自动重建 SQL 执行计划并重新推导输出 Schema，无需手动干预。 +::: + +## 作业配置示例 + +```hocon +env { + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + row.num = 100 + schema = { + fields { + id = "int" + name = "string" + age = "int" + phone = "string" + } + } + } +} + +transform { + Calcite { + plugin_input = "fake" + plugin_output = "result" + sql = "SELECT id, UPPER(name) AS name, age + 1 AS age, MASK(phone, 3, 7, '*') AS phone FROM fake WHERE age >= 0" + } +} + +sink { + Console { + plugin_input = "result" + } +} +``` + +## 更新日志 + +### next-release + +- 新增 Calcite Transform 插件 diff --git a/seatunnel-dist/release-docs/NOTICE b/seatunnel-dist/release-docs/NOTICE index f354b32bf9ae..6dd8daa88024 100644 --- a/seatunnel-dist/release-docs/NOTICE +++ b/seatunnel-dist/release-docs/NOTICE @@ -717,6 +717,23 @@ originating from the Calcite project (https://github.com/apache/calcite). ========================================================================= +Apache Calcite NOTICE + +========================================================================= + +Apache Calcite +Copyright 2012-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product is based on source code originally developed +by DynamoBI Corporation, LucidEra Inc., SQLstream Inc. and others +under the auspices of the Eigenbase Foundation +and released as the LucidDB project. + +========================================================================= + Apache Hadoop NOTICE ========================================================================= diff --git a/seatunnel-dist/src/test/java/org/apache/seatunnel/api/connector/TransformSpecificationCheckTest.java b/seatunnel-dist/src/test/java/org/apache/seatunnel/api/connector/TransformSpecificationCheckTest.java index ce51350644ff..d963d315df56 100644 --- a/seatunnel-dist/src/test/java/org/apache/seatunnel/api/connector/TransformSpecificationCheckTest.java +++ b/seatunnel-dist/src/test/java/org/apache/seatunnel/api/connector/TransformSpecificationCheckTest.java @@ -43,7 +43,7 @@ void testAllTransformUseFactory() { FactoryUtil.discoverFactories( Thread.currentThread().getContextClassLoader(), TableTransformFactory.class); - Assertions.assertEquals(21, factories.size()); + Assertions.assertEquals(22, factories.size()); } @Test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java index 213771b4e96a..81fab333d518 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java @@ -344,105 +344,94 @@ public class LocalFileIT extends TestSuiteBase { }; @TestTemplate - public void testLocalFileReadAndWrite(TestContainer container) - throws IOException, InterruptedException { + public void testLocalFileCsv(TestContainer container) throws IOException, InterruptedException { TestHelper helper = new TestHelper(container); helper.execute("/csv/fake_to_local_csv.conf"); helper.execute("/csv/local_csv_to_assert.conf"); helper.execute("/csv/local_csv_enable_split_to_assert.conf"); helper.execute("/csv/csv_with_header_to_assert.conf"); helper.execute("/csv/breakline_csv_to_assert.conf"); + } + + @TestTemplate + public void testLocalFileExcel(TestContainer container) + throws IOException, InterruptedException { + TestHelper helper = new TestHelper(container); helper.execute("/excel/fake_to_local_excel.conf"); helper.execute("/excel/local_excel_to_assert.conf"); helper.execute("/excel/local_excel_projection_to_assert.conf"); helper.execute("/excel/special_excel_to_assert.conf"); - // test write local text file + helper.execute("/excel/local_filter_excel_to_assert.conf"); + helper.execute("/excel/local_filter_regex_excel_to_assert.conf"); + } + + @TestTemplate + public void testLocalFileText(TestContainer container) + throws IOException, InterruptedException { + TestHelper helper = new TestHelper(container); helper.execute("/text/fake_to_local_file_text.conf"); helper.execute("/text/local_file_text_lzo_to_assert.conf"); helper.execute("/text/local_file_delimiter_assert.conf"); helper.execute("/text/local_file_time_format_assert.conf"); - // test read skip header helper.execute("/text/local_file_text_skip_headers.conf"); - // test read local text file helper.execute("/text/local_file_text_to_assert.conf"); - // test read local text file with projection helper.execute("/text/local_file_text_projection_to_assert.conf"); - // test read local csv file with assigning encoding helper.execute("/text/fake_to_local_file_with_encoding.conf"); - // test read local csv file with assigning encoding helper.execute("/text/local_file_text_to_console_with_encoding.conf"); helper.execute("/text/local_file_null_format_assert.conf"); + } - // test write local json file + @TestTemplate + public void testLocalFileJson(TestContainer container) + throws IOException, InterruptedException { + TestHelper helper = new TestHelper(container); helper.execute("/json/fake_to_local_file_json.conf"); - // test read local json file helper.execute("/json/local_file_json_to_assert.conf"); helper.execute("/json/local_file_json_enable_split_to_assert.conf"); helper.execute("/json/local_file_json_lzo_to_console.conf"); - // test read local json file with assigning encoding helper.execute("/json/fake_to_local_file_json_with_encoding.conf"); - // test write local json file with assigning encoding helper.execute("/json/local_file_json_to_console_with_encoding.conf"); + helper.execute("/json/local_file_to_console.conf"); + } - // test write local orc file + @TestTemplate + public void testLocalFileOrcParquetBinaryXml(TestContainer container) + throws IOException, InterruptedException { + TestHelper helper = new TestHelper(container); helper.execute("/orc/fake_to_local_file_orc.conf"); - // test read local orc file helper.execute("/orc/local_file_orc_to_assert.conf"); - // test read local orc file with projection helper.execute("/orc/local_file_orc_projection_to_assert.conf"); - // test read local orc file with projection and type cast helper.execute("/orc/local_file_orc_to_assert_with_time_and_cast.conf"); - // test write local parquet file helper.execute("/parquet/fake_to_local_file_parquet.conf"); - // test read local parquet file helper.execute("/parquet/local_file_parquet_to_assert.conf"); helper.execute("/parquet/local_file_parquet_enable_split_to_assert.conf"); - // test read local parquet file with projection helper.execute("/parquet/local_file_parquet_projection_to_assert.conf"); - // test read filtered local file - helper.execute("/excel/local_filter_excel_to_assert.conf"); - // test read filtered local file with regex - helper.execute("/excel/local_filter_regex_excel_to_assert.conf"); - - // test read empty directory - helper.execute("/json/local_file_to_console.conf"); helper.execute("/parquet/local_file_to_console.conf"); - - // test binary file helper.execute("/binary/local_file_binary_to_local_file_binary.conf"); if (!container.identifier().getEngineType().equals(EngineType.FLINK)) { - // the file generated by local_file_binary_to_local_file_binary in taskManager, so read - // from jobManager will be failed in Flink helper.execute("/binary/local_file_binary_to_assert.conf"); } - helper.execute("/xml/local_file_xml_to_assert.conf"); - /** Compressed file test */ - // test read single local text file with zip compression + } + + @TestTemplate + public void testLocalFileCompressed(TestContainer container) + throws IOException, InterruptedException { + TestHelper helper = new TestHelper(container); helper.execute("/text/local_file_zip_text_to_assert.conf"); helper.execute("/text/local_file_gz_text_to_assert.conf"); - // test read multi local text file with zip compression helper.execute("/text/local_file_multi_zip_text_to_assert.conf"); - // test read single local text file with tar compression helper.execute("/text/local_file_tar_text_to_assert.conf"); helper.execute("/text/local_file_text_enable_split_to_assert.conf"); - // test read multi local text file with tar compression helper.execute("/text/local_file_multi_tar_text_to_assert.conf"); - // test read single local text file with tar.gz compression helper.execute("/text/local_file_tar_gz_text_to_assert.conf"); - // test read multi local text file with tar.gz compression helper.execute("/text/local_file_multi_tar_gz_text_to_assert.conf"); - // test read single local json file with zip compression helper.execute("/json/local_file_json_zip_to_assert.conf"); helper.execute("/json/local_file_json_gz_to_assert.conf"); - // test read multi local json file with zip compression helper.execute("/json/local_file_json_multi_zip_to_assert.conf"); - // test read single local xml file with zip compression helper.execute("/xml/local_file_zip_xml_to_assert.conf"); helper.execute("/xml/local_file_gz_xml_to_assert.conf"); - // test read single local excel file with zip compression helper.execute("/excel/local_excel_zip_to_assert.conf"); - // test read multi local excel file with zip compression helper.execute("/excel/local_excel_multi_zip_to_assert.conf"); helper.execute("/excel/local_excel_xls_gz_to_assert.conf"); helper.execute("/excel/local_excel_xlsx_gz_to_assert.conf"); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileWithMetadataIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileWithMetadataIT.java index 99301d7a9503..64b44b4421d1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileWithMetadataIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileWithMetadataIT.java @@ -39,7 +39,6 @@ import java.nio.file.Paths; import java.util.Arrays; -import java.util.Collections; import static org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH; @@ -87,7 +86,7 @@ public void startUp() throws Exception { .withEnv("TZ", "UTC") .withCommand(buildStartCommand()) .withNetworkAliases("server") - .withExposedPorts() + .withExposedPorts(5801, 8080) .withFileSystemBind("/tmp", "/opt/hive") .withLogConsumer( new Slf4jLogConsumer( @@ -153,8 +152,6 @@ private void startMySQLContainer() throws Exception { .waitingFor(Wait.forHealthcheck()) .withLogConsumer( new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); - mysqlContainer.setPortBindings( - Collections.singletonList(String.format("%s:%s", MYSQL_PORT, MYSQL_PORT))); mysqlContainer.start(); log.info("MySQL container started at {}", mysqlContainer.getHost()); // Wait for MySQL to be fully ready @@ -171,8 +168,6 @@ private void startGravitinoServer() throws Exception { new Slf4jLogConsumer( DockerLoggerFactory.getLogger( "gravitino:" + GRAVITINO_IMAGE))); - gravitinoContainer.setPortBindings( - Collections.singletonList(String.format("%s:%s", GRAVITINO_PORT, GRAVITINO_PORT))); gravitinoContainer.start(); log.info("Gravitino server started at {}", gravitinoContainer.getHost()); // Create metalake and catalog using curl with MySQL as backend diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/java/org/apache/seatunnel/e2e/transform/TestCalciteIT.java b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/java/org/apache/seatunnel/e2e/transform/TestCalciteIT.java new file mode 100644 index 000000000000..9c2b591d87b9 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/java/org/apache/seatunnel/e2e/transform/TestCalciteIT.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.transform; + +import org.apache.seatunnel.e2e.common.container.TestContainer; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; + +import java.io.IOException; + +public class TestCalciteIT extends TestSuiteBase { + + @TestTemplate + public void testCalciteSelectWhere(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = + container.executeJob("/calcite_transform/calcite_select_where.conf"); + Assertions.assertEquals(0, result.getExitCode()); + } + + @TestTemplate + public void testCalciteBuiltinFunctions(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = + container.executeJob("/calcite_transform/calcite_builtin_functions.conf"); + Assertions.assertEquals(0, result.getExitCode()); + } + + @TestTemplate + public void testCalciteJson(TestContainer container) throws IOException, InterruptedException { + Container.ExecResult result = container.executeJob("/calcite_transform/calcite_json.conf"); + Assertions.assertEquals(0, result.getExitCode()); + } + + @TestTemplate + public void testCalciteMaskUdf(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = + container.executeJob("/calcite_transform/calcite_mask_udf.conf"); + Assertions.assertEquals(0, result.getExitCode()); + } + + @TestTemplate + public void testCalciteDesUdf(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = + container.executeJob("/calcite_transform/calcite_des_udf.conf"); + Assertions.assertEquals(0, result.getExitCode()); + } +} diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_builtin_functions.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_builtin_functions.conf new file mode 100644 index 000000000000..ead73d13f443 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_builtin_functions.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + row.num = 100 + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + } +} + +transform { + Calcite { + plugin_input = "fake" + plugin_output = "calcite_out" + sql = "SELECT id, UPPER(name) AS name_upper, ABS(age) AS abs_age, COALESCE(name, 'unknown') AS safe_name FROM fake" + } +} + +sink { + Assert { + plugin_input = "calcite_out" + rules = { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 100 + } + ] + field_rules = [ + { + field_name = name_upper + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = abs_age + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_des_udf.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_des_udf.conf new file mode 100644 index 000000000000..1afba7182091 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_des_udf.conf @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + schema = { + fields { + id = "int" + secret = "string" + } + } + rows = [ + {fields = [1, "seatunnel-transform"], kind = INSERT}, + {fields = [2, "seatunnel-connector"], kind = INSERT} + ] + } +} + +transform { + Calcite { + plugin_input = "fake" + plugin_output = "encrypted" + sql = "SELECT id, secret, DES_ENCRYPT('12345678', secret) AS encrypted_val FROM fake" + } + + Calcite { + plugin_input = "encrypted" + plugin_output = "decrypted" + sql = "SELECT id, secret, encrypted_val, DES_DECRYPT('12345678', encrypted_val) AS decrypted_val FROM encrypted" + } +} + +sink { + Assert { + plugin_input = "decrypted" + rules = { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 2 + } + ] + field_rules = [ + { + field_name = encrypted_val + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = decrypted_val + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_json.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_json.conf new file mode 100644 index 000000000000..42a32757af33 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_json.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + schema = { + fields { + id = "int" + json_data = "string" + } + } + rows = [ + {fields = [1, "{\"name\":\"Tom\",\"age\":25}"], kind = INSERT}, + {fields = [2, "{\"name\":\"Alice\",\"age\":30}"], kind = INSERT} + ] + } +} + +transform { + Calcite { + plugin_input = "fake" + plugin_output = "calcite_out" + sql = "SELECT id, JSON_VALUE(json_data, '$.name') AS user_name FROM fake" + } +} + +sink { + Assert { + plugin_input = "calcite_out" + rules = { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 2 + } + ] + field_rules = [ + { + field_name = user_name + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_mask_udf.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_mask_udf.conf new file mode 100644 index 000000000000..9fb1f5495e4b --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_mask_udf.conf @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + schema = { + fields { + id = "int" + phone = "string" + } + } + rows = [ + {fields = [1, "13812345678"], kind = INSERT}, + {fields = [2, "13987654321"], kind = INSERT} + ] + } +} + +transform { + Calcite { + plugin_input = "fake" + plugin_output = "calcite_out" + sql = "SELECT id, MASK(phone, 3, 7, '*') AS masked_phone, MASK_HASH(phone) AS phone_hash FROM fake" + } +} + +sink { + Assert { + plugin_input = "calcite_out" + rules = { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 2 + } + ] + field_rules = [ + { + field_name = masked_phone + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = phone_hash + field_type = string + field_value = [ + { + rule_type = NOT_NULL + }, + { + rule_type = MIN_LENGTH + rule_value = 64 + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_select_where.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_select_where.conf new file mode 100644 index 000000000000..f4bb4b33f6b6 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/calcite_transform/calcite_select_where.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + row.num = 100 + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + } +} + +transform { + Calcite { + plugin_input = "fake" + plugin_output = "calcite_out" + sql = "SELECT id, name FROM fake WHERE age >= 0" + } +} + +sink { + Assert { + plugin_input = "calcite_out" + rules = { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + } + ] + field_rules = [ + { + field_name = id + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = name + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-shade/pom.xml b/seatunnel-shade/pom.xml index 63a207b96a26..416ca691c880 100644 --- a/seatunnel-shade/pom.xml +++ b/seatunnel-shade/pom.xml @@ -33,6 +33,7 @@ seatunnel-thrift-service seatunnel-hazelcast seatunnel-janino + seatunnel-calcite seatunnel-scala-compiler seatunnel-jetty9-9.4.56 seatunnel-hadoop-aws diff --git a/seatunnel-shade/seatunnel-calcite/pom.xml b/seatunnel-shade/seatunnel-calcite/pom.xml new file mode 100644 index 000000000000..035f92a8e61e --- /dev/null +++ b/seatunnel-shade/seatunnel-calcite/pom.xml @@ -0,0 +1,153 @@ + + + + 4.0.0 + + + org.apache.seatunnel + seatunnel-shade + ${revision} + + + seatunnel-calcite + SeaTunnel : Shade : Calcite + + 1.38.0 + + + + + org.apache.calcite + calcite-core + ${calcite.version} + true + + + com.fasterxml.jackson.core + * + + + com.fasterxml.jackson.dataformat + * + + + org.slf4j + slf4j-api + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + seatunnel-calcite + ${enableSourceJarCreation} + true + false + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.apache.calcite + ${seatunnel.shade.package}.org.apache.calcite + + + org.apache.calcite.avatica + ${seatunnel.shade.package}.org.apache.calcite.avatica + + + org.codehaus.janino + ${seatunnel.shade.package}.org.codehaus.janino + + + org.codehaus.commons + ${seatunnel.shade.package}.org.codehaus.commons + + + org.apache.commons.codec + ${seatunnel.shade.package}.org.apache.commons.codec + + + com.jayway.jsonpath + ${seatunnel.shade.package}.com.jayway.jsonpath + + + net.minidev + ${seatunnel.shade.package}.net.minidev + + + org.apiguardian + ${seatunnel.shade.package}.org.apiguardian + + + org.checkerframework + ${seatunnel.shade.package}.org.checkerframework + + + com.google + ${seatunnel.shade.package}.com.google + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-artifacts + + attach-artifact + + package + + + + ${basedir}/target/seatunnel-calcite.jar + jar + optional + + + + + + + + + + diff --git a/seatunnel-shade/seatunnel-janino/pom.xml b/seatunnel-shade/seatunnel-janino/pom.xml index a3cd3667421b..c384a9c86614 100644 --- a/seatunnel-shade/seatunnel-janino/pom.xml +++ b/seatunnel-shade/seatunnel-janino/pom.xml @@ -26,7 +26,7 @@ seatunnel-janino SeaTunnel : Shade : Janino - 3.0.11 + 3.1.12 diff --git a/seatunnel-transforms-v2/pom.xml b/seatunnel-transforms-v2/pom.xml index 01c49fa2b342..3187d9900184 100644 --- a/seatunnel-transforms-v2/pom.xml +++ b/seatunnel-transforms-v2/pom.xml @@ -101,6 +101,12 @@ ${project.version} optional + + org.apache.seatunnel + seatunnel-calcite + ${project.version} + optional + org.apache.httpcomponents httpclient diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteMultiCatalogTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteMultiCatalogTransform.java new file mode 100644 index 000000000000..5137f9482053 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteMultiCatalogTransform.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.transform.SeaTunnelFlatMapTransform; +import org.apache.seatunnel.api.transform.SeaTunnelTransform; +import org.apache.seatunnel.transform.common.AbstractMultiCatalogFlatMapTransform; +import org.apache.seatunnel.transform.common.IdentityFlatMapTransform; + +import java.util.List; + +public class CalciteMultiCatalogTransform extends AbstractMultiCatalogFlatMapTransform { + + public CalciteMultiCatalogTransform( + List inputCatalogTables, ReadonlyConfig config) { + super(inputCatalogTables, config); + } + + @Override + public String getPluginName() { + return CalciteTransform.PLUGIN_NAME; + } + + @Override + protected SeaTunnelFlatMapTransform buildTransform( + CatalogTable inputCatalogTable, ReadonlyConfig config) { + return new CalciteTransform(config, inputCatalogTable); + } + + @Override + protected SeaTunnelTransform createIdentityTransform(CatalogTable catalogTable) { + return new IdentityFlatMapTransform(catalogTable); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransform.java new file mode 100644 index 000000000000..89be320516ad --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransform.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.options.ConnectorCommonOptions; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.Column; +import org.apache.seatunnel.api.table.catalog.ConstraintKey; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.schema.event.AlterTableEvent; +import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent; +import org.apache.seatunnel.api.table.schema.handler.AlterTableSchemaEventHandler; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.transform.calcite.engine.CalciteSQLEngine; +import org.apache.seatunnel.transform.common.AbstractCatalogSupportFlatMapTransform; +import org.apache.seatunnel.transform.common.TransformCommonOptions; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class CalciteTransform extends AbstractCatalogSupportFlatMapTransform { + + public static final String PLUGIN_NAME = "Calcite"; + + private final String sql; + private final String inputTableName; + private transient CalciteSQLEngine engine; + + public CalciteTransform(@NonNull ReadonlyConfig config, @NonNull CatalogTable catalogTable) { + super(catalogTable, config.get(TransformCommonOptions.ROW_ERROR_HANDLE_WAY_OPTION)); + this.sql = config.get(CalciteTransformConfig.SQL); + List pluginInputIdentifiers = config.get(ConnectorCommonOptions.PLUGIN_INPUT); + if (pluginInputIdentifiers != null && !pluginInputIdentifiers.isEmpty()) { + this.inputTableName = pluginInputIdentifiers.get(0); + } else { + this.inputTableName = catalogTable.getTableId().getTableName(); + } + } + + @Override + public String getPluginName() { + return PLUGIN_NAME; + } + + @Override + public void open() { + engine = new CalciteSQLEngine(sql, inputTableName, inputCatalogTable.getSeaTunnelRowType()); + engine.init(); + } + + private void tryOpen() { + if (engine == null) { + open(); + } + } + + @Override + protected List transformRow(SeaTunnelRow inputRow) { + tryOpen(); + return engine.execute(inputRow); + } + + @Override + protected TableSchema transformTableSchema() { + tryOpen(); + SeaTunnelRowType outRowType = engine.getOutputRowType(); + List outputColumns = Arrays.asList(outRowType.getFieldNames()); + + TableSchema.Builder builder = TableSchema.builder(); + if (inputCatalogTable.getTableSchema().getPrimaryKey() != null + && new HashSet<>(outputColumns) + .containsAll( + inputCatalogTable + .getTableSchema() + .getPrimaryKey() + .getColumnNames())) { + builder.primaryKey(inputCatalogTable.getTableSchema().getPrimaryKey().copy()); + } + + List outputConstraintKeys = + inputCatalogTable.getTableSchema().getConstraintKeys().stream() + .filter( + key -> { + List constraintColumnNames = + key.getColumnNames().stream() + .map( + ConstraintKey.ConstraintKeyColumn + ::getColumnName) + .collect(Collectors.toList()); + return new HashSet<>(outputColumns) + .containsAll(constraintColumnNames); + }) + .map(ConstraintKey::copy) + .collect(Collectors.toList()); + builder.constraintKey(outputConstraintKeys); + + String[] fieldNames = outRowType.getFieldNames(); + SeaTunnelDataType[] fieldTypes = outRowType.getFieldTypes(); + List columns = new ArrayList<>(fieldNames.length); + for (int i = 0; i < fieldNames.length; i++) { + Column inputColumn = findInputColumn(fieldNames[i]); + Column column; + if (inputColumn != null) { + column = + new PhysicalColumn( + fieldNames[i], + fieldTypes[i], + inputColumn.getColumnLength(), + inputColumn.getScale(), + inputColumn.isNullable(), + inputColumn.getDefaultValue(), + inputColumn.getComment(), + inputColumn.getSourceType(), + inputColumn.getOptions()); + } else { + column = PhysicalColumn.of(fieldNames[i], fieldTypes[i], 0, true, null, null); + } + columns.add(column); + } + return builder.columns(columns).build(); + } + + private Column findInputColumn(String name) { + for (Column col : inputCatalogTable.getTableSchema().getColumns()) { + if (col.getName().equalsIgnoreCase(name)) { + return col; + } + } + return null; + } + + @Override + protected TableIdentifier transformTableIdentifier() { + return inputCatalogTable.getTableId().copy(); + } + + @Override + public SchemaChangeEvent mapSchemaChangeEvent(SchemaChangeEvent event) { + if (event instanceof AlterTableEvent) { + TableSchema newSchema = + new AlterTableSchemaEventHandler() + .reset(inputCatalogTable.getTableSchema()) + .apply(event); + inputCatalogTable = + CatalogTable.of( + inputCatalogTable.getTableId(), + newSchema, + inputCatalogTable.getOptions(), + inputCatalogTable.getPartitionKeys(), + inputCatalogTable.getComment(), + inputCatalogTable.getTableId().getCatalogName(), + inputCatalogTable.getMetadataSchema()); + closeEngine(); + outputCatalogTable = null; + } + return event; + } + + @Override + public void setInputCatalogTable(@NonNull CatalogTable inputCatalogTable) { + super.setInputCatalogTable(inputCatalogTable); + closeEngine(); + } + + private void closeEngine() { + if (engine != null) { + engine.close(); + engine = null; + } + } + + @Override + public void close() { + closeEngine(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransformConfig.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransformConfig.java new file mode 100644 index 000000000000..d513959110ff --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransformConfig.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; + +public final class CalciteTransformConfig { + + public static final Option SQL = + Options.key("sql") + .stringType() + .noDefaultValue() + .withDescription("The SQL statement to execute using Apache Calcite engine"); +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransformFactory.java new file mode 100644 index 000000000000..c135d3389eb5 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/CalciteTransformFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.table.connector.TableTransform; +import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.TableTransformFactory; +import org.apache.seatunnel.api.table.factory.TableTransformFactoryContext; +import org.apache.seatunnel.transform.common.TransformCommonOptions; + +import com.google.auto.service.AutoService; + +@AutoService(Factory.class) +public class CalciteTransformFactory implements TableTransformFactory { + + @Override + public String factoryIdentifier() { + return CalciteTransform.PLUGIN_NAME; + } + + @Override + public OptionRule optionRule() { + return OptionRule.builder() + .required(CalciteTransformConfig.SQL) + .optional(TransformCommonOptions.MULTI_TABLES) + .optional(TransformCommonOptions.TABLE_MATCH_REGEX) + .optional(TransformCommonOptions.ROW_ERROR_HANDLE_WAY_OPTION) + .optional(TransformCommonOptions.COLUMN_ERROR_HANDLE_WAY_OPTION) + .build(); + } + + @Override + public TableTransform createTransform(TableTransformFactoryContext context) { + return () -> + new CalciteMultiCatalogTransform(context.getCatalogTables(), context.getOptions()); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/adapter/SeaTunnelDataContext.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/adapter/SeaTunnelDataContext.java new file mode 100644 index 000000000000..504c01890a7f --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/adapter/SeaTunnelDataContext.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.adapter; + +import org.apache.seatunnel.shade.org.apache.calcite.DataContext; +import org.apache.seatunnel.shade.org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.seatunnel.shade.org.apache.calcite.linq4j.QueryProvider; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.seatunnel.shade.org.apache.calcite.schema.SchemaPlus; + +import java.util.Locale; +import java.util.TimeZone; + +/** Minimal {@link DataContext} for Bindable execution without a full {@code CalciteConnection}. */ +public final class SeaTunnelDataContext implements DataContext { + + private final SchemaPlus rootSchema; + private final RelDataTypeFactory typeFactory; + private final TimeZone timeZone; + private final Locale locale; + private final String user; + + private long currentEpochMillis; + + public SeaTunnelDataContext(SchemaPlus rootSchema, RelDataTypeFactory typeFactory) { + this( + rootSchema, + typeFactory, + TimeZone.getDefault(), + Locale.getDefault(), + System.getProperty("user.name", "")); + } + + public SeaTunnelDataContext( + SchemaPlus rootSchema, + RelDataTypeFactory typeFactory, + TimeZone timeZone, + Locale locale, + String user) { + this.rootSchema = rootSchema; + this.typeFactory = typeFactory; + this.timeZone = timeZone; + this.locale = locale; + this.user = user; + this.currentEpochMillis = System.currentTimeMillis(); + } + + /** + * Snapshots the current wall-clock time. The engine should invoke this once per row execution + * so that {@code CURRENT_TIMESTAMP} / {@code LOCAL_TIMESTAMP} / {@code UTC_TIMESTAMP} remain + * stable for the duration of a single statement evaluation. + */ + public void refreshNow() { + this.currentEpochMillis = System.currentTimeMillis(); + } + + @Override + public SchemaPlus getRootSchema() { + return rootSchema; + } + + @Override + public JavaTypeFactory getTypeFactory() { + return (JavaTypeFactory) typeFactory; + } + + @Override + public QueryProvider getQueryProvider() { + return null; + } + + @Override + public Object get(String name) { + if (name == null) { + return null; + } + if (DataContext.Variable.CURRENT_TIMESTAMP.camelName.equals(name) + || DataContext.Variable.LOCAL_TIMESTAMP.camelName.equals(name) + || DataContext.Variable.UTC_TIMESTAMP.camelName.equals(name)) { + return currentEpochMillis; + } + if (DataContext.Variable.TIME_ZONE.camelName.equals(name)) { + return timeZone; + } + if (DataContext.Variable.LOCALE.camelName.equals(name)) { + return locale; + } + if (DataContext.Variable.USER.camelName.equals(name) + || DataContext.Variable.SYSTEM_USER.camelName.equals(name)) { + return user; + } + return null; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/adapter/SeaTunnelScannableTable.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/adapter/SeaTunnelScannableTable.java new file mode 100644 index 000000000000..c1638e3e3e8e --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/adapter/SeaTunnelScannableTable.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.adapter; + +import org.apache.seatunnel.shade.org.apache.calcite.DataContext; +import org.apache.seatunnel.shade.org.apache.calcite.linq4j.Enumerable; +import org.apache.seatunnel.shade.org.apache.calcite.linq4j.Linq4j; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataType; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.seatunnel.shade.org.apache.calcite.schema.ScannableTable; +import org.apache.seatunnel.shade.org.apache.calcite.schema.impl.AbstractTable; + +import lombok.Setter; + +import java.util.Collections; + +/** + * A Calcite {@link ScannableTable} backed by a single SeaTunnel row. Each call to {@link + * #scan(DataContext)} returns an {@link Enumerable} containing only the current row. The row is + * injected before each SQL execution via {@link #setCurrentRow(Object[])}. + */ +public class SeaTunnelScannableTable extends AbstractTable implements ScannableTable { + + private final RelDataType rowType; + + @Setter private Object[] currentRow; + + public SeaTunnelScannableTable(RelDataType rowType) { + this.rowType = rowType; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return rowType; + } + + @Override + public Enumerable scan(DataContext root) { + if (currentRow == null) { + return Linq4j.emptyEnumerable(); + } + return Linq4j.asEnumerable(Collections.singletonList(currentRow)); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/engine/CalciteSQLEngine.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/engine/CalciteSQLEngine.java new file mode 100644 index 000000000000..cb31f12e2267 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/engine/CalciteSQLEngine.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.engine; + +import org.apache.seatunnel.shade.org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.seatunnel.shade.org.apache.calcite.adapter.enumerable.EnumerableInterpretable; +import org.apache.seatunnel.shade.org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.seatunnel.shade.org.apache.calcite.avatica.util.Casing; +import org.apache.seatunnel.shade.org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.seatunnel.shade.org.apache.calcite.rel.RelNode; +import org.apache.seatunnel.shade.org.apache.calcite.rel.RelRoot; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataType; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.seatunnel.shade.org.apache.calcite.runtime.Bindable; +import org.apache.seatunnel.shade.org.apache.calcite.schema.SchemaPlus; +import org.apache.seatunnel.shade.org.apache.calcite.sql.SqlNode; +import org.apache.seatunnel.shade.org.apache.calcite.sql.parser.SqlParser; +import org.apache.seatunnel.shade.org.apache.calcite.tools.FrameworkConfig; +import org.apache.seatunnel.shade.org.apache.calcite.tools.Frameworks; +import org.apache.seatunnel.shade.org.apache.calcite.tools.Planner; +import org.apache.seatunnel.shade.org.apache.calcite.tools.Programs; + +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.transform.calcite.adapter.SeaTunnelDataContext; +import org.apache.seatunnel.transform.calcite.adapter.SeaTunnelScannableTable; +import org.apache.seatunnel.transform.calcite.type.CalciteValueConverter; +import org.apache.seatunnel.transform.calcite.type.OutputRowTypeDeriver; +import org.apache.seatunnel.transform.calcite.udf.BuiltinFunctions; +import org.apache.seatunnel.transform.calcite.udf.CalciteUdfContext; +import org.apache.seatunnel.transform.exception.TransformCommonError; +import org.apache.seatunnel.transform.exception.TransformException; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Core Calcite SQL engine that parses, validates, compiles and executes SQL against a single + * SeaTunnel row. The execution plan is compiled once and reused for each row. + */ +@Slf4j +public class CalciteSQLEngine implements AutoCloseable { + + private final String sql; + private final String tableName; + private final SeaTunnelRowType inputRowType; + + private SchemaPlus rootSchema; + private SeaTunnelScannableTable scannableTable; + private Bindable bindable; + + @Getter private SeaTunnelRowType outputRowType; + private BuiltinFunctions builtinFunctions; + private RelDataTypeFactory typeFactory; + private SeaTunnelDataContext dataContext; + + public CalciteSQLEngine(String sql, String tableName, SeaTunnelRowType inputRowType) { + this.sql = sql; + this.tableName = tableName; + this.inputRowType = inputRowType; + } + + /** + * Initializes the engine: parses, validates and compiles the SQL into a reusable Bindable plan. + * Must be called before {@link #execute(SeaTunnelRow)}. + */ + @SuppressWarnings("unchecked") + public void init() { + Planner planner = null; + try { + rootSchema = Frameworks.createRootSchema(true); + typeFactory = new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + + scannableTable = + CalciteSchemaFactory.registerTable( + rootSchema, tableName, inputRowType, typeFactory); + + builtinFunctions = new BuiltinFunctions(); + builtinFunctions.discoverAndRegister(rootSchema); + dataContext = new SeaTunnelDataContext(rootSchema, typeFactory); + + SqlParser.Config parserConfig = + SqlParser.config() + .withCaseSensitive(false) + .withQuotedCasing(Casing.UNCHANGED) + .withUnquotedCasing(Casing.UNCHANGED); + + FrameworkConfig frameworkConfig = + Frameworks.newConfigBuilder() + .defaultSchema(rootSchema) + .parserConfig(parserConfig) + .programs(Programs.standard()) + .build(); + + planner = Frameworks.getPlanner(frameworkConfig); + SqlNode parsed = planner.parse(sql); + SqlNode validated = planner.validate(parsed); + + RelRoot relRoot = planner.rel(validated); + RelNode logicalPlan = relRoot.rel; + RelDataType validatedRowType = relRoot.validatedRowType; + + RelNode enumerablePlan = + Programs.standard() + .run( + logicalPlan.getCluster().getPlanner(), + logicalPlan, + logicalPlan + .getTraitSet() + .replace(EnumerableConvention.INSTANCE), + Collections.emptyList(), + Collections.emptyList()); + + bindable = + EnumerableInterpretable.toBindable( + Collections.emptyMap(), + null, + (EnumerableRel) enumerablePlan, + EnumerableRel.Prefer.ARRAY); + + outputRowType = + new OutputRowTypeDeriver(inputRowType).derive(validated, validatedRowType); + + log.info("Calcite SQL engine initialized successfully for table '{}'", tableName); + } catch (TransformException e) { + close(); + throw e; + } catch (Exception e) { + close(); + throw TransformCommonError.sqlExpressionError(sql, e); + } finally { + if (planner != null) { + planner.close(); + } + } + } + + /** + * Executes the pre-compiled SQL plan against a single input row. Returns a list of output rows + * (typically 1, but UNNEST may produce 0-N rows). + */ + public List execute(SeaTunnelRow inputRow) { + Object[] calciteRow = toCalciteRow(inputRow); + scannableTable.setCurrentRow(calciteRow); + dataContext.refreshNow(); + + List results = new ArrayList<>(); + try (CalciteUdfContext.Scope ignored = + CalciteUdfContext.enter(inputRow.getTableId(), inputRow.getRowKind())) { + for (Object rawRow : bindable.bind(dataContext)) { + Object[] row; + if (rawRow instanceof Object[]) { + row = (Object[]) rawRow; + } else { + row = new Object[] {rawRow}; + } + results.add(toSeaTunnelRow(row, inputRow)); + } + } catch (Exception e) { + throw TransformCommonError.sqlExpressionError(sql, e); + } finally { + scannableTable.setCurrentRow(null); + } + return results; + } + + private Object[] toCalciteRow(SeaTunnelRow row) { + Object[] values = new Object[row.getArity()]; + for (int i = 0; i < row.getArity(); i++) { + values[i] = CalciteValueConverter.toCalcite(row.getField(i)); + } + return values; + } + + private SeaTunnelRow toSeaTunnelRow(Object[] calciteRow, SeaTunnelRow inputRow) { + Object[] values = new Object[calciteRow.length]; + SeaTunnelDataType[] fieldTypes = outputRowType.getFieldTypes(); + for (int i = 0; i < calciteRow.length; i++) { + values[i] = CalciteValueConverter.fromCalcite(calciteRow[i], fieldTypes[i]); + } + SeaTunnelRow result = new SeaTunnelRow(values); + result.setTableId(inputRow.getTableId()); + result.setRowKind(inputRow.getRowKind()); + result.setOptions(inputRow.getOptions()); + return result; + } + + @Override + public void close() { + if (builtinFunctions != null) { + builtinFunctions.close(); + builtinFunctions = null; + } + rootSchema = null; + scannableTable = null; + bindable = null; + dataContext = null; + typeFactory = null; + outputRowType = null; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/engine/CalciteSchemaFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/engine/CalciteSchemaFactory.java new file mode 100644 index 000000000000..b411714c27ab --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/engine/CalciteSchemaFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.engine; + +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataType; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.seatunnel.shade.org.apache.calcite.schema.SchemaPlus; + +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.transform.calcite.adapter.SeaTunnelScannableTable; +import org.apache.seatunnel.transform.calcite.type.CalciteTypeConverter; + +import lombok.experimental.UtilityClass; + +import java.util.ArrayList; +import java.util.List; + +/** + * Registers a SeaTunnel table (by name and row-type) into a Calcite {@link SchemaPlus} as a {@link + * SeaTunnelScannableTable}. + */ +@UtilityClass +public final class CalciteSchemaFactory { + + /** + * Registers a table with the given name into the Calcite schema. + * + * @return the created {@link SeaTunnelScannableTable} so the caller can inject rows later + */ + public static SeaTunnelScannableTable registerTable( + SchemaPlus schema, + String tableName, + SeaTunnelRowType seaTunnelRowType, + RelDataTypeFactory typeFactory) { + + String[] fieldNames = seaTunnelRowType.getFieldNames(); + List names = new ArrayList<>(fieldNames.length); + List types = new ArrayList<>(fieldNames.length); + for (int i = 0; i < fieldNames.length; i++) { + names.add(fieldNames[i]); + RelDataType fieldType = + CalciteTypeConverter.toCalciteType( + typeFactory, seaTunnelRowType.getFieldType(i)); + types.add(typeFactory.createTypeWithNullability(fieldType, true)); + } + RelDataType rowType = typeFactory.createStructType(types, names); + + SeaTunnelScannableTable table = new SeaTunnelScannableTable(rowType); + schema.add(tableName, table); + return table; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/CalciteTypeConverter.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/CalciteTypeConverter.java new file mode 100644 index 000000000000..60306fa2e713 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/CalciteTypeConverter.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.type; + +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataType; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.seatunnel.shade.org.apache.calcite.sql.type.SqlTypeName; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; +import org.apache.seatunnel.transform.exception.TransformCommonErrorCode; +import org.apache.seatunnel.transform.exception.TransformException; + +import lombok.experimental.UtilityClass; + +import java.util.ArrayList; +import java.util.List; + +/** + * Bidirectional type converter between {@link SeaTunnelDataType} and Calcite {@link RelDataType}. + * Covers all types defined in {@link SqlType}. + */ +@UtilityClass +public final class CalciteTypeConverter { + + /** + * Converts a SeaTunnel type to the corresponding Calcite RelDataType. + * + * @param factory Calcite type factory + * @param seaTunnelType SeaTunnel data type + * @return the corresponding Calcite type + */ + public static RelDataType toCalciteType( + RelDataTypeFactory factory, SeaTunnelDataType seaTunnelType) { + SqlType sqlType = seaTunnelType.getSqlType(); + switch (sqlType) { + case BOOLEAN: + return factory.createSqlType(SqlTypeName.BOOLEAN); + case TINYINT: + return factory.createSqlType(SqlTypeName.TINYINT); + case SMALLINT: + return factory.createSqlType(SqlTypeName.SMALLINT); + case INT: + return factory.createSqlType(SqlTypeName.INTEGER); + case BIGINT: + return factory.createSqlType(SqlTypeName.BIGINT); + case FLOAT: + return factory.createSqlType(SqlTypeName.REAL); + case DOUBLE: + return factory.createSqlType(SqlTypeName.DOUBLE); + case DECIMAL: + DecimalType decimalType = (DecimalType) seaTunnelType; + return factory.createSqlType( + SqlTypeName.DECIMAL, decimalType.getPrecision(), decimalType.getScale()); + case STRING: + return factory.createSqlType(SqlTypeName.VARCHAR); + case BYTES: + return factory.createSqlType(SqlTypeName.VARBINARY); + case DATE: + return factory.createSqlType(SqlTypeName.DATE); + case TIME: + return factory.createSqlType(SqlTypeName.TIME, 3); + case TIMESTAMP: + return factory.createSqlType(SqlTypeName.TIMESTAMP); + case TIMESTAMP_TZ: + return factory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE); + case NULL: + return factory.createSqlType(SqlTypeName.NULL); + case ARRAY: + ArrayType arrayType = (ArrayType) seaTunnelType; + SeaTunnelDataType elementType = arrayType.getElementType(); + RelDataType calciteElementType = toCalciteType(factory, elementType); + return factory.createArrayType(calciteElementType, -1); + case MAP: + MapType mapType = (MapType) seaTunnelType; + RelDataType keyType = toCalciteType(factory, mapType.getKeyType()); + RelDataType valueType = toCalciteType(factory, mapType.getValueType()); + return factory.createMapType(keyType, valueType); + case ROW: + SeaTunnelRowType rowType = (SeaTunnelRowType) seaTunnelType; + List fieldNames = new ArrayList<>(); + List fieldTypes = new ArrayList<>(); + for (int i = 0; i < rowType.getTotalFields(); i++) { + fieldNames.add(rowType.getFieldName(i)); + fieldTypes.add(toCalciteType(factory, rowType.getFieldType(i))); + } + return factory.createStructType(fieldTypes, fieldNames); + case MULTIPLE_ROW: + return factory.createSqlType(SqlTypeName.ANY); + case BINARY_VECTOR: + case FLOAT_VECTOR: + case FLOAT16_VECTOR: + case BFLOAT16_VECTOR: + case SPARSE_FLOAT_VECTOR: + return factory.createSqlType(SqlTypeName.VARBINARY); + default: + throw new TransformException( + TransformCommonErrorCode.EXPRESSION_EXECUTE_ERROR, + "Unsupported SeaTunnel type for Calcite mapping: " + sqlType); + } + } + + /** + * Converts a Calcite RelDataType back to the corresponding SeaTunnel type. + * + * @param calciteType Calcite type + * @return the corresponding SeaTunnel data type + */ + public static SeaTunnelDataType toSeaTunnelType(RelDataType calciteType) { + SqlTypeName typeName = calciteType.getSqlTypeName(); + switch (typeName) { + case BOOLEAN: + return BasicType.BOOLEAN_TYPE; + case TINYINT: + return BasicType.BYTE_TYPE; + case SMALLINT: + return BasicType.SHORT_TYPE; + case INTEGER: + return BasicType.INT_TYPE; + case BIGINT: + return BasicType.LONG_TYPE; + case REAL: + case FLOAT: + return BasicType.FLOAT_TYPE; + case DOUBLE: + return BasicType.DOUBLE_TYPE; + case DECIMAL: + return new DecimalType(calciteType.getPrecision(), calciteType.getScale()); + case CHAR: + case VARCHAR: + return BasicType.STRING_TYPE; + case BINARY: + case VARBINARY: + return PrimitiveByteArrayType.INSTANCE; + case DATE: + return LocalTimeType.LOCAL_DATE_TYPE; + case TIME: + case TIME_WITH_LOCAL_TIME_ZONE: + return LocalTimeType.LOCAL_TIME_TYPE; + case TIMESTAMP: + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return LocalTimeType.OFFSET_DATE_TIME_TYPE; + case NULL: + return BasicType.VOID_TYPE; + case ANY: + return BasicType.STRING_TYPE; + case ARRAY: + case MULTISET: + RelDataType componentType = calciteType.getComponentType(); + if (componentType == null) { + return ArrayType.STRING_ARRAY_TYPE; + } + SeaTunnelDataType stElementType = toSeaTunnelType(componentType); + return convertToArrayType(stElementType); + case MAP: + RelDataType keyRelType = calciteType.getKeyType(); + RelDataType valueRelType = calciteType.getValueType(); + if (keyRelType == null || valueRelType == null) { + return new MapType<>(BasicType.STRING_TYPE, BasicType.STRING_TYPE); + } + return new MapType<>(toSeaTunnelType(keyRelType), toSeaTunnelType(valueRelType)); + case ROW: + return convertStructToRowType(calciteType); + case INTERVAL_YEAR: + case INTERVAL_YEAR_MONTH: + case INTERVAL_MONTH: + case INTERVAL_DAY: + case INTERVAL_DAY_HOUR: + case INTERVAL_DAY_MINUTE: + case INTERVAL_DAY_SECOND: + case INTERVAL_HOUR: + case INTERVAL_HOUR_MINUTE: + case INTERVAL_HOUR_SECOND: + case INTERVAL_MINUTE: + case INTERVAL_MINUTE_SECOND: + case INTERVAL_SECOND: + return BasicType.LONG_TYPE; + default: + if (calciteType.isStruct()) { + return convertStructToRowType(calciteType); + } + throw new TransformException( + TransformCommonErrorCode.EXPRESSION_EXECUTE_ERROR, + "Unsupported Calcite type for SeaTunnel mapping: " + typeName); + } + } + + private static SeaTunnelRowType convertStructToRowType(RelDataType structType) { + List names = new ArrayList<>(); + List> types = new ArrayList<>(); + structType + .getFieldList() + .forEach( + field -> { + names.add(field.getName()); + types.add(toSeaTunnelType(field.getType())); + }); + return new SeaTunnelRowType( + names.toArray(new String[0]), types.toArray(new SeaTunnelDataType[0])); + } + + private static ArrayType convertToArrayType(SeaTunnelDataType elementType) { + SqlType sqlType = elementType.getSqlType(); + switch (sqlType) { + case STRING: + return ArrayType.STRING_ARRAY_TYPE; + case BOOLEAN: + return ArrayType.BOOLEAN_ARRAY_TYPE; + case TINYINT: + return ArrayType.BYTE_ARRAY_TYPE; + case SMALLINT: + return ArrayType.SHORT_ARRAY_TYPE; + case INT: + return ArrayType.INT_ARRAY_TYPE; + case BIGINT: + return ArrayType.LONG_ARRAY_TYPE; + case FLOAT: + return ArrayType.FLOAT_ARRAY_TYPE; + case DOUBLE: + return ArrayType.DOUBLE_ARRAY_TYPE; + case MAP: + MapType mapType = (MapType) elementType; + return new ArrayType<>(MapType.class, mapType); + case ARRAY: + ArrayType arrayType = (ArrayType) elementType; + return ArrayType.of(arrayType); + case DECIMAL: + case DATE: + case TIME: + case TIMESTAMP: + case TIMESTAMP_TZ: + case ROW: + case BYTES: + return new ArrayType<>(elementType.getTypeClass(), elementType); + default: + return ArrayType.STRING_ARRAY_TYPE; + } + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/CalciteValueConverter.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/CalciteValueConverter.java new file mode 100644 index 000000000000..f7610afa1f3b --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/CalciteValueConverter.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.type; + +import org.apache.seatunnel.shade.org.apache.calcite.avatica.util.ByteString; + +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; + +import lombok.experimental.UtilityClass; + +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; + +/** Bidirectional value converter between SeaTunnel runtime values and Calcite runtime values. */ +@UtilityClass +public final class CalciteValueConverter { + + /** + * Converts a SeaTunnel runtime value into the representation Calcite expects when scanning a + * row. {@code null} is returned as-is. + */ + public static Object toCalcite(Object value) { + if (value == null) { + return null; + } + if (value instanceof LocalDate) { + return Date.valueOf((LocalDate) value); + } + if (value instanceof LocalTime) { + return (int) (((LocalTime) value).toNanoOfDay() / 1_000_000L); + } + if (value instanceof LocalDateTime) { + return Timestamp.valueOf((LocalDateTime) value); + } + if (value instanceof OffsetDateTime) { + return Timestamp.from(((OffsetDateTime) value).toInstant()); + } + if (value instanceof ByteString) { + return value; + } + if (value instanceof byte[]) { + return new ByteString((byte[]) value); + } + if (value instanceof ByteBuffer) { + ByteBuffer buf = (ByteBuffer) value; + byte[] bytes = new byte[buf.remaining()]; + buf.duplicate().get(bytes); + return new ByteString(bytes); + } + return value; + } + + /** + * Converts a Calcite runtime value back to the SeaTunnel representation expected for {@code + * targetType}. {@code null} is returned as-is. + */ + public static Object fromCalcite(Object value, SeaTunnelDataType targetType) { + if (value == null) { + return null; + } + if (value instanceof ByteString) { + return convertBinary(((ByteString) value).getBytes(), targetType); + } + if (value instanceof byte[]) { + return convertBinary((byte[]) value, targetType); + } + switch (targetType.getSqlType()) { + case DATE: + if (value instanceof Date) { + return ((Date) value).toLocalDate(); + } + if (value instanceof Number) { + return LocalDate.ofEpochDay(((Number) value).longValue()); + } + return value; + case TIME: + if (value instanceof Number) { + long millis = ((Number) value).longValue(); + return LocalTime.ofNanoOfDay(millis * 1_000_000L); + } + if (value instanceof Time) { + return ((Time) value).toLocalTime(); + } + return value; + case TIMESTAMP: + if (value instanceof Timestamp) { + return ((Timestamp) value).toLocalDateTime(); + } + if (value instanceof Number) { + return new Timestamp(((Number) value).longValue()).toLocalDateTime(); + } + return value; + case TIMESTAMP_TZ: + if (value instanceof Timestamp) { + return ((Timestamp) value).toInstant().atOffset(ZoneOffset.UTC); + } + if (value instanceof Number) { + return Instant.ofEpochMilli(((Number) value).longValue()) + .atOffset(ZoneOffset.UTC); + } + return value; + case TINYINT: + if (value instanceof Number) { + return ((Number) value).byteValue(); + } + return value; + case SMALLINT: + if (value instanceof Number) { + return ((Number) value).shortValue(); + } + return value; + case INT: + if (value instanceof Number) { + return ((Number) value).intValue(); + } + return value; + case BIGINT: + if (value instanceof Number) { + return ((Number) value).longValue(); + } + return value; + case FLOAT: + if (value instanceof Number) { + return ((Number) value).floatValue(); + } + return value; + case DOUBLE: + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + return value; + default: + return value; + } + } + + private static Object convertBinary(byte[] value, SeaTunnelDataType targetType) { + switch (targetType.getSqlType()) { + case BYTES: + return value; + case BINARY_VECTOR: + case FLOAT_VECTOR: + case FLOAT16_VECTOR: + case BFLOAT16_VECTOR: + case SPARSE_FLOAT_VECTOR: + return ByteBuffer.wrap(value); + default: + return value; + } + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/OutputRowTypeDeriver.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/OutputRowTypeDeriver.java new file mode 100644 index 000000000000..2a75f2fe9685 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/type/OutputRowTypeDeriver.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.type; + +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataType; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.seatunnel.shade.org.apache.calcite.sql.SqlCall; +import org.apache.seatunnel.shade.org.apache.calcite.sql.SqlIdentifier; +import org.apache.seatunnel.shade.org.apache.calcite.sql.SqlKind; +import org.apache.seatunnel.shade.org.apache.calcite.sql.SqlNode; +import org.apache.seatunnel.shade.org.apache.calcite.sql.SqlNodeList; +import org.apache.seatunnel.shade.org.apache.calcite.sql.SqlSelect; + +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; + +import java.util.List; + +/** Derives the SeaTunnel output row type for a validated SQL statement. */ +public final class OutputRowTypeDeriver { + + private final SeaTunnelRowType inputRowType; + + public OutputRowTypeDeriver(SeaTunnelRowType inputRowType) { + this.inputRowType = inputRowType; + } + + /** + * Builds the output row type using both the validated row type from Calcite and the parsed + * select list. The select list is used as a fallback when Calcite's type collapses lossier + * representations such as {@code VARBINARY}. + */ + public SeaTunnelRowType derive(SqlNode validated, RelDataType validatedRowType) { + List fields = validatedRowType.getFieldList(); + String[] names = new String[fields.size()]; + SeaTunnelDataType[] types = new SeaTunnelDataType[fields.size()]; + SqlSelect select = validated instanceof SqlSelect ? (SqlSelect) validated : null; + SqlNodeList selectList = select == null ? null : select.getSelectList(); + boolean aligned = selectList != null && selectList.size() == fields.size(); + for (int i = 0; i < fields.size(); i++) { + names[i] = fields.get(i).getName(); + SeaTunnelDataType defaultType = + CalciteTypeConverter.toSeaTunnelType(fields.get(i).getType()); + if (aligned) { + types[i] = inferProjectedType(selectList.get(i), defaultType); + } else { + SeaTunnelDataType inputFieldType = findInputFieldType(names[i]); + types[i] = inputFieldType != null ? inputFieldType : defaultType; + } + } + return new SeaTunnelRowType(names, types); + } + + private SeaTunnelDataType inferProjectedType( + SqlNode selectItem, SeaTunnelDataType defaultType) { + SqlNode node = unwrapAlias(selectItem); + if (node instanceof SqlIdentifier) { + SqlIdentifier identifier = (SqlIdentifier) node; + if (identifier.isStar() && inputRowType.getTotalFields() == 1) { + return inputRowType.getFieldType(0); + } + SeaTunnelDataType inputFieldType = + findInputFieldType(identifier.names.get(identifier.names.size() - 1)); + return inputFieldType != null ? inputFieldType : defaultType; + } + if (node instanceof SqlCall) { + SqlCall call = (SqlCall) node; + if (isVectorReturningFunction(call) && !call.getOperandList().isEmpty()) { + SeaTunnelDataType operandType = + inferProjectedType(call.getOperandList().get(0), defaultType); + if (isVectorType(operandType)) { + return operandType; + } + } + } + return defaultType; + } + + private SqlNode unwrapAlias(SqlNode node) { + if (node.getKind() == SqlKind.AS && node instanceof SqlCall) { + return ((SqlCall) node).operand(0); + } + return node; + } + + private SeaTunnelDataType findInputFieldType(String fieldName) { + for (int i = 0; i < inputRowType.getTotalFields(); i++) { + if (inputRowType.getFieldName(i).equalsIgnoreCase(fieldName)) { + return inputRowType.getFieldType(i); + } + } + return null; + } + + private static boolean isVectorReturningFunction(SqlCall call) { + String functionName = call.getOperator().getName(); + return "VECTOR_NORMALIZE".equalsIgnoreCase(functionName) + || "VECTOR_REDUCE".equalsIgnoreCase(functionName); + } + + private static boolean isVectorType(SeaTunnelDataType type) { + if (type == null) { + return false; + } + switch (type.getSqlType()) { + case BINARY_VECTOR: + case FLOAT_VECTOR: + case FLOAT16_VECTOR: + case BFLOAT16_VECTOR: + case SPARSE_FLOAT_VECTOR: + return true; + default: + return false; + } + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BinaryAwareScalarFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BinaryAwareScalarFunction.java new file mode 100644 index 000000000000..11be3572d555 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BinaryAwareScalarFunction.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.shade.org.apache.calcite.adapter.enumerable.CallImplementor; +import org.apache.seatunnel.shade.org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.seatunnel.shade.org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.seatunnel.shade.org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.seatunnel.shade.org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.seatunnel.shade.org.apache.calcite.avatica.util.ByteString; +import org.apache.seatunnel.shade.org.apache.calcite.linq4j.tree.Expression; +import org.apache.seatunnel.shade.org.apache.calcite.linq4j.tree.Expressions; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataType; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.seatunnel.shade.org.apache.calcite.rex.RexCall; +import org.apache.seatunnel.shade.org.apache.calcite.schema.ImplementableFunction; +import org.apache.seatunnel.shade.org.apache.calcite.schema.ScalarFunction; +import org.apache.seatunnel.shade.org.apache.calcite.schema.impl.ReflectiveFunctionBase; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +/** Bridges {@code byte[]} eval parameters and return values with Calcite's {@link ByteString}. */ +final class BinaryAwareScalarFunction extends ReflectiveFunctionBase + implements ScalarFunction, ImplementableFunction { + + private final CallImplementor implementor; + + BinaryAwareScalarFunction(Method method) { + super(method); + this.implementor = + RexImpTable.createImplementor( + new BinaryAwareNotNullImplementor(method), NullPolicy.STRICT, false); + } + + @Override + public RelDataType getReturnType(RelDataTypeFactory typeFactory) { + return typeFactory.createJavaType(method.getReturnType()); + } + + @Override + public CallImplementor getImplementor() { + return implementor; + } + + /** + * Detects whether the given method's signature involves binary types ({@code byte[]}) that + * require ByteString bridging. + */ + static boolean requiresBinaryBridging(Method method) { + if (method.getReturnType() == byte[].class) { + return true; + } + for (Class paramType : method.getParameterTypes()) { + if (paramType == byte[].class) { + return true; + } + } + return false; + } + + private static final class BinaryAwareNotNullImplementor implements NotNullImplementor { + + private final Method method; + + BinaryAwareNotNullImplementor(Method method) { + this.method = method; + } + + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + Class[] paramTypes = method.getParameterTypes(); + List args = new ArrayList<>(translatedOperands.size()); + for (int i = 0; i < translatedOperands.size(); i++) { + Expression operand = translatedOperands.get(i); + if (paramTypes[i] == byte[].class) { + operand = Expressions.call(operand, "getBytes"); + } + args.add(operand); + } + Expression result = Expressions.call(method, args); + if (method.getReturnType() == byte[].class) { + result = Expressions.new_(ByteString.class, result); + } + return result; + } + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BuiltinFunctions.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BuiltinFunctions.java new file mode 100644 index 000000000000..d88eb7e72da5 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BuiltinFunctions.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.shade.org.apache.calcite.schema.ScalarFunction; +import org.apache.seatunnel.shade.org.apache.calcite.schema.SchemaPlus; +import org.apache.seatunnel.shade.org.apache.calcite.schema.impl.ScalarFunctionImpl; + +import lombok.extern.slf4j.Slf4j; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.ServiceLoader; + +/** Discovers {@link CalciteUdf} implementations via {@link ServiceLoader} and registers them. */ +@Slf4j +public final class BuiltinFunctions { + + private final List loadedUdfs = new ArrayList<>(); + + /** + * Discovers all {@link CalciteUdf} implementations from the classpath, validates them, and + * registers their static {@code eval} methods into the given Calcite schema. + */ + public void discoverAndRegister(SchemaPlus schema) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + ServiceLoader.load(CalciteUdf.class, cl) + .forEach( + udf -> { + String rawName = udf.functionName(); + if (rawName == null || rawName.isEmpty()) { + log.warn( + "Skipping Calcite UDF with null/empty functionName: {}", + udf.getClass().getName()); + return; + } + String name = rawName.toUpperCase(); + boolean opened = false; + try { + Method evalMethod = findStaticEvalMethod(udf.getClass()); + ScalarFunction function; + if (evalMethod != null + && BinaryAwareScalarFunction.requiresBinaryBridging( + evalMethod)) { + function = new BinaryAwareScalarFunction(evalMethod); + } else { + function = ScalarFunctionImpl.create(udf.getClass(), "eval"); + } + if (function == null) { + log.warn( + "No valid static eval method found in Calcite UDF: {}", + name); + return; + } + udf.open(); + opened = true; + schema.add(name, function); + loadedUdfs.add(udf); + log.info("Registered Calcite UDF via SPI: {}", name); + } catch (Exception e) { + log.warn("Failed to register Calcite UDF: {}", name, e); + if (opened) { + try { + udf.close(); + } catch (Exception ce) { + log.warn("Failed to close Calcite UDF: {}", name, ce); + } + } + } + }); + } + + public void close() { + for (CalciteUdf udf : loadedUdfs) { + try { + udf.close(); + } catch (Exception e) { + log.warn("Failed to close Calcite UDF: {}", udf.functionName(), e); + } + } + loadedUdfs.clear(); + } + + /** Finds the {@code public static eval} method declared on the UDF class, if any. */ + private static Method findStaticEvalMethod(Class clazz) { + for (Method method : clazz.getMethods()) { + if ("eval".equals(method.getName()) + && Modifier.isStatic(method.getModifiers()) + && Modifier.isPublic(method.getModifiers())) { + return method; + } + } + return null; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CalciteUdf.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CalciteUdf.java new file mode 100644 index 000000000000..8a9948b39517 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CalciteUdf.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +/** + * SPI for Calcite SQL transform UDFs. Implementations must provide a {@code public static eval} + * method whose signature determines the SQL function's input/output types; for binary/vector data + * declare {@code byte[]} and the framework bridges Calcite's {@code ByteString} automatically. + * + *

Annotate the implementation with {@code @AutoService(CalciteUdf.class)} and ship the jar in + * {@code ${SEATUNNEL_HOME}/lib/} for SPI discovery. + */ +public interface CalciteUdf extends AutoCloseable { + + /** SQL function name used in queries, e.g. "MASK", "DES_ENCRYPT". */ + String functionName(); + + /** Open UDF resources. Called once before first eval. */ + default void open() {} + + @Override + default void close() throws Exception {} +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CalciteUdfContext.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CalciteUdfContext.java new file mode 100644 index 000000000000..b54421f8b8a5 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CalciteUdfContext.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.type.RowKind; + +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nullable; + +import java.util.Objects; + +/** + * Thread-local row context exposing per-row metadata (table id, {@link RowKind}) to Calcite UDFs. + * + *

UDFs read it via {@link #current()}; the engine opens a {@link Scope} around each row. + */ +@Slf4j +public final class CalciteUdfContext { + + private static final ThreadLocal HOLDER = new ThreadLocal<>(); + + private @Nullable String rawTableId; + private boolean tablePathResolved; + private @Nullable String database; + private @Nullable String schema; + private @Nullable String table; + private @Nullable RowKind rowKind; + + private CalciteUdfContext() {} + + /** + * Returns the context for the row currently being processed by the Calcite engine. Returns + * {@code null} when called outside of a UDF execution scope. + */ + @Nullable public static CalciteUdfContext current() { + return HOLDER.get(); + } + + /** + * Opens a UDF execution scope bound to the current thread, populated with the given row + * metadata. The returned handle MUST be closed (preferably via try-with-resources) so the + * thread-local context is properly torn down. + * + *

If a scope already exists on this thread (e.g. nested SQL transforms), the existing + * context instance is reused and its metadata is refreshed; nested scopes do not tear down + * their parent. + */ + public static Scope enter(@Nullable String tableId, @Nullable RowKind rowKind) { + CalciteUdfContext existing = HOLDER.get(); + if (existing != null) { + existing.update(tableId, rowKind); + return Scope.NOOP; + } + CalciteUdfContext fresh = new CalciteUdfContext(); + fresh.update(tableId, rowKind); + HOLDER.set(fresh); + return HOLDER::remove; + } + + private void update(@Nullable String tableId, @Nullable RowKind rowKind) { + this.rowKind = rowKind; + updateTableId(tableId); + } + + private void updateTableId(@Nullable String tableId) { + if (Objects.equals(this.rawTableId, tableId)) { + return; + } + this.rawTableId = tableId; + this.database = null; + this.schema = null; + this.table = null; + this.tablePathResolved = false; + } + + private void resolveTablePathIfNeeded() { + if (tablePathResolved) { + return; + } + tablePathResolved = true; + if (rawTableId == null) { + return; + } + try { + TablePath tablePath = TablePath.of(rawTableId); + this.database = tablePath.getDatabaseName(); + this.schema = tablePath.getSchemaName(); + this.table = tablePath.getTableName(); + } catch (IllegalArgumentException e) { + log.warn( + "Failed to parse tableId '{}' as TablePath, " + + "getDatabase()/getSchema()/getTable() will return null", + rawTableId, + e); + } + } + + @Nullable public String getRawTableId() { + return rawTableId; + } + + @Nullable public String getDatabase() { + resolveTablePathIfNeeded(); + return database; + } + + @Nullable public String getSchema() { + resolveTablePathIfNeeded(); + return schema; + } + + @Nullable public String getTable() { + resolveTablePathIfNeeded(); + return table; + } + + @Nullable public RowKind getRowKind() { + return rowKind; + } + + /** + * Auto-closeable handle returned by {@link #enter}. Closing it removes the thread-local context + * if this scope owns it. + */ + public interface Scope extends AutoCloseable { + + /** No-op scope used for nested {@link #enter} calls. */ + Scope NOOP = () -> {}; + + @Override + void close(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CosineDistanceFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CosineDistanceFunction.java new file mode 100644 index 000000000000..9f19dabf2b9d --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/CosineDistanceFunction.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import com.google.auto.service.AutoService; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Calculates cosine distance between two vectors. Returns 0 for identical vectors and 1 for + * orthogonal vectors. + * + *

Usage: {@code COSINE_DISTANCE(vector1, vector2)} + */ +@AutoService(CalciteUdf.class) +public class CosineDistanceFunction implements CalciteUdf { + + @Override + public String functionName() { + return "COSINE_DISTANCE"; + } + + public static Double eval(byte[] v1, byte[] v2) { + if (v1 == null || v2 == null) { + return null; + } + Float[] vector1 = VectorUtils.toFloatArray(ByteBuffer.wrap(v1)); + Float[] vector2 = VectorUtils.toFloatArray(ByteBuffer.wrap(v2)); + if (vector1.length != vector2.length) { + throw new IllegalArgumentException( + String.format( + "Vectors must have the same dimension: %d vs %d", + vector1.length, vector2.length)); + } + double dotProduct = + IntStream.range(0, vector1.length).mapToDouble(i -> vector1[i] * vector2[i]).sum(); + double norm1 = Arrays.stream(vector1).mapToDouble(v -> v * v).sum(); + double norm2 = Arrays.stream(vector2).mapToDouble(v -> v * v).sum(); + if (norm1 == 0.0 || norm2 == 0.0) { + return 1.0; + } + double cosineSimilarity = dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2)); + return 1.0 - cosineSimilarity; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/DesDecryptFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/DesDecryptFunction.java new file mode 100644 index 000000000000..7cdd9f363464 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/DesDecryptFunction.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.transform.sql.zeta.functions.udf.DESUtil; + +import com.google.auto.service.AutoService; + +/** + * DES decryption UDF. Password must be at least 8 characters. Input data should be Base64-encoded + * ciphertext. + * + *

Usage: {@code DES_DECRYPT(password, data)} + */ +@AutoService(CalciteUdf.class) +public class DesDecryptFunction implements CalciteUdf { + + @Override + public String functionName() { + return "DES_DECRYPT"; + } + + public static String eval(String password, String data) { + if (password == null || data == null) { + return null; + } + return DESUtil.decrypt(password, data); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/DesEncryptFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/DesEncryptFunction.java new file mode 100644 index 000000000000..b176cf70b72f --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/DesEncryptFunction.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.transform.sql.zeta.functions.udf.DESUtil; + +import com.google.auto.service.AutoService; + +/** + * DES encryption UDF. Password must be at least 8 characters. Returns Base64-encoded ciphertext. + * + *

Usage: {@code DES_ENCRYPT(password, data)} + */ +@AutoService(CalciteUdf.class) +public class DesEncryptFunction implements CalciteUdf { + + @Override + public String functionName() { + return "DES_ENCRYPT"; + } + + public static String eval(String password, String data) { + if (password == null || data == null) { + return null; + } + return DESUtil.encrypt(password, data); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/InnerProductFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/InnerProductFunction.java new file mode 100644 index 000000000000..3481b07dab59 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/InnerProductFunction.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import com.google.auto.service.AutoService; + +import java.nio.ByteBuffer; +import java.util.stream.IntStream; + +/** + * Calculates the inner product (dot product) of two vectors. + * + *

Usage: {@code INNER_PRODUCT(vector1, vector2)} + */ +@AutoService(CalciteUdf.class) +public class InnerProductFunction implements CalciteUdf { + + @Override + public String functionName() { + return "INNER_PRODUCT"; + } + + public static Double eval(byte[] v1, byte[] v2) { + if (v1 == null || v2 == null) { + return null; + } + Float[] vector1 = VectorUtils.toFloatArray(ByteBuffer.wrap(v1)); + Float[] vector2 = VectorUtils.toFloatArray(ByteBuffer.wrap(v2)); + if (vector1.length != vector2.length) { + throw new IllegalArgumentException( + String.format( + "Vectors must have the same dimension: %d vs %d", + vector1.length, vector2.length)); + } + return IntStream.range(0, vector1.length).mapToDouble(i -> vector1[i] * vector2[i]).sum(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/L1DistanceFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/L1DistanceFunction.java new file mode 100644 index 000000000000..cd23ea03f6ad --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/L1DistanceFunction.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import com.google.auto.service.AutoService; + +import java.nio.ByteBuffer; +import java.util.stream.IntStream; + +/** + * Calculates Manhattan (L1) distance between two vectors. + * + *

Usage: {@code L1_DISTANCE(vector1, vector2)} + */ +@AutoService(CalciteUdf.class) +public class L1DistanceFunction implements CalciteUdf { + + @Override + public String functionName() { + return "L1_DISTANCE"; + } + + public static Double eval(byte[] v1, byte[] v2) { + if (v1 == null || v2 == null) { + return null; + } + Float[] vector1 = VectorUtils.toFloatArray(ByteBuffer.wrap(v1)); + Float[] vector2 = VectorUtils.toFloatArray(ByteBuffer.wrap(v2)); + if (vector1.length != vector2.length) { + throw new IllegalArgumentException( + String.format( + "Vectors must have the same dimension: %d vs %d", + vector1.length, vector2.length)); + } + return IntStream.range(0, vector1.length) + .mapToDouble(i -> Math.abs(vector1[i] - vector2[i])) + .sum(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/L2DistanceFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/L2DistanceFunction.java new file mode 100644 index 000000000000..25fa89e46f4f --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/L2DistanceFunction.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import com.google.auto.service.AutoService; + +import java.nio.ByteBuffer; +import java.util.stream.IntStream; + +/** + * Calculates Euclidean (L2) distance between two vectors. + * + *

Usage: {@code L2_DISTANCE(vector1, vector2)} + */ +@AutoService(CalciteUdf.class) +public class L2DistanceFunction implements CalciteUdf { + + @Override + public String functionName() { + return "L2_DISTANCE"; + } + + public static Double eval(byte[] v1, byte[] v2) { + if (v1 == null || v2 == null) { + return null; + } + Float[] vector1 = VectorUtils.toFloatArray(ByteBuffer.wrap(v1)); + Float[] vector2 = VectorUtils.toFloatArray(ByteBuffer.wrap(v2)); + if (vector1.length != vector2.length) { + throw new IllegalArgumentException( + String.format( + "Vectors must have the same dimension: %d vs %d", + vector1.length, vector2.length)); + } + double sum = + IntStream.range(0, vector1.length) + .mapToDouble( + i -> { + double diff = vector1[i] - vector2[i]; + return diff * diff; + }) + .sum(); + return Math.sqrt(sum); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/MaskFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/MaskFunction.java new file mode 100644 index 000000000000..76146bc333de --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/MaskFunction.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import com.google.auto.service.AutoService; + +/** + * Data masking UDF: replaces characters in the specified range with a mask character. + * + *

Usage: {@code MASK(value, start, end, maskChar)} + * + *

Example: {@code MASK('13812345678', 3, 7, '*')} returns {@code '138****5678'} + */ +@AutoService(CalciteUdf.class) +public class MaskFunction implements CalciteUdf { + + @Override + public String functionName() { + return "MASK"; + } + + public static String eval(String value, int start, int end, String maskChar) { + if (value == null) { + return null; + } + if (start < 0 || end > value.length() || start >= end) { + return value; + } + char mask = (maskChar != null && !maskChar.isEmpty()) ? maskChar.charAt(0) : '*'; + char[] chars = value.toCharArray(); + for (int i = start; i < end; i++) { + chars[i] = mask; + } + return new String(chars); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/MaskHashFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/MaskHashFunction.java new file mode 100644 index 000000000000..00b16e903533 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/MaskHashFunction.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import com.google.auto.service.AutoService; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * SHA-256 hash UDF for data masking scenarios. Produces a deterministic hash of the original value + * for downstream comparison / JOIN / GROUP BY. + * + *

Usage: {@code MASK_HASH(value)} + */ +@AutoService(CalciteUdf.class) +public class MaskHashFunction implements CalciteUdf { + + @Override + public String functionName() { + return "MASK_HASH"; + } + + public static String eval(String value) { + if (value == null) { + return null; + } + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder hexString = new StringBuilder(hash.length * 2); + for (byte b : hash) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-256 algorithm not available", e); + } + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorDimsFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorDimsFunction.java new file mode 100644 index 000000000000..86f2da057d8c --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorDimsFunction.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import com.google.auto.service.AutoService; + +import java.nio.ByteBuffer; + +/** + * Returns the number of dimensions (elements) in a vector. + * + *

Usage: {@code VECTOR_DIMS(vector)} + */ +@AutoService(CalciteUdf.class) +public class VectorDimsFunction implements CalciteUdf { + + @Override + public String functionName() { + return "VECTOR_DIMS"; + } + + public static Integer eval(byte[] v) { + if (v == null) { + return null; + } + Float[] vector = VectorUtils.toFloatArray(ByteBuffer.wrap(v)); + return vector.length; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorNormFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorNormFunction.java new file mode 100644 index 000000000000..4288b1839c24 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorNormFunction.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import com.google.auto.service.AutoService; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * Calculates the L2 norm (Euclidean norm) of a vector. + * + *

Usage: {@code VECTOR_NORM(vector)} + */ +@AutoService(CalciteUdf.class) +public class VectorNormFunction implements CalciteUdf { + + @Override + public String functionName() { + return "VECTOR_NORM"; + } + + public static Double eval(byte[] v) { + if (v == null) { + return null; + } + Float[] vector = VectorUtils.toFloatArray(ByteBuffer.wrap(v)); + return Math.sqrt(Arrays.stream(vector).mapToDouble(val -> val * val).sum()); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorNormalizeFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorNormalizeFunction.java new file mode 100644 index 000000000000..9ef0f4702f6e --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorNormalizeFunction.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import com.google.auto.service.AutoService; + +import java.nio.ByteBuffer; + +/** + * Normalizes a vector to unit length (magnitude = 1). + * + *

Usage: {@code VECTOR_NORMALIZE(vector_field)} + */ +@AutoService(CalciteUdf.class) +public class VectorNormalizeFunction implements CalciteUdf { + + @Override + public String functionName() { + return "VECTOR_NORMALIZE"; + } + + public static byte[] eval(byte[] vectorData) { + if (vectorData == null) { + return null; + } + Float[] vector = VectorUtils.toFloatArray(ByteBuffer.wrap(vectorData)); + double magnitude = 0.0; + for (Float value : vector) { + if (value != null) { + magnitude += value * value; + } + } + magnitude = Math.sqrt(magnitude); + + if (magnitude == 0.0) { + return vectorData; + } + + Float[] normalized = new Float[vector.length]; + for (int i = 0; i < vector.length; i++) { + normalized[i] = vector[i] == null ? null : (float) (vector[i] / magnitude); + } + ByteBuffer buf = VectorUtils.toByteBuffer(normalized); + byte[] out = new byte[buf.remaining()]; + buf.get(out); + return out; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorReduceFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorReduceFunction.java new file mode 100644 index 000000000000..6bc2655f6885 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorReduceFunction.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import com.google.auto.service.AutoService; + +import java.nio.ByteBuffer; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Generic vector dimension reduction function. Supports TRUNCATE, RANDOM_PROJECTION, and + * SPARSE_RANDOM_PROJECTION methods. + * + *

Projection matrices are cached per (sourceDimension, targetDimension, method) triple so that + * all rows within the same job use the same matrix, ensuring mathematically consistent results. + * + *

Usage: {@code VECTOR_REDUCE(vector_field, target_dimension, method)} + */ +@AutoService(CalciteUdf.class) +public class VectorReduceFunction implements CalciteUdf { + + private static final long PROJECTION_SEED = 42L; + + private static final ConcurrentMap MATRIX_CACHE = new ConcurrentHashMap<>(); + + @Override + public String functionName() { + return "VECTOR_REDUCE"; + } + + public static byte[] eval(byte[] vectorData, Integer targetDimension, String method) { + if (vectorData == null || targetDimension == null || method == null) { + return null; + } + Float[] source = VectorUtils.toFloatArray(ByteBuffer.wrap(vectorData)); + if (source.length <= targetDimension) { + return vectorData; + } + + Float[] result; + switch (method.toUpperCase()) { + case "TRUNCATE": + result = new Float[targetDimension]; + System.arraycopy(source, 0, result, 0, targetDimension); + break; + case "RANDOM_PROJECTION": + result = + applyProjection( + source, + getOrCreateMatrix("GAUSSIAN", source.length, targetDimension), + targetDimension); + break; + case "SPARSE_RANDOM_PROJECTION": + result = + applyProjection( + source, + getOrCreateMatrix("SPARSE", source.length, targetDimension), + targetDimension); + break; + default: + throw new IllegalArgumentException("Unknown reduction method: " + method); + } + ByteBuffer buf = VectorUtils.toByteBuffer(result); + byte[] out = new byte[buf.remaining()]; + buf.get(out); + return out; + } + + private static float[][] getOrCreateMatrix( + String type, int sourceDimension, int targetDimension) { + String key = type + ":" + sourceDimension + ":" + targetDimension; + return MATRIX_CACHE.computeIfAbsent( + key, + k -> { + Random rng = new Random(PROJECTION_SEED); + if ("GAUSSIAN".equals(type)) { + return createGaussianProjectionMatrix( + rng, sourceDimension, targetDimension); + } else { + return createSparseProjectionMatrix(rng, sourceDimension, targetDimension); + } + }); + } + + private static Float[] applyProjection( + Float[] sourceVector, float[][] projectionMatrix, int targetDimension) { + Float[] result = new Float[targetDimension]; + for (int i = 0; i < targetDimension; i++) { + float sum = 0.0f; + for (int j = 0; j < sourceVector.length; j++) { + if (projectionMatrix[i][j] != 0 && sourceVector[j] != null) { + sum += sourceVector[j] * projectionMatrix[i][j]; + } + } + result[i] = sum; + } + return result; + } + + private static float[][] createGaussianProjectionMatrix( + Random rng, int sourceDimension, int targetDimension) { + float[][] matrix = new float[targetDimension][sourceDimension]; + float scale = (float) Math.sqrt(1.0 / targetDimension); + for (int i = 0; i < targetDimension; i++) { + for (int j = 0; j < sourceDimension; j++) { + matrix[i][j] = (float) rng.nextGaussian() * scale; + } + } + return matrix; + } + + private static float[][] createSparseProjectionMatrix( + Random rng, int sourceDimension, int targetDimension) { + float[][] matrix = new float[targetDimension][sourceDimension]; + float scale = (float) Math.sqrt(3.0); + double p1 = 1.0 / 6.0; + double p2 = 2.0 / 6.0; + for (int i = 0; i < targetDimension; i++) { + for (int j = 0; j < sourceDimension; j++) { + double rand = rng.nextDouble(); + if (rand < p1) { + matrix[i][j] = scale; + } else if (rand < p2) { + matrix[i][j] = -scale; + } else { + matrix[i][j] = 0; + } + } + } + return matrix; + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteSQLEngineTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteSQLEngineTest.java new file mode 100644 index 000000000000..d9245aa327d3 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteSQLEngineTest.java @@ -0,0 +1,4252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite; + +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.RowKind; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; +import org.apache.seatunnel.api.table.type.VectorType; +import org.apache.seatunnel.common.utils.VectorUtils; +import org.apache.seatunnel.transform.calcite.engine.CalciteSQLEngine; +import org.apache.seatunnel.transform.exception.TransformException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class CalciteSQLEngineTest { + + private SeaTunnelRowType buildRowType() { + return new SeaTunnelRowType( + new String[] {"id", "name", "age"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, BasicType.STRING_TYPE, BasicType.INT_TYPE + }); + } + + private SeaTunnelRowType numericRowType() { + return new SeaTunnelRowType( + new String[] {"a", "b"}, + new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.INT_TYPE}); + } + + private CalciteSQLEngine createAndInit(String sql, String table, SeaTunnelRowType rowType) { + CalciteSQLEngine engine = new CalciteSQLEngine(sql, table, rowType); + engine.init(); + return engine; + } + + private Object singleField(CalciteSQLEngine engine, Object[] fields) { + return engine.execute(new SeaTunnelRow(fields)).get(0).getField(0); + } + + private List exec(CalciteSQLEngine engine, Object[] fields) { + return engine.execute(new SeaTunnelRow(fields)); + } + + @Test + void testSelectWhere() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, name FROM test_table WHERE age > 20", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {1, "Alice", 25}); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(1, result.get(0).getField(0)); + Assertions.assertEquals("Alice", result.get(0).getField(1)); + + Assertions.assertTrue(exec(engine, new Object[] {2, "Bob", 18}).isEmpty()); + engine.close(); + } + + @Test + void testSelectStar() { + CalciteSQLEngine engine = + createAndInit("SELECT * FROM test_table", "test_table", buildRowType()); + + List result = exec(engine, new Object[] {1, "Alice", 25}); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(1, result.get(0).getField(0)); + Assertions.assertEquals("Alice", result.get(0).getField(1)); + Assertions.assertEquals(25, result.get(0).getField(2)); + Assertions.assertEquals(3, engine.getOutputRowType().getTotalFields()); + engine.close(); + } + + @Test + void testSelectWithFunctions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, UPPER(name) AS name_upper, age + 1 AS next_age FROM test_table", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {1, "alice", 30}); + Assertions.assertEquals("ALICE", result.get(0).getField(1)); + Assertions.assertEquals(31, result.get(0).getField(2)); + engine.close(); + } + + @Test + void testStringFunctions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT LOWER(name) AS lower_name, " + + "CHAR_LENGTH(name) AS name_len, " + + "SUBSTRING(name, 1, 3) AS name_sub " + + "FROM test_table", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {1, "ALICE", 25}); + Assertions.assertEquals("alice", result.get(0).getField(0)); + Assertions.assertEquals(5, result.get(0).getField(1)); + Assertions.assertEquals("ALI", result.get(0).getField(2)); + engine.close(); + } + + @Test + void testConcatOperator() { + CalciteSQLEngine engine = + createAndInit( + "SELECT name || '-' || CAST(age AS VARCHAR) AS combined FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("Alice-25", singleField(engine, new Object[] {1, "Alice", 25})); + engine.close(); + } + + @Test + void testTrimFunction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT TRIM(name) AS trimmed FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals( + "seatunnel", singleField(engine, new Object[] {1, " seatunnel ", 25})); + engine.close(); + } + + @Test + void testTrimLeading() { + CalciteSQLEngine engine = + createAndInit( + "SELECT TRIM(LEADING ' ' FROM name) AS trimmed FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals( + "seatunnel ", singleField(engine, new Object[] {1, " seatunnel ", 25})); + engine.close(); + } + + @Test + void testTrimTrailing() { + CalciteSQLEngine engine = + createAndInit( + "SELECT TRIM(TRAILING ' ' FROM name) AS trimmed FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals( + " seatunnel", singleField(engine, new Object[] {1, " seatunnel ", 25})); + engine.close(); + } + + @Test + void testReplaceFunction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT REPLACE(name, 'sea', 'lake') AS replaced FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals( + "laketunnel", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testPositionFunction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT POSITION('tunnel' IN name) AS pos FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(4, singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testPositionNotFound() { + CalciteSQLEngine engine = + createAndInit( + "SELECT POSITION('xyz' IN name) AS pos FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(0, singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testOverlayFunction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT OVERLAY(name PLACING 'ZETA' FROM 4 FOR 4) AS res FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals( + "seaZETAel", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testSubstringFrom() { + CalciteSQLEngine engine = + createAndInit( + "SELECT SUBSTRING(name FROM 4) AS sub FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("tunnel", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testInitcap() { + CalciteSQLEngine engine = + createAndInit( + "SELECT INITCAP(name) AS res FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals( + "Sea Tunnel", singleField(engine, new Object[] {1, "sea tunnel", 25})); + engine.close(); + } + + @Test + void testCharLengthEmptyString() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CHAR_LENGTH(name) AS len FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(0, singleField(engine, new Object[] {1, "", 25})); + engine.close(); + } + + @Test + void testArithmeticExpressions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT a + b AS sum_val, a * b AS product, a - b AS diff FROM t", + "t", + numericRowType()); + + List result = exec(engine, new Object[] {10, 3}); + Assertions.assertEquals(13, result.get(0).getField(0)); + Assertions.assertEquals(30, result.get(0).getField(1)); + Assertions.assertEquals(7, result.get(0).getField(2)); + engine.close(); + } + + @Test + void testDivision() { + CalciteSQLEngine engine = + createAndInit("SELECT a / b AS quotient FROM t", "t", numericRowType()); + + Assertions.assertEquals(3, singleField(engine, new Object[] {10, 3})); + engine.close(); + } + + @Test + void testModulo() { + CalciteSQLEngine engine = + createAndInit("SELECT MOD(a, b) AS remainder FROM t", "t", numericRowType()); + + Assertions.assertEquals(1, singleField(engine, new Object[] {10, 3})); + engine.close(); + } + + @Test + void testModuloZeroResult() { + CalciteSQLEngine engine = + createAndInit("SELECT MOD(a, b) AS remainder FROM t", "t", numericRowType()); + + Assertions.assertEquals(0, singleField(engine, new Object[] {9, 3})); + engine.close(); + } + + @Test + void testAbsPositive() { + CalciteSQLEngine engine = + createAndInit("SELECT ABS(a) AS abs_val FROM t", "t", numericRowType()); + + Assertions.assertEquals(5, singleField(engine, new Object[] {5, 0})); + engine.close(); + } + + @Test + void testAbsNegative() { + CalciteSQLEngine engine = + createAndInit("SELECT ABS(a) AS abs_val FROM t", "t", numericRowType()); + + Assertions.assertEquals(5, singleField(engine, new Object[] {-5, 0})); + engine.close(); + } + + @Test + void testAbsZero() { + CalciteSQLEngine engine = + createAndInit("SELECT ABS(a) AS abs_val FROM t", "t", numericRowType()); + + Assertions.assertEquals(0, singleField(engine, new Object[] {0, 0})); + engine.close(); + } + + @Test + void testCeilFunction() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT CEIL(val) AS res FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {3.2}); + Assertions.assertEquals(4.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testFloorFunction() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT FLOOR(val) AS res FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {3.8}); + Assertions.assertEquals(3.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testPowerFunction() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"base_val", "exponent"}, + new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE, BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT POWER(base_val, exponent) AS res FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {2.0, 10.0}); + Assertions.assertEquals(1024.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testSqrtFunction() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT SQRT(val) AS res FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {144.0}); + Assertions.assertEquals(12.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testLnFunction() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT LN(val) AS res FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {Math.E}); + Assertions.assertEquals(1.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testLog10Function() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT LOG10(val) AS res FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {1000.0}); + Assertions.assertEquals(3.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testExpFunction() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT EXP(val) AS res FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {0.0}); + Assertions.assertEquals(1.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testNegativeArithmetic() { + CalciteSQLEngine engine = + createAndInit("SELECT a * b AS product FROM t", "t", numericRowType()); + + Assertions.assertEquals(-15, singleField(engine, new Object[] {-3, 5})); + engine.close(); + } + + @Test + void testArithmeticWithZero() { + CalciteSQLEngine engine = + createAndInit("SELECT a + b AS sum_val FROM t", "t", numericRowType()); + + Assertions.assertEquals(0, singleField(engine, new Object[] {0, 0})); + engine.close(); + } + + @Test + void testIntegerOverflow() { + CalciteSQLEngine engine = + createAndInit("SELECT a + b AS sum_val FROM t", "t", numericRowType()); + // int overflow wraps around + Object result = singleField(engine, new Object[] {Integer.MAX_VALUE, 1}); + Assertions.assertNotNull(result); + engine.close(); + } + + @Test + void testCaseWhen() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, CASE WHEN age >= 18 THEN 'adult' ELSE 'minor' END AS category " + + "FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals( + "adult", exec(engine, new Object[] {1, "Alice", 25}).get(0).getField(1)); + Assertions.assertEquals( + "minor", exec(engine, new Object[] {2, "Bob", 10}).get(0).getField(1)); + engine.close(); + } + + @Test + void testCaseWhenMultipleBranches() { + CalciteSQLEngine engine = + createAndInit( + "SELECT TRIM(CASE " + + "WHEN age < 13 THEN 'child' " + + "WHEN age < 18 THEN 'teen' " + + "WHEN age < 65 THEN 'adult' " + + "ELSE 'senior' END) AS group_name " + + "FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("child", singleField(engine, new Object[] {1, "A", 8})); + Assertions.assertEquals("teen", singleField(engine, new Object[] {2, "B", 15})); + Assertions.assertEquals("adult", singleField(engine, new Object[] {3, "C", 30})); + Assertions.assertEquals("senior", singleField(engine, new Object[] {4, "D", 70})); + engine.close(); + } + + @Test + void testCaseWhenWithNullCondition() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CASE WHEN name IS NULL THEN 'no-name' ELSE name END AS safe " + + "FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("no-name", singleField(engine, new Object[] {1, null, 25})); + Assertions.assertEquals("Alice", singleField(engine, new Object[] {2, "Alice", 25})); + engine.close(); + } + + @Test + void testNestedCaseWhen() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CASE WHEN age > 18 THEN " + + "CASE WHEN name IS NOT NULL THEN name ELSE 'anon' END " + + "ELSE 'minor' END AS res FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("Alice", singleField(engine, new Object[] {1, "Alice", 25})); + Assertions.assertEquals("minor", singleField(engine, new Object[] {2, "Bob", 10})); + engine.close(); + } + + @Test + void testCaseWhenNoElse() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CASE WHEN age > 50 THEN 'old' END AS label FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertNull(singleField(engine, new Object[] {1, "A", 25})); + Assertions.assertEquals("old", singleField(engine, new Object[] {2, "B", 60})); + engine.close(); + } + + @Test + void testSimpleCaseExpression() { + CalciteSQLEngine engine = + createAndInit( + "SELECT TRIM(CASE age WHEN 10 THEN 'ten' WHEN 20 THEN 'twenty' " + + "ELSE 'other' END) AS label FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("ten", singleField(engine, new Object[] {1, "A", 10})); + Assertions.assertEquals("twenty", singleField(engine, new Object[] {2, "B", 20})); + Assertions.assertEquals("other", singleField(engine, new Object[] {3, "C", 30})); + engine.close(); + } + + @Test + void testNullValuePassthrough() { + CalciteSQLEngine engine = + createAndInit("SELECT id, name FROM test_table", "test_table", buildRowType()); + + List result = exec(engine, new Object[] {1, null, 25}); + Assertions.assertNull(result.get(0).getField(1)); + engine.close(); + } + + @Test + void testCoalesce() { + CalciteSQLEngine engine = + createAndInit( + "SELECT COALESCE(name, 'unknown') AS safe_name FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("unknown", singleField(engine, new Object[] {1, null, 25})); + Assertions.assertEquals("Alice", singleField(engine, new Object[] {2, "Alice", 25})); + engine.close(); + } + + @Test + void testCoalesceMultipleArgs() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"a", "b", "c"}, + new SeaTunnelDataType[] { + BasicType.STRING_TYPE, BasicType.STRING_TYPE, BasicType.STRING_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT COALESCE(a, b, c) AS res FROM t", "t", rt); + + Assertions.assertEquals( + "first", singleField(engine, new Object[] {"first", "second", "third"})); + Assertions.assertEquals( + "second", singleField(engine, new Object[] {null, "second", "third"})); + Assertions.assertEquals("third", singleField(engine, new Object[] {null, null, "third"})); + Assertions.assertNull(singleField(engine, new Object[] {null, null, null})); + engine.close(); + } + + @Test + void testNullif() { + CalciteSQLEngine engine = + createAndInit( + "SELECT NULLIF(name, 'N/A') AS res FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertNull(singleField(engine, new Object[] {1, "N/A", 25})); + Assertions.assertEquals("Alice", singleField(engine, new Object[] {2, "Alice", 25})); + engine.close(); + } + + @Test + void testIsNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT name IS NULL AS is_null FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(true, singleField(engine, new Object[] {1, null, 25})); + Assertions.assertEquals(false, singleField(engine, new Object[] {2, "Alice", 25})); + engine.close(); + } + + @Test + void testIsNotNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT name IS NOT NULL AS has_name FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(false, singleField(engine, new Object[] {1, null, 25})); + Assertions.assertEquals(true, singleField(engine, new Object[] {2, "Alice", 25})); + engine.close(); + } + + @Test + void testAllColumnsNull() { + CalciteSQLEngine engine = + createAndInit("SELECT id, name, age FROM test_table", "test_table", buildRowType()); + + List result = exec(engine, new Object[] {null, null, null}); + Assertions.assertEquals(1, result.size()); + Assertions.assertNull(result.get(0).getField(0)); + Assertions.assertNull(result.get(0).getField(1)); + Assertions.assertNull(result.get(0).getField(2)); + engine.close(); + } + + @Test + void testNullArithmetic() { + CalciteSQLEngine engine = + createAndInit("SELECT a + b AS sum_val FROM t", "t", numericRowType()); + + Assertions.assertNull(singleField(engine, new Object[] {null, 5})); + Assertions.assertNull(singleField(engine, new Object[] {5, null})); + engine.close(); + } + + @Test + void testNullInConcat() { + CalciteSQLEngine engine = + createAndInit( + "SELECT name || '-suffix' AS res FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testWhereEquals() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age = 25", "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "B", 26}).size()); + engine.close(); + } + + @Test + void testWhereNotEquals() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age <> 25", "test_table", buildRowType()); + + Assertions.assertEquals(0, exec(engine, new Object[] {1, "A", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "B", 26}).size()); + engine.close(); + } + + @Test + void testWhereLessThan() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age < 18", "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 10}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "B", 18}).size()); + engine.close(); + } + + @Test + void testWhereGreaterThanOrEqual() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age >= 18", "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 18}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "B", 17}).size()); + engine.close(); + } + + @Test + void testWhereAnd() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age > 18 AND name = 'Alice'", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Bob", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {3, "Alice", 10}).size()); + engine.close(); + } + + @Test + void testWhereOr() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age < 10 OR age > 60", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 5}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "B", 70}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {3, "C", 30}).size()); + engine.close(); + } + + @Test + void testWhereNot() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE NOT (age > 18)", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 10}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "B", 25}).size()); + engine.close(); + } + + @Test + void testWhereIn() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age IN (18, 25, 30)", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "B", 20}).size()); + engine.close(); + } + + @Test + void testWhereNotIn() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age NOT IN (18, 25, 30)", + "test_table", + buildRowType()); + + Assertions.assertEquals(0, exec(engine, new Object[] {1, "A", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "B", 20}).size()); + engine.close(); + } + + @Test + void testWhereBetween() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age BETWEEN 20 AND 30", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "B", 20}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {3, "C", 30}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {4, "D", 15}).size()); + engine.close(); + } + + @Test + void testWhereNotBetween() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age NOT BETWEEN 20 AND 30", + "test_table", + buildRowType()); + + Assertions.assertEquals(0, exec(engine, new Object[] {1, "A", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "B", 15}).size()); + engine.close(); + } + + @Test + void testWhereLike() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name LIKE 'A%'", + "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Bob", 20}).size()); + engine.close(); + } + + @Test + void testWhereLikeSuffix() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name LIKE '%nel'", + "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "seatunnel", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "connector", 25}).size()); + engine.close(); + } + + @Test + void testWhereLikeMiddle() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name LIKE '%tunn%'", + "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "seatunnel", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "connector", 25}).size()); + engine.close(); + } + + @Test + void testWhereLikeSingleChar() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name LIKE 'A____'", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Al", 25}).size()); + engine.close(); + } + + @Test + void testWhereNotLike() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name NOT LIKE 'A%'", + "test_table", buildRowType()); + + Assertions.assertEquals(0, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "Bob", 25}).size()); + engine.close(); + } + + @Test + void testWhereIsNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name IS NULL", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, null, 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Alice", 25}).size()); + engine.close(); + } + + @Test + void testComplexWhereCondition() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table " + + "WHERE (age >= 18 AND name LIKE 'A%') OR age > 60", + "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Bob", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {3, "Bob", 70}).size()); + engine.close(); + } + + @Test + void testCastIntToString() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(age AS VARCHAR) AS age_str FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("25", singleField(engine, new Object[] {1, "Alice", 25})); + engine.close(); + } + + @Test + void testCastStringToInt() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(name AS INTEGER) AS val FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(42, singleField(engine, new Object[] {1, "42", 25})); + engine.close(); + } + + @Test + void testCastIntToDouble() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(age AS DOUBLE) AS dbl FROM test_table", + "test_table", + buildRowType()); + + Object result = singleField(engine, new Object[] {1, "A", 25}); + Assertions.assertEquals(25.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testCastIntToBigint() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(age AS BIGINT) AS big FROM test_table", + "test_table", + buildRowType()); + + Object result = singleField(engine, new Object[] {1, "A", 25}); + Assertions.assertEquals(25L, ((Number) result).longValue()); + engine.close(); + } + + @Test + void testCastBooleanToVarchar() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"flag"}, new SeaTunnelDataType[] {BasicType.BOOLEAN_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT CAST(flag AS VARCHAR) AS res FROM t", "t", rt); + + Assertions.assertEquals("TRUE", singleField(engine, new Object[] {true})); + Assertions.assertEquals("FALSE", singleField(engine, new Object[] {false})); + engine.close(); + } + + @Test + void testBooleanExpressions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, age > 18 AS is_adult, name IS NOT NULL AS has_name " + + "FROM test_table", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {1, "Alice", 25}); + Assertions.assertEquals(true, result.get(0).getField(1)); + Assertions.assertEquals(true, result.get(0).getField(2)); + engine.close(); + } + + @Test + void testBooleanAnd() { + CalciteSQLEngine engine = + createAndInit( + "SELECT (age > 18 AND age < 65) AS working_age FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(true, singleField(engine, new Object[] {1, "A", 30})); + Assertions.assertEquals(false, singleField(engine, new Object[] {2, "B", 10})); + Assertions.assertEquals(false, singleField(engine, new Object[] {3, "C", 70})); + engine.close(); + } + + @Test + void testBooleanOr() { + CalciteSQLEngine engine = + createAndInit( + "SELECT (age < 10 OR age > 60) AS extreme_age FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(true, singleField(engine, new Object[] {1, "A", 5})); + Assertions.assertEquals(true, singleField(engine, new Object[] {2, "B", 70})); + Assertions.assertEquals(false, singleField(engine, new Object[] {3, "C", 30})); + engine.close(); + } + + @Test + void testBooleanNot() { + CalciteSQLEngine engine = + createAndInit( + "SELECT NOT (age > 18) AS is_minor FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals(true, singleField(engine, new Object[] {1, "A", 10})); + Assertions.assertEquals(false, singleField(engine, new Object[] {2, "B", 25})); + engine.close(); + } + + @Test + void testBooleanColumnDirect() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"id", "active"}, + new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.BOOLEAN_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT * FROM t WHERE active", "t", rt); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, true}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, false}).size()); + engine.close(); + } + + @Test + void testDateType() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"id", "dt"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, LocalTimeType.LOCAL_DATE_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT id, dt FROM t", "t", rowType); + + LocalDate date = LocalDate.of(2024, 6, 15); + List result = exec(engine, new Object[] {1, date}); + Assertions.assertInstanceOf(LocalDate.class, result.get(0).getField(1)); + Assertions.assertEquals(date, result.get(0).getField(1)); + engine.close(); + } + + @Test + void testTimestampType() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"id", "ts"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, LocalTimeType.LOCAL_DATE_TIME_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT id, ts FROM t", "t", rowType); + + LocalDateTime ts = LocalDateTime.of(2024, 6, 15, 10, 30, 0); + List result = exec(engine, new Object[] {1, ts}); + Assertions.assertInstanceOf(LocalDateTime.class, result.get(0).getField(1)); + Assertions.assertEquals(ts, result.get(0).getField(1)); + engine.close(); + } + + @Test + void testTimeType() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"id", "tm"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, LocalTimeType.LOCAL_TIME_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT id, tm FROM t", "t", rowType); + + LocalTime time = LocalTime.of(14, 30, 0); + List result = exec(engine, new Object[] {1, time}); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(time, result.get(0).getField(1)); + engine.close(); + } + + @Test + void testTimeTypeMillisecondRoundTrip() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"id", "tm"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, LocalTimeType.LOCAL_TIME_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT id, tm FROM t", "t", rowType); + + LocalTime time = LocalTime.of(14, 30, 12, 345_000_000); + List result = exec(engine, new Object[] {1, time}); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(time, result.get(0).getField(1)); + engine.close(); + } + + @Test + void testTimeTypeNanoTruncation() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"id", "tm"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, LocalTimeType.LOCAL_TIME_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT id, tm FROM t", "t", rowType); + + LocalTime input = LocalTime.of(14, 30, 12, 345_678_912); + LocalTime expected = LocalTime.of(14, 30, 12, 345_000_000); + List result = exec(engine, new Object[] {1, input}); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(expected, result.get(0).getField(1)); + engine.close(); + } + + @Test + void testCurrentDateOutputType() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CURRENT_DATE AS today FROM test_table", + "test_table", + buildRowType()); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(1, outType.getTotalFields()); + Assertions.assertEquals("today", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testDateExtractYearOutputType() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"dt"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT EXTRACT(YEAR FROM dt) AS yr FROM t", "t", rowType); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(1, outType.getTotalFields()); + Assertions.assertEquals("yr", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testDateExtractMonthOutputType() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"dt"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT EXTRACT(MONTH FROM dt) AS mon FROM t", "t", rowType); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals("mon", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testDateExtractDayOutputType() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"dt"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT EXTRACT(DAY FROM dt) AS d FROM t", "t", rowType); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals("d", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testNullDate() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"id", "dt"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, LocalTimeType.LOCAL_DATE_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT id, dt FROM t", "t", rowType); + + List result = exec(engine, new Object[] {1, null}); + Assertions.assertNull(result.get(0).getField(1)); + engine.close(); + } + + @Test + void testDateEpoch() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"dt"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT dt FROM t", "t", rowType); + + LocalDate epoch = LocalDate.of(1970, 1, 1); + Object result = singleField(engine, new Object[] {epoch}); + Assertions.assertEquals(epoch, result); + engine.close(); + } + + @Test + void testMultipleNumericTypes() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] { + "byte_val", "short_val", "long_val", "float_val", "double_val" + }, + new SeaTunnelDataType[] { + BasicType.BYTE_TYPE, + BasicType.SHORT_TYPE, + BasicType.LONG_TYPE, + BasicType.FLOAT_TYPE, + BasicType.DOUBLE_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT * FROM t", "t", rowType); + + SeaTunnelRow row = + new SeaTunnelRow(new Object[] {(byte) 1, (short) 200, 100000L, 3.14f, 2.718281828}); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(SqlType.TINYINT, outType.getFieldType(0).getSqlType()); + Assertions.assertEquals(SqlType.SMALLINT, outType.getFieldType(1).getSqlType()); + Assertions.assertEquals(SqlType.BIGINT, outType.getFieldType(2).getSqlType()); + Assertions.assertEquals(SqlType.FLOAT, outType.getFieldType(3).getSqlType()); + Assertions.assertEquals(SqlType.DOUBLE, outType.getFieldType(4).getSqlType()); + engine.close(); + } + + @Test + void testDecimalType() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"amount"}, new SeaTunnelDataType[] {new DecimalType(10, 2)}); + CalciteSQLEngine engine = createAndInit("SELECT amount FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {new BigDecimal("123.45")}); + Assertions.assertNotNull(result); + engine.close(); + } + + @Test + void testDecimalArithmetic() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"a", "b"}, + new SeaTunnelDataType[] {new DecimalType(10, 2), new DecimalType(10, 2)}); + CalciteSQLEngine engine = createAndInit("SELECT a + b AS total FROM t", "t", rt); + + Object result = + singleField( + engine, new Object[] {new BigDecimal("100.50"), new BigDecimal("200.75")}); + Assertions.assertNotNull(result); + Assertions.assertInstanceOf(BigDecimal.class, result); + Assertions.assertEquals(0, new BigDecimal("301.25").compareTo((BigDecimal) result)); + engine.close(); + } + + @Test + void testByteValueBoundary() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.BYTE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {Byte.MAX_VALUE}); + Assertions.assertEquals(Byte.MAX_VALUE, ((Number) result).byteValue()); + engine.close(); + } + + @Test + void testShortValueBoundary() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.SHORT_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {Short.MAX_VALUE}); + Assertions.assertEquals(Short.MAX_VALUE, ((Number) result).shortValue()); + engine.close(); + } + + @Test + void testLongValueBoundary() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.LONG_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {Long.MAX_VALUE}); + Assertions.assertEquals(Long.MAX_VALUE, ((Number) result).longValue()); + engine.close(); + } + + @Test + void testFloatPrecision() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.FLOAT_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {0.1f}); + Assertions.assertEquals(0.1f, ((Number) result).floatValue(), 0.0001f); + engine.close(); + } + + @Test + void testDoubleNaN() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {Double.NaN}); + Assertions.assertTrue(Double.isNaN(((Number) result).doubleValue())); + engine.close(); + } + + @Test + void testDoubleInfinity() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {Double.POSITIVE_INFINITY}); + Assertions.assertTrue(Double.isInfinite(((Number) result).doubleValue())); + engine.close(); + } + + @Test + void testOutputRowType() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, UPPER(name) AS name_upper FROM test_table", + "test_table", + buildRowType()); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(2, outType.getTotalFields()); + Assertions.assertEquals("id", outType.getFieldName(0)); + Assertions.assertEquals("name_upper", outType.getFieldName(1)); + engine.close(); + } + + @Test + void testOutputRowTypeWithAlias() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id AS user_id, name AS user_name, age AS user_age FROM test_table", + "test_table", + buildRowType()); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals("user_id", outType.getFieldName(0)); + Assertions.assertEquals("user_name", outType.getFieldName(1)); + Assertions.assertEquals("user_age", outType.getFieldName(2)); + engine.close(); + } + + @Test + void testOutputTypeSingleColumn() { + CalciteSQLEngine engine = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(1, outType.getTotalFields()); + Assertions.assertEquals(SqlType.STRING, outType.getFieldType(0).getSqlType()); + engine.close(); + } + + @Test + void testOutputTypeComputedColumn() { + CalciteSQLEngine engine = + createAndInit( + "SELECT age * 2 AS doubled FROM test_table", "test_table", buildRowType()); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals("doubled", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testOutputTypeManyColumns() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, name, age, id + age AS sum_val, " + + "UPPER(name) AS upper_name, age > 18 AS adult " + + "FROM test_table", + "test_table", + buildRowType()); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(6, outType.getTotalFields()); + engine.close(); + } + + @Test + void testConstantExpression() { + CalciteSQLEngine engine = + createAndInit("SELECT 42 AS answer FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals(42, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testStringConstant() { + CalciteSQLEngine engine = + createAndInit( + "SELECT 'seatunnel' AS name FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals("seatunnel", singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testNullConstant() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(NULL AS VARCHAR) AS null_val FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertNull(singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testConstantArithmetic() { + CalciteSQLEngine engine = + createAndInit("SELECT 2 + 3 AS five FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals(5, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testEmptyString() { + CalciteSQLEngine engine = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals("", singleField(engine, new Object[] {1, "", 25})); + engine.close(); + } + + @Test + void testSpecialCharactersInString() { + CalciteSQLEngine engine = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals( + "sea'tunnel", singleField(engine, new Object[] {1, "sea'tunnel", 25})); + engine.close(); + } + + @Test + void testLongString() { + CalciteSQLEngine engine = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + sb.append("seatunnel"); + } + String longStr = sb.toString(); + Assertions.assertEquals(longStr, singleField(engine, new Object[] {1, longStr, 25})); + engine.close(); + } + + @Test + void testSingleColumnTable() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.INT_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + + Assertions.assertEquals(42, singleField(engine, new Object[] {42})); + engine.close(); + } + + @Test + void testManyColumnsTable() { + String[] names = new String[20]; + SeaTunnelDataType[] types = new SeaTunnelDataType[20]; + Object[] values = new Object[20]; + for (int i = 0; i < 20; i++) { + names[i] = "col_" + i; + types[i] = BasicType.INT_TYPE; + values[i] = i; + } + SeaTunnelRowType rt = new SeaTunnelRowType(names, types); + CalciteSQLEngine engine = createAndInit("SELECT * FROM t", "t", rt); + + List result = exec(engine, values); + Assertions.assertEquals(20, result.get(0).getArity()); + for (int i = 0; i < 20; i++) { + Assertions.assertEquals(i, result.get(0).getField(i)); + } + engine.close(); + } + + @Test + void testEngineReuse() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, name FROM test_table WHERE age > 20", + "test_table", + buildRowType()); + + for (int i = 0; i < 100; i++) { + int age = i % 40; + List result = exec(engine, new Object[] {i, "user_" + i, age}); + if (age > 20) { + Assertions.assertEquals(1, result.size()); + } else { + Assertions.assertTrue(result.isEmpty()); + } + } + engine.close(); + } + + @Test + void testEngineReuseWithDifferentData() { + CalciteSQLEngine engine = + createAndInit( + "SELECT UPPER(name) AS upper_name FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("ALICE", singleField(engine, new Object[] {1, "alice", 25})); + Assertions.assertEquals("BOB", singleField(engine, new Object[] {2, "bob", 30})); + Assertions.assertEquals("", singleField(engine, new Object[] {3, "", 20})); + engine.close(); + } + + @Test + void testMultipleEngineInstances() { + CalciteSQLEngine engine1 = + createAndInit("SELECT id FROM test_table", "test_table", buildRowType()); + CalciteSQLEngine engine2 = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals(1, singleField(engine1, new Object[] {1, "Alice", 25})); + Assertions.assertEquals("Alice", singleField(engine2, new Object[] {1, "Alice", 25})); + + engine1.close(); + engine2.close(); + } + + @Test + void testCloseAndReinit() { + CalciteSQLEngine engine = + new CalciteSQLEngine("SELECT id FROM test_table", "test_table", buildRowType()); + engine.init(); + Assertions.assertEquals(1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + + engine.init(); + Assertions.assertEquals(2, singleField(engine, new Object[] {2, "B", 30})); + engine.close(); + } + + @Test + void testInvalidSqlThrows() { + CalciteSQLEngine engine = + new CalciteSQLEngine("SELECTTTTT * FROM test_table", "test_table", buildRowType()); + Assertions.assertThrows(TransformException.class, engine::init); + } + + @Test + void testInvalidColumnThrows() { + CalciteSQLEngine engine = + new CalciteSQLEngine( + "SELECT nonexistent_col FROM test_table", "test_table", buildRowType()); + Assertions.assertThrows(TransformException.class, engine::init); + } + + @Test + void testEmptySqlThrows() { + CalciteSQLEngine engine = new CalciteSQLEngine("", "test_table", buildRowType()); + Assertions.assertThrows(TransformException.class, engine::init); + } + + @Test + void testInvalidFunctionThrows() { + CalciteSQLEngine engine = + new CalciteSQLEngine( + "SELECT NO_SUCH_FUNCTION(name) FROM test_table", + "test_table", + buildRowType()); + Assertions.assertThrows(TransformException.class, engine::init); + } + + @Test + void testWrongTableNameThrows() { + CalciteSQLEngine engine = + new CalciteSQLEngine("SELECT * FROM wrong_table", "test_table", buildRowType()); + Assertions.assertThrows(TransformException.class, engine::init); + } + + @Test + void testSyntaxErrorThrows() { + CalciteSQLEngine engine = + new CalciteSQLEngine("SELECT FROM test_table WHERE", "test_table", buildRowType()); + Assertions.assertThrows(TransformException.class, engine::init); + } + + @Test + void testUnclosedStringThrows() { + CalciteSQLEngine engine = + new CalciteSQLEngine( + "SELECT * FROM test_table WHERE name = 'unclosed", + "test_table", + buildRowType()); + Assertions.assertThrows(TransformException.class, engine::init); + } + + @Test + void testMixedComputedColumns() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, " + + "UPPER(name) AS upper_name, " + + "age * 2 AS double_age, " + + "CHAR_LENGTH(name) AS name_len, " + + "age > 18 AS is_adult " + + "FROM test_table", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {1, "seatunnel", 25}); + Assertions.assertEquals(1, result.get(0).getField(0)); + Assertions.assertEquals("SEATUNNEL", result.get(0).getField(1)); + Assertions.assertEquals(50, result.get(0).getField(2)); + Assertions.assertEquals(9, result.get(0).getField(3)); + Assertions.assertEquals(true, result.get(0).getField(4)); + engine.close(); + } + + @Test + void testWhereWithComputedValue() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table WHERE age * 2 > 50", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 30}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "B", 20}).size()); + engine.close(); + } + + @Test + void testWhereWithStringFunction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE CHAR_LENGTH(name) > 5", + "test_table", + buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "seatunnel", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "zeta", 25}).size()); + engine.close(); + } + + @Test + void testSubqueryLiteral() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, age, CASE " + + "WHEN age IN (18, 25, 30, 60) THEN 'milestone' " + + "ELSE 'normal' END AS tag " + + "FROM test_table", + "test_table", + buildRowType()); + + List r1 = exec(engine, new Object[] {1, "A", 25}); + Assertions.assertTrue(r1.get(0).getField(2).toString().contains("milestone")); + List r2 = exec(engine, new Object[] {2, "B", 22}); + Assertions.assertTrue(r2.get(0).getField(2).toString().contains("normal")); + engine.close(); + } + + @Test + void testCaseInsensitiveColumn() { + CalciteSQLEngine engine = + createAndInit("SELECT ID, NAME, AGE FROM test_table", "test_table", buildRowType()); + + List result = exec(engine, new Object[] {1, "Alice", 25}); + Assertions.assertEquals(1, result.get(0).getField(0)); + Assertions.assertEquals("Alice", result.get(0).getField(1)); + engine.close(); + } + + @Test + void testCaseInsensitiveFunction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT upper(name) AS res FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals("ALICE", singleField(engine, new Object[] {1, "Alice", 25})); + engine.close(); + } + + @Test + void testCaseInsensitiveKeywords() { + CalciteSQLEngine engine = + createAndInit( + "select id from test_table where age > 20", "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 25}).size()); + engine.close(); + } + + @Test + void testCountConstant() { + CalciteSQLEngine engine = + createAndInit("SELECT 1 AS cnt FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals(1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testConcatMultiple() { + CalciteSQLEngine engine = + createAndInit( + "SELECT name || ' (id=' || CAST(id AS VARCHAR) || ')' AS label FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("Alice (id=1)", singleField(engine, new Object[] {1, "Alice", 25})); + engine.close(); + } + + @Test + void testSubstringWithLength() { + CalciteSQLEngine engine = + createAndInit( + "SELECT SUBSTRING(name, 1, 3) AS prefix FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("sea", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testSubstringBeyondLength() { + CalciteSQLEngine engine = + createAndInit( + "SELECT SUBSTRING(name, 1, 100) AS res FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals("abc", singleField(engine, new Object[] {1, "abc", 25})); + engine.close(); + } + + @Test + void testNestedStringFunctions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT UPPER(TRIM(name)) AS res FROM test_table", + "test_table", + buildRowType()); + + Assertions.assertEquals( + "SEATUNNEL", singleField(engine, new Object[] {1, " seatunnel ", 25})); + engine.close(); + } + + @Test + void testDistinctFromAllRows() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id + 0 AS same FROM test_table", "test_table", buildRowType()); + + Assertions.assertEquals(1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testMultipleWhereConditionsParenthesized() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table " + + "WHERE (age > 10 AND age < 30) AND (name LIKE 'A%' OR name LIKE 'B%')", + "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "Bob", 20}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {3, "Charlie", 25}).size()); + engine.close(); + } + + @Test + void testTableId() { + CalciteSQLEngine engine = + createAndInit("SELECT id FROM test_table", "test_table", buildRowType()); + + SeaTunnelRow input = new SeaTunnelRow(new Object[] {1, "A", 25}); + input.setTableId("source_table"); + List result = engine.execute(input); + Assertions.assertEquals("source_table", result.get(0).getTableId()); + engine.close(); + } + + @Test + void testFilteredRowReturnsEmpty() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table WHERE 1 = 0", "test_table", buildRowType()); + + Assertions.assertTrue(exec(engine, new Object[] {1, "A", 25}).isEmpty()); + engine.close(); + } + + @Test + void testAlwaysTrueWhere() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table WHERE 1 = 1", "test_table", buildRowType()); + + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 25}).size()); + engine.close(); + } + + @Test + void testUpperNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT UPPER(name) AS res FROM test_table", "test_table", buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testLowerNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT LOWER(name) AS res FROM test_table", "test_table", buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testCharLengthNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CHAR_LENGTH(name) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testSubstringNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT SUBSTRING(name, 1, 3) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testTrimNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT TRIM(name) AS res FROM test_table", "test_table", buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testReplaceNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT REPLACE(name, 'a', 'b') AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testPositionNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT POSITION('x' IN name) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testAbsNull() { + CalciteSQLEngine engine = + createAndInit("SELECT ABS(a) AS res FROM t", "t", numericRowType()); + Assertions.assertNull(singleField(engine, new Object[] {null, 0})); + engine.close(); + } + + @Test + void testCeilNull() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT CEIL(val) AS res FROM t", "t", rt); + Assertions.assertNull(singleField(engine, new Object[] {null})); + engine.close(); + } + + @Test + void testFloorNull() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT FLOOR(val) AS res FROM t", "t", rt); + Assertions.assertNull(singleField(engine, new Object[] {null})); + engine.close(); + } + + @Test + void testSignPositive() { + CalciteSQLEngine engine = + createAndInit("SELECT SIGN(a) AS res FROM t", "t", numericRowType()); + Assertions.assertEquals(1, singleField(engine, new Object[] {42, 0})); + engine.close(); + } + + @Test + void testSignNegative() { + CalciteSQLEngine engine = + createAndInit("SELECT SIGN(a) AS res FROM t", "t", numericRowType()); + Assertions.assertEquals(-1, singleField(engine, new Object[] {-42, 0})); + engine.close(); + } + + @Test + void testSignZero() { + CalciteSQLEngine engine = + createAndInit("SELECT SIGN(a) AS res FROM t", "t", numericRowType()); + Assertions.assertEquals(0, singleField(engine, new Object[] {0, 0})); + engine.close(); + } + + @Test + void testCeilNegative() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT CEIL(val) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {-3.2}); + Assertions.assertEquals(-3.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testFloorNegative() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT FLOOR(val) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {-3.2}); + Assertions.assertEquals(-4.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testSqrtZero() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT SQRT(val) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {0.0}); + Assertions.assertEquals(0.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testPowerZeroExponent() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"base_val", "exponent"}, + new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE, BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT POWER(base_val, exponent) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {999.0, 0.0}); + Assertions.assertEquals(1.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testPowerNegativeExponent() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"base_val", "exponent"}, + new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE, BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT POWER(base_val, exponent) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {2.0, -1.0}); + Assertions.assertEquals(0.5, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testModNegative() { + CalciteSQLEngine engine = + createAndInit("SELECT MOD(a, b) AS res FROM t", "t", numericRowType()); + Object result = singleField(engine, new Object[] {-10, 3}); + Assertions.assertEquals(-1, result); + engine.close(); + } + + @Test + void testNestedArithmetic() { + CalciteSQLEngine engine = + createAndInit("SELECT (a + b) * (a - b) AS res FROM t", "t", numericRowType()); + Assertions.assertEquals(91, singleField(engine, new Object[] {10, 3})); + engine.close(); + } + + @Test + void testArithmeticPrecedence() { + CalciteSQLEngine engine = + createAndInit("SELECT a + b * 2 AS res FROM t", "t", numericRowType()); + Assertions.assertEquals(16, singleField(engine, new Object[] {10, 3})); + engine.close(); + } + + @Test + void testNullEqualsNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name = name", "test_table", buildRowType()); + Assertions.assertEquals(0, exec(engine, new Object[] {1, null, 25}).size()); + engine.close(); + } + + @Test + void testNullNotEqualsAnything() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name = 'Alice'", + "test_table", + buildRowType()); + Assertions.assertEquals(0, exec(engine, new Object[] {1, null, 25}).size()); + engine.close(); + } + + @Test + void testNullInBetween() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age BETWEEN 10 AND 30", + "test_table", + buildRowType()); + Assertions.assertEquals(0, exec(engine, new Object[] {1, "A", null}).size()); + engine.close(); + } + + @Test + void testNullInLike() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name LIKE 'A%'", + "test_table", buildRowType()); + Assertions.assertEquals(0, exec(engine, new Object[] {1, null, 25}).size()); + engine.close(); + } + + @Test + void testNullInIn() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age IN (10, 20, 30)", + "test_table", + buildRowType()); + Assertions.assertEquals(0, exec(engine, new Object[] {1, "A", null}).size()); + engine.close(); + } + + @Test + void testCoalesceAllNonNull() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"a", "b"}, + new SeaTunnelDataType[] {BasicType.STRING_TYPE, BasicType.STRING_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT COALESCE(a, b) AS res FROM t", "t", rt); + Assertions.assertEquals("first", singleField(engine, new Object[] {"first", "second"})); + engine.close(); + } + + @Test + void testNullifBothSame() { + CalciteSQLEngine engine = + createAndInit( + "SELECT NULLIF(name, name) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, "Alice", 25})); + engine.close(); + } + + @Test + void testNullifWithNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT NULLIF(name, 'X') AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, null, 25})); + engine.close(); + } + + @Test + void testSelectColumnReorder() { + CalciteSQLEngine engine = + createAndInit("SELECT age, name, id FROM test_table", "test_table", buildRowType()); + + List result = exec(engine, new Object[] {1, "Alice", 25}); + Assertions.assertEquals(25, result.get(0).getField(0)); + Assertions.assertEquals("Alice", result.get(0).getField(1)); + Assertions.assertEquals(1, result.get(0).getField(2)); + engine.close(); + } + + @Test + void testSelectDuplicateColumn() { + CalciteSQLEngine engine = + createAndInit( + "SELECT name AS n1, name AS n2 FROM test_table", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {1, "Alice", 25}); + Assertions.assertEquals("Alice", result.get(0).getField(0)); + Assertions.assertEquals("Alice", result.get(0).getField(1)); + engine.close(); + } + + @Test + void testSelectSameColumnMultipleTimes() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, id + 1 AS id_next, id * 2 AS id_double FROM test_table", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {5, "A", 25}); + Assertions.assertEquals(5, result.get(0).getField(0)); + Assertions.assertEquals(6, result.get(0).getField(1)); + Assertions.assertEquals(10, result.get(0).getField(2)); + engine.close(); + } + + @Test + void testCastIntToFloat() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(age AS FLOAT) AS res FROM test_table", + "test_table", + buildRowType()); + + Object result = singleField(engine, new Object[] {1, "A", 25}); + Assertions.assertNotNull(result); + Assertions.assertEquals(25.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testCastStringToDouble() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT CAST(val AS DOUBLE) AS res FROM t", "t", rt); + + Object result = singleField(engine, new Object[] {"3.14"}); + Assertions.assertEquals(3.14, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testCastNullToInt() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(NULL AS INTEGER) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testCastNullToDouble() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(NULL AS DOUBLE) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testCastNullToBigint() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(NULL AS BIGINT) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertNull(singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testCastChain() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(CAST(age AS VARCHAR) AS INTEGER) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals(25, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testCastDecimalToInt() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {new DecimalType(10, 2)}); + CalciteSQLEngine engine = + createAndInit("SELECT CAST(val AS INTEGER) AS res FROM t", "t", rt); + Assertions.assertEquals(123, singleField(engine, new Object[] {new BigDecimal("123.99")})); + engine.close(); + } + + @Test + void testCastIntToDecimal() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(age AS DECIMAL(10,2)) AS res FROM test_table", + "test_table", + buildRowType()); + Object result = singleField(engine, new Object[] {1, "A", 25}); + Assertions.assertInstanceOf(BigDecimal.class, result); + Assertions.assertEquals(0, new BigDecimal("25.00").compareTo((BigDecimal) result)); + engine.close(); + } + + @Test + void testDecimalSubtraction() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"a", "b"}, + new SeaTunnelDataType[] {new DecimalType(10, 2), new DecimalType(10, 2)}); + CalciteSQLEngine engine = createAndInit("SELECT a - b AS diff FROM t", "t", rt); + Object result = + singleField( + engine, new Object[] {new BigDecimal("100.50"), new BigDecimal("30.25")}); + Assertions.assertInstanceOf(BigDecimal.class, result); + Assertions.assertEquals(0, new BigDecimal("70.25").compareTo((BigDecimal) result)); + engine.close(); + } + + @Test + void testDecimalMultiplication() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"a", "b"}, + new SeaTunnelDataType[] {new DecimalType(10, 2), new DecimalType(10, 2)}); + CalciteSQLEngine engine = createAndInit("SELECT a * b AS product FROM t", "t", rt); + Object result = + singleField(engine, new Object[] {new BigDecimal("10.00"), new BigDecimal("3.50")}); + Assertions.assertInstanceOf(BigDecimal.class, result); + Assertions.assertEquals(0, new BigDecimal("35.0000").compareTo((BigDecimal) result)); + engine.close(); + } + + @Test + void testDecimalDivision() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"a", "b"}, + new SeaTunnelDataType[] {new DecimalType(10, 2), new DecimalType(10, 2)}); + CalciteSQLEngine engine = createAndInit("SELECT a / b AS quotient FROM t", "t", rt); + Object result = + singleField( + engine, new Object[] {new BigDecimal("100.00"), new BigDecimal("4.00")}); + Assertions.assertNotNull(result); + Assertions.assertInstanceOf(BigDecimal.class, result); + engine.close(); + } + + @Test + void testDecimalComparisonInWhere() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"amount"}, new SeaTunnelDataType[] {new DecimalType(10, 2)}); + CalciteSQLEngine engine = + createAndInit("SELECT amount FROM t WHERE amount > 100.00", "t", rt); + Assertions.assertEquals(1, exec(engine, new Object[] {new BigDecimal("200.50")}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {new BigDecimal("50.00")}).size()); + engine.close(); + } + + @Test + void testMixedIntDoubleArithmetic() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"i", "d"}, + new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT i + d AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {10, 3.14}); + Assertions.assertEquals(13.14, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testMixedIntLongArithmetic() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"i", "l"}, + new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.LONG_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT i + l AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {10, 100000L}); + Assertions.assertEquals(100010L, ((Number) result).longValue()); + engine.close(); + } + + @Test + void testBooleanWithNull() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"flag"}, new SeaTunnelDataType[] {BasicType.BOOLEAN_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT flag FROM t", "t", rt); + Assertions.assertNull(singleField(engine, new Object[] {null})); + engine.close(); + } + + @Test + void testBooleanInCaseWhen() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"active"}, new SeaTunnelDataType[] {BasicType.BOOLEAN_TYPE}); + CalciteSQLEngine engine = + createAndInit( + "SELECT TRIM(CASE WHEN active THEN 'on' ELSE 'off' END) AS status FROM t", + "t", + rt); + Assertions.assertEquals("on", singleField(engine, new Object[] {true})); + Assertions.assertEquals("off", singleField(engine, new Object[] {false})); + engine.close(); + } + + @Test + void testMultipleCaseWhenInSelect() { + CalciteSQLEngine engine = + createAndInit( + "SELECT " + + "CASE WHEN age >= 18 THEN 'adult' ELSE 'minor' END AS cat1, " + + "CASE WHEN name IS NULL THEN 'anon' ELSE name END AS cat2 " + + "FROM test_table", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {1, null, 10}); + Assertions.assertEquals("minor", result.get(0).getField(0)); + Assertions.assertEquals("anon", result.get(0).getField(1)); + engine.close(); + } + + @Test + void testCaseWhenWithArithmetic() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CASE WHEN age > 18 THEN age * 2 ELSE age END AS res " + + "FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals(50, singleField(engine, new Object[] {1, "A", 25})); + Assertions.assertEquals(10, singleField(engine, new Object[] {2, "B", 10})); + engine.close(); + } + + @Test + void testCaseWhenInWhere() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table " + + "WHERE CASE WHEN age > 18 THEN 1 ELSE 0 END = 1", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "B", 10}).size()); + engine.close(); + } + + @Test + void testWhereWithMultipleColumnsComplex() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table " + + "WHERE age > 18 AND CHAR_LENGTH(name) > 3 AND id > 0", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Al", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {3, "Alice", 10}).size()); + engine.close(); + } + + @Test + void testStringComparisonInWhere() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name > 'M'", "test_table", buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Zeta", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Alice", 25}).size()); + engine.close(); + } + + @Test + void testStringEqualityInWhere() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name = 'seatunnel'", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "seatunnel", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "SeaTunnel", 25}).size()); + engine.close(); + } + + @Test + void testInWithStrings() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name IN ('Alice', 'Bob', 'Charlie')", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "Bob", 30}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {3, "Dave", 20}).size()); + engine.close(); + } + + @Test + void testLikeEmptyPattern() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name LIKE '%'", + "test_table", buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "anything", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "", 25}).size()); + engine.close(); + } + + @Test + void testLikeExactMatch() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name LIKE 'seatunnel'", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "seatunnel", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "seatunnel2", 25}).size()); + engine.close(); + } + + @Test + void testWhereOnComputedExpression() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table WHERE UPPER(name) = 'SEATUNNEL'", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "SeaTunnel", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "seatunnel", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {3, "zeta", 25}).size()); + engine.close(); + } + + @Test + void testWhereOnConcatExpression() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table WHERE name || '-suffix' = 'Alice-suffix'", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Bob", 25}).size()); + engine.close(); + } + + @Test + void testTimestampExtractOutputType() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"ts"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TIME_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT EXTRACT(HOUR FROM ts) AS h FROM t", "t", rt); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(1, outType.getTotalFields()); + Assertions.assertEquals("h", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testDateWithIntFilter() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"id", "dt"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, LocalTimeType.LOCAL_DATE_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT id, dt FROM t WHERE id > 0", "t", rt); + List rows = exec(engine, new Object[] {1, LocalDate.of(2024, 6, 15)}); + Assertions.assertEquals(1, rows.size()); + Assertions.assertInstanceOf(LocalDate.class, rows.get(0).getField(1)); + engine.close(); + } + + @Test + void testTimestampWithIntFilter() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"id", "ts"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, LocalTimeType.LOCAL_DATE_TIME_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT id, ts FROM t WHERE id > 0", "t", rt); + List rows = + exec(engine, new Object[] {1, LocalDateTime.of(2024, 6, 15, 10, 30, 0)}); + Assertions.assertEquals(1, rows.size()); + Assertions.assertInstanceOf(LocalDateTime.class, rows.get(0).getField(1)); + engine.close(); + } + + @Test + void testCurrentTimestampOutputType() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CURRENT_TIMESTAMP AS ts_now FROM test_table", + "test_table", + buildRowType()); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals("ts_now", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testCurrentTimeOutputType() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CURRENT_TIME AS t_now FROM test_table", + "test_table", + buildRowType()); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals("t_now", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testDateLeapYear() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"dt"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT dt FROM t", "t", rt); + LocalDate leapDay = LocalDate.of(2024, 2, 29); + Object result = singleField(engine, new Object[] {leapDay}); + Assertions.assertEquals(leapDay, result); + engine.close(); + } + + @Test + void testDateFarFuture() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"dt"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT dt FROM t", "t", rt); + LocalDate future = LocalDate.of(9999, 12, 31); + Object result = singleField(engine, new Object[] {future}); + Assertions.assertEquals(future, result); + engine.close(); + } + + @Test + void testTimestampMidnight() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"ts"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TIME_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT ts FROM t", "t", rt); + LocalDateTime midnight = LocalDateTime.of(2024, 1, 1, 0, 0, 0); + Object result = singleField(engine, new Object[] {midnight}); + Assertions.assertEquals(midnight, result); + engine.close(); + } + + @Test + void testTimestampEndOfDay() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"ts"}, + new SeaTunnelDataType[] {LocalTimeType.LOCAL_DATE_TIME_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT ts FROM t", "t", rt); + LocalDateTime endOfDay = LocalDateTime.of(2024, 12, 31, 23, 59, 59); + Object result = singleField(engine, new Object[] {endOfDay}); + Assertions.assertEquals(endOfDay, result); + engine.close(); + } + + @Test + void testByteMinValue() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.BYTE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + Object result = singleField(engine, new Object[] {Byte.MIN_VALUE}); + Assertions.assertEquals(Byte.MIN_VALUE, ((Number) result).byteValue()); + engine.close(); + } + + @Test + void testShortMinValue() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.SHORT_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + Object result = singleField(engine, new Object[] {Short.MIN_VALUE}); + Assertions.assertEquals(Short.MIN_VALUE, ((Number) result).shortValue()); + engine.close(); + } + + @Test + void testLongMinValue() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.LONG_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + Object result = singleField(engine, new Object[] {Long.MIN_VALUE}); + Assertions.assertEquals(Long.MIN_VALUE, ((Number) result).longValue()); + engine.close(); + } + + @Test + void testDoubleNegativeInfinity() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + Object result = singleField(engine, new Object[] {Double.NEGATIVE_INFINITY}); + Assertions.assertTrue(Double.isInfinite(((Number) result).doubleValue())); + Assertions.assertTrue(((Number) result).doubleValue() < 0); + engine.close(); + } + + @Test + void testDoubleMinValue() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + Object result = singleField(engine, new Object[] {Double.MIN_VALUE}); + Assertions.assertEquals(Double.MIN_VALUE, ((Number) result).doubleValue(), 0.0); + engine.close(); + } + + @Test + void testDoubleMaxValue() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + Object result = singleField(engine, new Object[] {Double.MAX_VALUE}); + Assertions.assertEquals(Double.MAX_VALUE, ((Number) result).doubleValue(), 0.0); + engine.close(); + } + + @Test + void testDecimalZero() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {new DecimalType(10, 2)}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + Object result = singleField(engine, new Object[] {BigDecimal.ZERO}); + Assertions.assertInstanceOf(BigDecimal.class, result); + Assertions.assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) result)); + engine.close(); + } + + @Test + void testDecimalNegative() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {new DecimalType(10, 2)}); + CalciteSQLEngine engine = createAndInit("SELECT val FROM t", "t", rt); + Object result = singleField(engine, new Object[] {new BigDecimal("-99.99")}); + Assertions.assertInstanceOf(BigDecimal.class, result); + Assertions.assertEquals(0, new BigDecimal("-99.99").compareTo((BigDecimal) result)); + engine.close(); + } + + @Test + void testOutputTypeForCaseWhen() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CASE WHEN age > 18 THEN 'adult' ELSE 'minor' END AS label " + + "FROM test_table", + "test_table", + buildRowType()); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(1, outType.getTotalFields()); + Assertions.assertEquals("label", outType.getFieldName(0)); + engine.close(); + } + + @Test + void testOutputTypeForArithmetic() { + CalciteSQLEngine engine = + createAndInit( + "SELECT age + 1 AS next_age, age * 2 AS double_age FROM test_table", + "test_table", + buildRowType()); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(2, outType.getTotalFields()); + Assertions.assertEquals("next_age", outType.getFieldName(0)); + Assertions.assertEquals("double_age", outType.getFieldName(1)); + engine.close(); + } + + @Test + void testOutputTypeForBoolean() { + CalciteSQLEngine engine = + createAndInit( + "SELECT age > 18 AS is_adult FROM test_table", + "test_table", + buildRowType()); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(SqlType.BOOLEAN, outType.getFieldType(0).getSqlType()); + engine.close(); + } + + @Test + void testOutputTypeForCast() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(age AS BIGINT) AS big_age FROM test_table", + "test_table", + buildRowType()); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(SqlType.BIGINT, outType.getFieldType(0).getSqlType()); + engine.close(); + } + + @Test + void testOutputTypeForConcat() { + CalciteSQLEngine engine = + createAndInit( + "SELECT name || '-suffix' AS combined FROM test_table", + "test_table", + buildRowType()); + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(SqlType.STRING, outType.getFieldType(0).getSqlType()); + engine.close(); + } + + @Test + void testStringWithNewline() { + CalciteSQLEngine engine = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + Assertions.assertEquals( + "line1\nline2", singleField(engine, new Object[] {1, "line1\nline2", 25})); + engine.close(); + } + + @Test + void testStringWithTab() { + CalciteSQLEngine engine = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + Assertions.assertEquals( + "col1\tcol2", singleField(engine, new Object[] {1, "col1\tcol2", 25})); + engine.close(); + } + + @Test + void testStringWithBackslash() { + CalciteSQLEngine engine = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + Assertions.assertEquals( + "path\\to\\file", singleField(engine, new Object[] {1, "path\\to\\file", 25})); + engine.close(); + } + + @Test + void testStringWithDoubleQuotes() { + CalciteSQLEngine engine = + createAndInit("SELECT name FROM test_table", "test_table", buildRowType()); + Assertions.assertEquals( + "say \"hi\"", singleField(engine, new Object[] {1, "say \"hi\"", 25})); + engine.close(); + } + + @Test + void testReplaceEmptyString() { + CalciteSQLEngine engine = + createAndInit( + "SELECT REPLACE(name, '-', '') AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals( + "seatunnel", singleField(engine, new Object[] {1, "sea-tunnel", 25})); + engine.close(); + } + + @Test + void testReplaceNoMatch() { + CalciteSQLEngine engine = + createAndInit( + "SELECT REPLACE(name, 'xyz', 'abc') AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals( + "seatunnel", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testReplaceMultipleOccurrences() { + CalciteSQLEngine engine = + createAndInit( + "SELECT REPLACE(name, 'e', 'E') AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals( + "sEatunnEl", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testSubstringStartAtEnd() { + CalciteSQLEngine engine = + createAndInit( + "SELECT SUBSTRING(name, 10) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals("", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testOverlayAtStart() { + CalciteSQLEngine engine = + createAndInit( + "SELECT OVERLAY(name PLACING 'NEW' FROM 1 FOR 3) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals( + "NEWtunnel", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testOverlayAtEnd() { + CalciteSQLEngine engine = + createAndInit( + "SELECT OVERLAY(name PLACING 'XYZ' FROM 7 FOR 3) AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals( + "seatunXYZ", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testDeepNestedFunctions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT UPPER(TRIM(REPLACE(LOWER(name), 'sea', 'lake'))) AS res " + + "FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals( + "LAKETUNNEL", singleField(engine, new Object[] {1, " SeaTunnel ", 25})); + engine.close(); + } + + @Test + void testComplexSelectWithManyExpressions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id, " + + "name, " + + "age, " + + "UPPER(name) AS upper_name, " + + "LOWER(name) AS lower_name, " + + "CHAR_LENGTH(name) AS name_len, " + + "age + 1 AS next_age, " + + "age * 2 AS double_age, " + + "age > 18 AS is_adult, " + + "CASE WHEN age >= 65 THEN 'senior' ELSE 'non-senior' END AS cat " + + "FROM test_table", + "test_table", + buildRowType()); + + List result = exec(engine, new Object[] {1, "Seatunnel", 30}); + Assertions.assertEquals(10, result.get(0).getArity()); + Assertions.assertEquals(1, result.get(0).getField(0)); + Assertions.assertEquals("Seatunnel", result.get(0).getField(1)); + Assertions.assertEquals(30, result.get(0).getField(2)); + Assertions.assertEquals("SEATUNNEL", result.get(0).getField(3)); + Assertions.assertEquals("seatunnel", result.get(0).getField(4)); + Assertions.assertEquals(9, result.get(0).getField(5)); + Assertions.assertEquals(31, result.get(0).getField(6)); + Assertions.assertEquals(60, result.get(0).getField(7)); + Assertions.assertEquals(true, result.get(0).getField(8)); + engine.close(); + } + + @Test + void testWhereWithBooleanExpression() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table WHERE (age > 18) = true", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "B", 10}).size()); + engine.close(); + } + + @Test + void testWhereMultipleOrConditions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table " + + "WHERE name = 'Alice' OR name = 'Bob' OR name = 'Charlie'", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "Bob", 30}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {3, "Charlie", 35}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {4, "Dave", 40}).size()); + engine.close(); + } + + @Test + void testBetweenWithStrings() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE name BETWEEN 'A' AND 'M'", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Zeta", 25}).size()); + engine.close(); + } + + @Test + void testTableIdPreservedWithFilter() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table WHERE age > 18", "test_table", buildRowType()); + + SeaTunnelRow input = new SeaTunnelRow(new Object[] {1, "Alice", 25}); + input.setTableId("my_source"); + List result = engine.execute(input); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals("my_source", result.get(0).getTableId()); + engine.close(); + } + + @Test + void testTableIdPreservedWhenFiltered() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table WHERE age > 30", "test_table", buildRowType()); + + SeaTunnelRow input = new SeaTunnelRow(new Object[] {1, "Alice", 25}); + input.setTableId("my_source"); + List result = engine.execute(input); + Assertions.assertTrue(result.isEmpty()); + engine.close(); + } + + @Test + void testDefaultTableId() { + CalciteSQLEngine engine = + createAndInit("SELECT id FROM test_table", "test_table", buildRowType()); + + SeaTunnelRow input = new SeaTunnelRow(new Object[] {1, "A", 25}); + List result = engine.execute(input); + Assertions.assertEquals("", result.get(0).getTableId()); + engine.close(); + } + + @Test + void testEngineReuseStressTest() { + CalciteSQLEngine engine = + createAndInit( + "SELECT UPPER(name) AS res FROM test_table", "test_table", buildRowType()); + + for (int i = 0; i < 1000; i++) { + Object result = singleField(engine, new Object[] {i, "test_" + i, i % 100}); + Assertions.assertEquals("TEST_" + i, result); + } + engine.close(); + } + + @Test + void testDifferentTableNames() { + CalciteSQLEngine engine1 = + createAndInit("SELECT id FROM source_data", "source_data", buildRowType()); + CalciteSQLEngine engine2 = + createAndInit("SELECT id FROM sink_data", "sink_data", buildRowType()); + + Assertions.assertEquals(1, singleField(engine1, new Object[] {1, "A", 25})); + Assertions.assertEquals(2, singleField(engine2, new Object[] {2, "B", 30})); + + engine1.close(); + engine2.close(); + } + + @Test + void testLongTableName() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 10; i++) { + sb.append("seatunnel_"); + } + sb.append("tbl"); + String longName = sb.toString(); + CalciteSQLEngine engine = + createAndInit("SELECT id FROM " + longName, longName, buildRowType()); + Assertions.assertEquals(1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testUnderscoreInTableName() { + CalciteSQLEngine engine = + createAndInit("SELECT id FROM my_source_table", "my_source_table", buildRowType()); + Assertions.assertEquals(1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testNumericTableName() { + CalciteSQLEngine engine = createAndInit("SELECT id FROM t123", "t123", buildRowType()); + Assertions.assertEquals(1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testColumnNameWithUnderscore() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"user_id", "user_name", "user_age"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, BasicType.STRING_TYPE, BasicType.INT_TYPE + }); + CalciteSQLEngine engine = createAndInit("SELECT user_id, user_name FROM t", "t", rt); + List result = exec(engine, new Object[] {1, "Alice", 25}); + Assertions.assertEquals(1, result.get(0).getField(0)); + Assertions.assertEquals("Alice", result.get(0).getField(1)); + engine.close(); + } + + @Test + void testMixedCaseColumnName() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"userId", "userName"}, + new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.STRING_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT userId, userName FROM t", "t", rt); + List result = exec(engine, new Object[] {1, "Alice"}); + Assertions.assertEquals(1, result.get(0).getField(0)); + Assertions.assertEquals("Alice", result.get(0).getField(1)); + engine.close(); + } + + @Test + void testSelectStarWithWhere() { + CalciteSQLEngine engine = + createAndInit( + "SELECT * FROM test_table WHERE age > 18 AND name IS NOT NULL", + "test_table", + buildRowType()); + List result = exec(engine, new Object[] {1, "Alice", 25}); + Assertions.assertEquals(3, result.get(0).getArity()); + Assertions.assertEquals(1, result.get(0).getField(0)); + engine.close(); + } + + @Test + void testSelectStarFilterAll() { + CalciteSQLEngine engine = + createAndInit("SELECT * FROM test_table WHERE 1 = 0", "test_table", buildRowType()); + Assertions.assertTrue(exec(engine, new Object[] {1, "A", 25}).isEmpty()); + engine.close(); + } + + @Test + void testMultipleAliasesOnSameExpression() { + CalciteSQLEngine engine = + createAndInit( + "SELECT age + 1 AS next1, age + 1 AS next2 FROM test_table", + "test_table", + buildRowType()); + List result = exec(engine, new Object[] {1, "A", 25}); + Assertions.assertEquals(26, result.get(0).getField(0)); + Assertions.assertEquals(26, result.get(0).getField(1)); + engine.close(); + } + + @Test + void testLiteralTrue() { + CalciteSQLEngine engine = + createAndInit("SELECT TRUE AS flag FROM test_table", "test_table", buildRowType()); + Assertions.assertEquals(true, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testLiteralFalse() { + CalciteSQLEngine engine = + createAndInit("SELECT FALSE AS flag FROM test_table", "test_table", buildRowType()); + Assertions.assertEquals(false, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testNegativeConstant() { + CalciteSQLEngine engine = + createAndInit("SELECT -1 AS neg FROM test_table", "test_table", buildRowType()); + Assertions.assertEquals(-1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testStringConcatWithConstants() { + CalciteSQLEngine engine = + createAndInit( + "SELECT 'prefix-' || name || '-suffix' AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals( + "prefix-seatunnel-suffix", singleField(engine, new Object[] {1, "seatunnel", 25})); + engine.close(); + } + + @Test + void testArithmeticWithConstants() { + CalciteSQLEngine engine = + createAndInit( + "SELECT age + 100 - 50 * 2 AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals(25, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testNestedCoalesce() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"a", "b"}, + new SeaTunnelDataType[] {BasicType.STRING_TYPE, BasicType.STRING_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT COALESCE(a, COALESCE(b, 'fallback')) AS res FROM t", "t", rt); + Assertions.assertEquals("first", singleField(engine, new Object[] {"first", "second"})); + Assertions.assertEquals("second", singleField(engine, new Object[] {null, "second"})); + Assertions.assertEquals("fallback", singleField(engine, new Object[] {null, null})); + engine.close(); + } + + @Test + void testCaseWhenWithStringFunction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CASE WHEN CHAR_LENGTH(name) > 5 THEN UPPER(name) " + + "ELSE LOWER(name) END AS res FROM test_table", + "test_table", + buildRowType()); + Assertions.assertEquals( + "SEATUNNEL", singleField(engine, new Object[] {1, "Seatunnel", 25})); + Assertions.assertEquals("zeta", singleField(engine, new Object[] {2, "Zeta", 25})); + engine.close(); + } + + @Test + void testWhereWithNullSafeCheck() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table " + + "WHERE name IS NOT NULL AND CHAR_LENGTH(name) > 0", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, null, 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {3, "", 25}).size()); + engine.close(); + } + + @Test + void testWhereWithNestedOr() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table " + + "WHERE (age = 10 OR age = 20) OR (age = 30 OR age = 40)", + "test_table", + buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "A", 10}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {2, "B", 30}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {3, "C", 25}).size()); + engine.close(); + } + + @Test + void testWhereWithMixedAndOr() { + CalciteSQLEngine engine = + createAndInit( + "SELECT id FROM test_table " + + "WHERE (age > 20 OR age < 10) AND name LIKE 'A%'", + "test_table", buildRowType()); + Assertions.assertEquals(1, exec(engine, new Object[] {1, "Alice", 25}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {2, "Alice", 15}).size()); + Assertions.assertEquals(1, exec(engine, new Object[] {3, "Alice", 5}).size()); + Assertions.assertEquals(0, exec(engine, new Object[] {4, "Bob", 25}).size()); + engine.close(); + } + + @Test + void testInitCalledTwice() { + CalciteSQLEngine engine = + new CalciteSQLEngine("SELECT id FROM test_table", "test_table", buildRowType()); + engine.init(); + engine.init(); + Assertions.assertEquals(1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + } + + @Test + void testCloseCalledTwice() { + CalciteSQLEngine engine = + createAndInit("SELECT id FROM test_table", "test_table", buildRowType()); + Assertions.assertEquals(1, singleField(engine, new Object[] {1, "A", 25})); + engine.close(); + engine.close(); + } + + @Test + void testReinitWithDifferentSql() { + CalciteSQLEngine engine1 = + new CalciteSQLEngine("SELECT id FROM test_table", "test_table", buildRowType()); + engine1.init(); + Assertions.assertEquals(1, singleField(engine1, new Object[] {1, "A", 25})); + engine1.close(); + + CalciteSQLEngine engine2 = + new CalciteSQLEngine("SELECT name FROM test_table", "test_table", buildRowType()); + engine2.init(); + Assertions.assertEquals("Alice", singleField(engine2, new Object[] {1, "Alice", 25})); + engine2.close(); + } + + @Test + void testExpOne() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT EXP(val) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {1.0}); + Assertions.assertEquals(Math.E, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testLnOne() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT LN(val) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {1.0}); + Assertions.assertEquals(0.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testLog10One() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT LOG10(val) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {1.0}); + Assertions.assertEquals(0.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testPowerOneBase() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"base_val", "exponent"}, + new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE, BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = + createAndInit("SELECT POWER(base_val, exponent) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {1.0, 100.0}); + Assertions.assertEquals(1.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testSqrtPerfectSquare() { + SeaTunnelRowType rt = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.DOUBLE_TYPE}); + CalciteSQLEngine engine = createAndInit("SELECT SQRT(val) AS res FROM t", "t", rt); + Object result = singleField(engine, new Object[] {10000.0}); + Assertions.assertEquals(100.0, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + private static final String FRAUD_JSON = + "{" + + "\"request_id\":\"req-0042\"," + + "\"model\":\"fraud-detection-v3\"," + + "\"timestamp\":\"2026-06-11T15:30:00Z\"," + + "\"input\":{" + + " \"user_id\":\"U123\"," + + " \"device\":{\"type\":\"mobile\",\"os\":\"iOS 19\",\"ip\":\"10.0.0.1\"}" + + "}," + + "\"predictions\":[" + + " {" + + " \"rule_id\":\"R001\",\"rule_name\":\"high_freq_small_amount\"," + + " \"score\":0.92,\"label\":\"FRAUD\"," + + " \"evidence\":[" + + " {\"feature\":\"txn_count_1h\",\"value\":47,\"threshold\":20,\"contrib\":0.35}," + + " {\"feature\":\"avg_amount_1h\",\"value\":12.5,\"threshold\":50,\"contrib\":0.28}," + + " {\"feature\":\"distinct_merchant\",\"value\":15,\"threshold\":5,\"contrib\":0.29}" + + " ]" + + " }," + + " {" + + " \"rule_id\":\"R002\",\"rule_name\":\"geo_anomaly\"," + + " \"score\":0.45,\"label\":\"NORMAL\"," + + " \"evidence\":[" + + " {\"feature\":\"geo_distance_km\",\"value\":8.2,\"threshold\":100,\"contrib\":0.45}" + + " ]" + + " }," + + " {" + + " \"rule_id\":\"R003\",\"rule_name\":\"device_fingerprint_anomaly\"," + + " \"score\":0.78,\"label\":\"SUSPECT\"," + + " \"evidence\":[" + + " {\"feature\":\"device_age_days\",\"value\":1,\"threshold\":7,\"contrib\":0.40}," + + " {\"feature\":\"fingerprint_change\",\"value\":1,\"threshold\":0,\"contrib\":0.38}" + + " ]" + + " }" + + "]" + + "}"; + + private SeaTunnelRowType jsonRowType() { + return new SeaTunnelRowType( + new String[] {"raw"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + } + + @Test + void testJsonValueTopLevelScalar() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.request_id') AS req_id FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("req-0042", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueModel() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.model') AS model_name FROM t", + "t", + jsonRowType()); + Assertions.assertEquals( + "fraud-detection-v3", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueTimestamp() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.timestamp') AS ts FROM t", "t", jsonRowType()); + Assertions.assertEquals( + "2026-06-11T15:30:00Z", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueNestedUserId() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.input.user_id') AS uid FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("U123", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueDeepNestedDeviceType() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.input.device.type') AS dev_type FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("mobile", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueDeepNestedDeviceOs() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.input.device.os') AS dev_os FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("iOS 19", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueDeepNestedDeviceIp() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.input.device.ip') AS dev_ip FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("10.0.0.1", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueArrayIndexRuleId() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].rule_id') AS rid FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("R001", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueArrayIndexRuleName() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].rule_name') AS rn FROM t", + "t", + jsonRowType()); + Assertions.assertEquals( + "high_freq_small_amount", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueArrayIndexScore() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].score') AS score FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("0.92", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueArrayIndexLabel() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].label') AS lbl FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("FRAUD", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueSecondPrediction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[1].rule_id') AS rid FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("R002", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueThirdPrediction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[2].rule_id') AS rid FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("R003", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueThirdPredictionLabel() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[2].label') AS lbl FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("SUSPECT", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueDeepNestedEvidence() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].evidence[0].feature') AS feat FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("txn_count_1h", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueEvidenceValue() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].evidence[0].value') AS val FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("47", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueEvidenceThreshold() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].evidence[0].threshold') AS th FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("20", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueEvidenceContrib() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].evidence[0].contrib') AS c FROM t", + "t", + jsonRowType()); + Assertions.assertEquals("0.35", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueCrossArrayEvidence() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[2].evidence[1].feature') AS feat FROM t", + "t", + jsonRowType()); + Assertions.assertEquals( + "fingerprint_change", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueNonExistentPath() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.nonexistent') AS missing FROM t", + "t", + jsonRowType()); + Assertions.assertNull(singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueNonExistentNested() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.input.address.city') AS city FROM t", + "t", + jsonRowType()); + Assertions.assertNull(singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueOutOfBoundsIndex() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[99].rule_id') AS rid FROM t", + "t", + jsonRowType()); + Assertions.assertNull(singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonValueFromNull() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.request_id') AS req_id FROM t", + "t", + jsonRowType()); + Assertions.assertNull(singleField(engine, new Object[] {(Object) null})); + engine.close(); + } + + @Test + void testJsonQueryPredictionsArray() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_QUERY(raw, '$.predictions') AS preds FROM t", + "t", + jsonRowType()); + String result = (String) singleField(engine, new Object[] {FRAUD_JSON}); + Assertions.assertNotNull(result); + Assertions.assertTrue(result.startsWith("[")); + Assertions.assertTrue(result.contains("R001")); + Assertions.assertTrue(result.contains("R002")); + Assertions.assertTrue(result.contains("R003")); + engine.close(); + } + + @Test + void testJsonQueryInputObject() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_QUERY(raw, '$.input') AS inp FROM t", "t", jsonRowType()); + String result = (String) singleField(engine, new Object[] {FRAUD_JSON}); + Assertions.assertNotNull(result); + Assertions.assertTrue(result.contains("U123")); + Assertions.assertTrue(result.contains("mobile")); + engine.close(); + } + + @Test + void testJsonQueryDeviceObject() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_QUERY(raw, '$.input.device') AS dev FROM t", + "t", + jsonRowType()); + String result = (String) singleField(engine, new Object[] {FRAUD_JSON}); + Assertions.assertNotNull(result); + Assertions.assertTrue(result.contains("mobile")); + Assertions.assertTrue(result.contains("iOS 19")); + Assertions.assertTrue(result.contains("10.0.0.1")); + engine.close(); + } + + @Test + void testJsonQuerySinglePrediction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_QUERY(raw, '$.predictions[0]') AS pred FROM t", + "t", + jsonRowType()); + String result = (String) singleField(engine, new Object[] {FRAUD_JSON}); + Assertions.assertNotNull(result); + Assertions.assertTrue(result.contains("R001")); + Assertions.assertTrue(result.contains("FRAUD")); + Assertions.assertTrue(result.contains("0.92")); + engine.close(); + } + + @Test + void testJsonQueryEvidenceArray() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_QUERY(raw, '$.predictions[0].evidence') AS ev FROM t", + "t", + jsonRowType()); + String result = (String) singleField(engine, new Object[] {FRAUD_JSON}); + Assertions.assertNotNull(result); + Assertions.assertTrue(result.startsWith("[")); + Assertions.assertTrue(result.contains("txn_count_1h")); + Assertions.assertTrue(result.contains("avg_amount_1h")); + Assertions.assertTrue(result.contains("distinct_merchant")); + engine.close(); + } + + @Test + void testJsonMultiFieldExtraction() { + CalciteSQLEngine engine = + createAndInit( + "SELECT " + + "JSON_VALUE(raw, '$.request_id') AS req_id, " + + "JSON_VALUE(raw, '$.input.user_id') AS uid, " + + "JSON_VALUE(raw, '$.predictions[0].rule_id') AS rid, " + + "JSON_VALUE(raw, '$.predictions[0].rule_name') AS rn, " + + "JSON_VALUE(raw, '$.predictions[0].score') AS score, " + + "JSON_VALUE(raw, '$.predictions[0].label') AS lbl " + + "FROM t", + "t", + jsonRowType()); + + List result = exec(engine, new Object[] {FRAUD_JSON}); + Assertions.assertEquals(1, result.size()); + SeaTunnelRow row = result.get(0); + Assertions.assertEquals("req-0042", row.getField(0)); + Assertions.assertEquals("U123", row.getField(1)); + Assertions.assertEquals("R001", row.getField(2)); + Assertions.assertEquals("high_freq_small_amount", row.getField(3)); + Assertions.assertEquals("0.92", row.getField(4)); + Assertions.assertEquals("FRAUD", row.getField(5)); + engine.close(); + } + + @Test + void testJsonExtractTopRiskRule() { + CalciteSQLEngine engine = + createAndInit( + "SELECT " + + "JSON_VALUE(raw, '$.request_id') AS req_id, " + + "JSON_VALUE(raw, '$.input.user_id') AS uid, " + + "JSON_VALUE(raw, '$.predictions[0].rule_id') AS rid, " + + "JSON_VALUE(raw, '$.predictions[0].rule_name') AS rn, " + + "CAST(JSON_VALUE(raw, '$.predictions[0].score') AS DOUBLE) AS score, " + + "JSON_VALUE(raw, '$.predictions[0].label') AS lbl " + + "FROM t " + + "WHERE JSON_VALUE(raw, '$.predictions[0].label') IN ('FRAUD', 'SUSPECT')", + "t", + jsonRowType()); + + List result = exec(engine, new Object[] {FRAUD_JSON}); + Assertions.assertEquals(1, result.size()); + SeaTunnelRow row = result.get(0); + Assertions.assertEquals("req-0042", row.getField(0)); + Assertions.assertEquals("U123", row.getField(1)); + Assertions.assertEquals("R001", row.getField(2)); + Assertions.assertEquals("high_freq_small_amount", row.getField(3)); + Assertions.assertEquals(0.92, ((Number) row.getField(4)).doubleValue(), 0.001); + Assertions.assertEquals("FRAUD", row.getField(5)); + engine.close(); + } + + @Test + void testJsonFilterByLabel() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[1].label') AS lbl FROM t " + + "WHERE JSON_VALUE(raw, '$.predictions[1].label') = 'NORMAL'", + "t", + jsonRowType()); + + List result = exec(engine, new Object[] {FRAUD_JSON}); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals("NORMAL", result.get(0).getField(0)); + engine.close(); + } + + @Test + void testJsonFilterByLabelNotMatch() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[1].label') AS lbl FROM t " + + "WHERE JSON_VALUE(raw, '$.predictions[1].label') = 'FRAUD'", + "t", + jsonRowType()); + + List result = exec(engine, new Object[] {FRAUD_JSON}); + Assertions.assertTrue(result.isEmpty()); + engine.close(); + } + + @Test + void testJsonCastScoreToDouble() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(JSON_VALUE(raw, '$.predictions[0].score') AS DOUBLE) AS score FROM t", + "t", + jsonRowType()); + + Object result = singleField(engine, new Object[] {FRAUD_JSON}); + Assertions.assertEquals(0.92, ((Number) result).doubleValue(), 0.001); + engine.close(); + } + + @Test + void testJsonCastScoreFilterHighRisk() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].rule_id') AS rid FROM t " + + "WHERE CAST(JSON_VALUE(raw, '$.predictions[0].score') AS DOUBLE) > 0.8", + "t", + jsonRowType()); + + Assertions.assertEquals("R001", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonCastScoreFilterLowRisk() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[1].rule_id') AS rid FROM t " + + "WHERE CAST(JSON_VALUE(raw, '$.predictions[1].score') AS DOUBLE) > 0.8", + "t", + jsonRowType()); + + Assertions.assertTrue(exec(engine, new Object[] {FRAUD_JSON}).isEmpty()); + engine.close(); + } + + @Test + void testJsonCastEvidenceValueToInt() { + CalciteSQLEngine engine = + createAndInit( + "SELECT CAST(JSON_VALUE(raw, '$.predictions[0].evidence[0].value') AS INTEGER) AS val FROM t", + "t", + jsonRowType()); + Assertions.assertEquals(47, singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonEvidenceExceedThresholdCheck() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].evidence[0].feature') AS feat FROM t " + + "WHERE CAST(JSON_VALUE(raw, '$.predictions[0].evidence[0].value') AS INTEGER) " + + " > CAST(JSON_VALUE(raw, '$.predictions[0].evidence[0].threshold') AS INTEGER)", + "t", + jsonRowType()); + Assertions.assertEquals("txn_count_1h", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonEvidenceBelowThresholdFiltered() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.predictions[0].evidence[1].feature') AS feat FROM t " + + "WHERE CAST(JSON_VALUE(raw, '$.predictions[0].evidence[1].value') AS DOUBLE) " + + " > CAST(JSON_VALUE(raw, '$.predictions[0].evidence[1].threshold') AS DOUBLE)", + "t", + jsonRowType()); + Assertions.assertTrue(exec(engine, new Object[] {FRAUD_JSON}).isEmpty()); + engine.close(); + } + + @Test + void testJsonComplexFraudRiskReport() { + CalciteSQLEngine engine = + createAndInit( + "SELECT " + + "JSON_VALUE(raw, '$.request_id') AS req_id, " + + "JSON_VALUE(raw, '$.input.user_id') AS uid, " + + "JSON_VALUE(raw, '$.input.device.type') AS dev_type, " + + "JSON_VALUE(raw, '$.input.device.ip') AS dev_ip, " + + "JSON_VALUE(raw, '$.predictions[0].rule_name') AS top_rule, " + + "CAST(JSON_VALUE(raw, '$.predictions[0].score') AS DOUBLE) AS top_score, " + + "JSON_VALUE(raw, '$.predictions[0].label') AS top_label, " + + "TRIM(CASE WHEN JSON_VALUE(raw, '$.predictions[0].label') = 'FRAUD' " + + " THEN 'BLOCK' ELSE 'PASS' END) AS action " + + "FROM t", + "t", + jsonRowType()); + + List result = exec(engine, new Object[] {FRAUD_JSON}); + Assertions.assertEquals(1, result.size()); + SeaTunnelRow row = result.get(0); + Assertions.assertEquals("req-0042", row.getField(0)); + Assertions.assertEquals("U123", row.getField(1)); + Assertions.assertEquals("mobile", row.getField(2)); + Assertions.assertEquals("10.0.0.1", row.getField(3)); + Assertions.assertEquals("high_freq_small_amount", row.getField(4)); + Assertions.assertEquals(0.92, ((Number) row.getField(5)).doubleValue(), 0.001); + Assertions.assertEquals("FRAUD", row.getField(6)); + Assertions.assertEquals("BLOCK", row.getField(7)); + engine.close(); + } + + @Test + void testJsonWithNormalLabelAction() { + String normalJson = + "{\"request_id\":\"req-0099\",\"model\":\"fraud-v3\"," + + "\"input\":{\"user_id\":\"U456\",\"device\":{\"type\":\"desktop\",\"os\":\"Windows\",\"ip\":\"192.168.1.1\"}}," + + "\"predictions\":[" + + "{\"rule_id\":\"R001\",\"rule_name\":\"high_freq\",\"score\":0.15,\"label\":\"NORMAL\",\"evidence\":[]}" + + "]}"; + + CalciteSQLEngine engine = + createAndInit( + "SELECT " + + "JSON_VALUE(raw, '$.request_id') AS req_id, " + + "JSON_VALUE(raw, '$.input.user_id') AS uid, " + + "TRIM(CASE WHEN JSON_VALUE(raw, '$.predictions[0].label') = 'FRAUD' " + + " THEN 'BLOCK' ELSE 'PASS' END) AS action " + + "FROM t", + "t", + jsonRowType()); + + List result = exec(engine, new Object[] {normalJson}); + Assertions.assertEquals("req-0099", result.get(0).getField(0)); + Assertions.assertEquals("U456", result.get(0).getField(1)); + Assertions.assertEquals("PASS", result.get(0).getField(2)); + engine.close(); + } + + @Test + void testJsonExtractAllThreePredictions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT " + + "JSON_VALUE(raw, '$.predictions[0].rule_id') AS r0, " + + "JSON_VALUE(raw, '$.predictions[0].label') AS l0, " + + "JSON_VALUE(raw, '$.predictions[1].rule_id') AS r1, " + + "JSON_VALUE(raw, '$.predictions[1].label') AS l1, " + + "JSON_VALUE(raw, '$.predictions[2].rule_id') AS r2, " + + "JSON_VALUE(raw, '$.predictions[2].label') AS l2 " + + "FROM t", + "t", + jsonRowType()); + + List result = exec(engine, new Object[] {FRAUD_JSON}); + SeaTunnelRow row = result.get(0); + Assertions.assertEquals("R001", row.getField(0)); + Assertions.assertEquals("FRAUD", row.getField(1)); + Assertions.assertEquals("R002", row.getField(2)); + Assertions.assertEquals("NORMAL", row.getField(3)); + Assertions.assertEquals("R003", row.getField(4)); + Assertions.assertEquals("SUSPECT", row.getField(5)); + engine.close(); + } + + @Test + void testJsonMaxScoreAmongPredictions() { + CalciteSQLEngine engine = + createAndInit( + "SELECT " + + "CASE " + + " WHEN CAST(JSON_VALUE(raw, '$.predictions[0].score') AS DOUBLE) >= " + + " CAST(JSON_VALUE(raw, '$.predictions[1].score') AS DOUBLE) " + + " AND CAST(JSON_VALUE(raw, '$.predictions[0].score') AS DOUBLE) >= " + + " CAST(JSON_VALUE(raw, '$.predictions[2].score') AS DOUBLE) " + + " THEN JSON_VALUE(raw, '$.predictions[0].rule_id') " + + " WHEN CAST(JSON_VALUE(raw, '$.predictions[1].score') AS DOUBLE) >= " + + " CAST(JSON_VALUE(raw, '$.predictions[2].score') AS DOUBLE) " + + " THEN JSON_VALUE(raw, '$.predictions[1].rule_id') " + + " ELSE JSON_VALUE(raw, '$.predictions[2].rule_id') " + + "END AS max_rule " + + "FROM t", + "t", + jsonRowType()); + + Assertions.assertEquals("R001", singleField(engine, new Object[] {FRAUD_JSON})); + engine.close(); + } + + @Test + void testJsonWithSimpleObject() { + String simpleJson = "{\"name\":\"seatunnel\",\"version\":\"3.0.0\",\"type\":\"etl\"}"; + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.name') AS n, " + + "JSON_VALUE(raw, '$.version') AS v FROM t", + "t", + jsonRowType()); + + List result = exec(engine, new Object[] {simpleJson}); + Assertions.assertEquals("seatunnel", result.get(0).getField(0)); + Assertions.assertEquals("3.0.0", result.get(0).getField(1)); + engine.close(); + } + + @Test + void testJsonEmptyObject() { + CalciteSQLEngine engine = + createAndInit("SELECT JSON_VALUE(raw, '$.name') AS n FROM t", "t", jsonRowType()); + Assertions.assertNull(singleField(engine, new Object[] {"{}"})); + engine.close(); + } + + @Test + void testJsonEmptyArray() { + String json = "{\"items\":[]}"; + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.items[0]') AS first_item FROM t", + "t", + jsonRowType()); + Assertions.assertNull(singleField(engine, new Object[] {json})); + engine.close(); + } + + @Test + void testJsonMultipleRowsReuse() { + CalciteSQLEngine engine = + createAndInit( + "SELECT JSON_VALUE(raw, '$.request_id') AS req_id, " + + "JSON_VALUE(raw, '$.predictions[0].label') AS lbl FROM t", + "t", + jsonRowType()); + + List r1 = exec(engine, new Object[] {FRAUD_JSON}); + Assertions.assertEquals("req-0042", r1.get(0).getField(0)); + Assertions.assertEquals("FRAUD", r1.get(0).getField(1)); + + String anotherJson = + "{\"request_id\":\"req-0100\",\"predictions\":[{\"label\":\"NORMAL\"}]}"; + List r2 = exec(engine, new Object[] {anotherJson}); + Assertions.assertEquals("req-0100", r2.get(0).getField(0)); + Assertions.assertEquals("NORMAL", r2.get(0).getField(1)); + + engine.close(); + } + + @Test + void testJsonOutputSchema() { + CalciteSQLEngine engine = + createAndInit( + "SELECT " + + "JSON_VALUE(raw, '$.request_id') AS req_id, " + + "JSON_VALUE(raw, '$.input.user_id') AS uid, " + + "CAST(JSON_VALUE(raw, '$.predictions[0].score') AS DOUBLE) AS score " + + "FROM t", + "t", + jsonRowType()); + + SeaTunnelRowType outType = engine.getOutputRowType(); + Assertions.assertEquals(3, outType.getTotalFields()); + Assertions.assertEquals("req_id", outType.getFieldName(0)); + Assertions.assertEquals("uid", outType.getFieldName(1)); + Assertions.assertEquals("score", outType.getFieldName(2)); + Assertions.assertEquals(SqlType.STRING, outType.getFieldType(0).getSqlType()); + Assertions.assertEquals(SqlType.STRING, outType.getFieldType(1).getSqlType()); + Assertions.assertEquals(SqlType.DOUBLE, outType.getFieldType(2).getSqlType()); + engine.close(); + } + + private SeaTunnelRowType twoVectorRowType() { + return new SeaTunnelRowType( + new String[] {"vec1", "vec2"}, + new SeaTunnelDataType[] { + VectorType.VECTOR_FLOAT_TYPE, VectorType.VECTOR_FLOAT_TYPE + }); + } + + private SeaTunnelRowType singleVectorRowType() { + return new SeaTunnelRowType( + new String[] {"vec"}, new SeaTunnelDataType[] {VectorType.VECTOR_FLOAT_TYPE}); + } + + private static ByteBuffer floatVec(Float... values) { + return VectorUtils.toByteBuffer(values); + } + + @Test + void testCosineDistanceIdentical() { + CalciteSQLEngine engine = + createAndInit( + "SELECT COSINE_DISTANCE(vec1, vec2) AS dist FROM t", + "t", + twoVectorRowType()); + ByteBuffer v = floatVec(1.0f, 2.0f, 3.0f); + Object result = singleField(engine, new Object[] {v, v.duplicate()}); + Assertions.assertEquals(0.0, ((Number) result).doubleValue(), 1e-9); + engine.close(); + } + + @Test + void testCosineDistanceOrthogonal() { + CalciteSQLEngine engine = + createAndInit( + "SELECT COSINE_DISTANCE(vec1, vec2) AS dist FROM t", + "t", + twoVectorRowType()); + Object result = + singleField(engine, new Object[] {floatVec(1.0f, 0.0f), floatVec(0.0f, 1.0f)}); + Assertions.assertEquals(1.0, ((Number) result).doubleValue(), 1e-9); + engine.close(); + } + + @Test + void testL1Distance() { + CalciteSQLEngine engine = + createAndInit( + "SELECT L1_DISTANCE(vec1, vec2) AS dist FROM t", "t", twoVectorRowType()); + Object result = + singleField( + engine, + new Object[] {floatVec(2.0f, 4.0f, 6.0f), floatVec(1.0f, 2.0f, 3.0f)}); + Assertions.assertEquals(6.0, ((Number) result).doubleValue(), 1e-9); + engine.close(); + } + + @Test + void testL2Distance() { + CalciteSQLEngine engine = + createAndInit( + "SELECT L2_DISTANCE(vec1, vec2) AS dist FROM t", "t", twoVectorRowType()); + Object result = + singleField( + engine, + new Object[] {floatVec(2.0f, 4.0f, 4.0f), floatVec(1.0f, 2.0f, 2.0f)}); + Assertions.assertEquals(3.0, ((Number) result).doubleValue(), 1e-9); + engine.close(); + } + + @Test + void testVectorDims() { + CalciteSQLEngine engine = + createAndInit("SELECT VECTOR_DIMS(vec) AS dims FROM t", "t", singleVectorRowType()); + Object result = singleField(engine, new Object[] {floatVec(1.0f, 2.0f, 3.0f)}); + Assertions.assertEquals(3, ((Number) result).intValue()); + engine.close(); + } + + @Test + void testVectorNorm() { + CalciteSQLEngine engine = + createAndInit("SELECT VECTOR_NORM(vec) AS norm FROM t", "t", singleVectorRowType()); + Object result = singleField(engine, new Object[] {floatVec(1.0f, 2.0f, 2.0f)}); + Assertions.assertEquals(3.0, ((Number) result).doubleValue(), 1e-9); + engine.close(); + } + + @Test + void testInnerProduct() { + CalciteSQLEngine engine = + createAndInit( + "SELECT INNER_PRODUCT(vec1, vec2) AS ip FROM t", "t", twoVectorRowType()); + Object result = + singleField( + engine, + new Object[] {floatVec(1.0f, 2.0f, 3.0f), floatVec(7.0f, 8.0f, 9.0f)}); + Assertions.assertEquals(50.0, ((Number) result).doubleValue(), 1e-9); + engine.close(); + } + + @Test + void testVectorReduceTruncate() { + CalciteSQLEngine engine = + createAndInit( + "SELECT VECTOR_REDUCE(vec, 2, 'TRUNCATE') AS reduced FROM t", + "t", + singleVectorRowType()); + Object result = singleField(engine, new Object[] {floatVec(1.0f, 2.0f, 3.0f, 4.0f)}); + Assertions.assertNotNull(result); + Assertions.assertEquals( + VectorType.VECTOR_FLOAT_TYPE, engine.getOutputRowType().getFieldType(0)); + Float[] reduced = VectorUtils.toFloatArray((ByteBuffer) result); + Assertions.assertArrayEquals(new Float[] {1.0f, 2.0f}, reduced); + engine.close(); + } + + @Test + void testVectorNormalize() { + CalciteSQLEngine engine = + createAndInit( + "SELECT VECTOR_NORMALIZE(vec) AS nvec FROM t", "t", singleVectorRowType()); + Object result = singleField(engine, new Object[] {floatVec(3.0f, 4.0f)}); + Assertions.assertNotNull(result); + Assertions.assertEquals( + VectorType.VECTOR_FLOAT_TYPE, engine.getOutputRowType().getFieldType(0)); + Float[] normalized = VectorUtils.toFloatArray((ByteBuffer) result); + Assertions.assertEquals(2, normalized.length); + double norm = Math.sqrt(normalized[0] * normalized[0] + normalized[1] * normalized[1]); + Assertions.assertEquals(1.0, norm, 1e-6); + engine.close(); + } + + @Test + void testVectorAliasPreservesType() { + CalciteSQLEngine engine = + createAndInit("SELECT vec AS alias_vec FROM t", "t", singleVectorRowType()); + Object result = singleField(engine, new Object[] {floatVec(1.0f, 2.0f, 3.0f)}); + Assertions.assertInstanceOf(ByteBuffer.class, result); + Assertions.assertEquals( + VectorType.VECTOR_FLOAT_TYPE, engine.getOutputRowType().getFieldType(0)); + Float[] values = VectorUtils.toFloatArray((ByteBuffer) result); + Assertions.assertArrayEquals(new Float[] {1.0f, 2.0f, 3.0f}, values); + engine.close(); + } + + @Test + void testSelectStarPreservesVectorType() { + CalciteSQLEngine engine = createAndInit("SELECT * FROM t", "t", singleVectorRowType()); + List results = exec(engine, new Object[] {floatVec(1.0f, 2.0f, 3.0f)}); + Assertions.assertEquals(1, results.size()); + Assertions.assertEquals( + VectorType.VECTOR_FLOAT_TYPE, engine.getOutputRowType().getFieldType(0)); + Assertions.assertInstanceOf(ByteBuffer.class, results.get(0).getField(0)); + Float[] values = VectorUtils.toFloatArray((ByteBuffer) results.get(0).getField(0)); + Assertions.assertArrayEquals(new Float[] {1.0f, 2.0f, 3.0f}, values); + engine.close(); + } + + @Test + void testRowKindPropagation() { + CalciteSQLEngine engine = + createAndInit("SELECT id, name FROM test_table", "test_table", buildRowType()); + + for (RowKind kind : RowKind.values()) { + SeaTunnelRow input = new SeaTunnelRow(new Object[] {1, "Alice", 25}); + input.setRowKind(kind); + input.setTableId("db.schema.users"); + + List results = engine.execute(input); + Assertions.assertEquals(1, results.size()); + SeaTunnelRow output = results.get(0); + Assertions.assertEquals( + kind, output.getRowKind(), "RowKind should be preserved for " + kind); + Assertions.assertEquals("db.schema.users", output.getTableId()); + } + engine.close(); + } + + @Test + void testRowOptionsPropagation() { + CalciteSQLEngine engine = + createAndInit("SELECT id, name FROM test_table", "test_table", buildRowType()); + + SeaTunnelRow input = new SeaTunnelRow(new Object[] {1, "Alice", 25}); + input.setRowKind(RowKind.UPDATE_AFTER); + input.setTableId("mydb.mytable"); + Map options = new HashMap<>(); + options.put("partition", "p0"); + options.put("offset", "42"); + input.setOptions(options); + + List results = engine.execute(input); + Assertions.assertEquals(1, results.size()); + SeaTunnelRow output = results.get(0); + Assertions.assertEquals(RowKind.UPDATE_AFTER, output.getRowKind()); + Assertions.assertEquals("mydb.mytable", output.getTableId()); + Assertions.assertNotNull(output.getOptions()); + Assertions.assertEquals("p0", output.getOptions().get("partition")); + Assertions.assertEquals("42", output.getOptions().get("offset")); + engine.close(); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteTransformFactoryTest.java new file mode 100644 index 000000000000..3b8bf698aadc --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteTransformFactoryTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite; + +import org.apache.seatunnel.api.configuration.util.OptionRule; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class CalciteTransformFactoryTest { + + @Test + void testFactoryIdentifier() { + CalciteTransformFactory factory = new CalciteTransformFactory(); + Assertions.assertEquals("Calcite", factory.factoryIdentifier()); + } + + @Test + void testOptionRule() { + CalciteTransformFactory factory = new CalciteTransformFactory(); + OptionRule rule = factory.optionRule(); + Assertions.assertNotNull(rule); + Assertions.assertFalse(rule.getRequiredOptions().isEmpty()); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteTypeConverterTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteTypeConverterTest.java new file mode 100644 index 000000000000..d0c1dea8a775 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteTypeConverterTest.java @@ -0,0 +1,681 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite; + +import org.apache.seatunnel.shade.org.apache.calcite.avatica.util.TimeUnit; +import org.apache.seatunnel.shade.org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataType; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.seatunnel.shade.org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.seatunnel.shade.org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.seatunnel.shade.org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.seatunnel.shade.org.apache.calcite.sql.type.SqlTypeName; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; +import org.apache.seatunnel.api.table.type.VectorType; +import org.apache.seatunnel.transform.calcite.type.CalciteTypeConverter; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class CalciteTypeConverterTest { + + private static RelDataTypeFactory typeFactory; + + @BeforeAll + static void setUp() { + typeFactory = new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + } + + @Test + void testBooleanRoundTrip() { + assertRoundTrip(BasicType.BOOLEAN_TYPE, SqlTypeName.BOOLEAN); + } + + @Test + void testByteRoundTrip() { + assertRoundTrip(BasicType.BYTE_TYPE, SqlTypeName.TINYINT); + } + + @Test + void testShortRoundTrip() { + assertRoundTrip(BasicType.SHORT_TYPE, SqlTypeName.SMALLINT); + } + + @Test + void testIntRoundTrip() { + assertRoundTrip(BasicType.INT_TYPE, SqlTypeName.INTEGER); + } + + @Test + void testLongRoundTrip() { + assertRoundTrip(BasicType.LONG_TYPE, SqlTypeName.BIGINT); + } + + @Test + void testFloatRoundTrip() { + assertRoundTrip(BasicType.FLOAT_TYPE, SqlTypeName.REAL); + } + + @Test + void testDoubleRoundTrip() { + assertRoundTrip(BasicType.DOUBLE_TYPE, SqlTypeName.DOUBLE); + } + + @Test + void testStringRoundTrip() { + assertRoundTrip(BasicType.STRING_TYPE, SqlTypeName.VARCHAR); + } + + @Test + void testDecimalType() { + DecimalType decimal = new DecimalType(18, 6); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, decimal); + Assertions.assertEquals(SqlTypeName.DECIMAL, calciteType.getSqlTypeName()); + Assertions.assertEquals(18, calciteType.getPrecision()); + Assertions.assertEquals(6, calciteType.getScale()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertInstanceOf(DecimalType.class, back); + Assertions.assertEquals(18, ((DecimalType) back).getPrecision()); + Assertions.assertEquals(6, ((DecimalType) back).getScale()); + } + + @Test + void testDecimalSmallPrecision() { + DecimalType smallDecimal = new DecimalType(5, 2); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, smallDecimal); + Assertions.assertEquals(5, calciteType.getPrecision()); + Assertions.assertEquals(2, calciteType.getScale()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertEquals(5, ((DecimalType) back).getPrecision()); + Assertions.assertEquals(2, ((DecimalType) back).getScale()); + } + + @Test + void testDecimalMaxCalcitePrecision() { + DecimalType bigDecimal = new DecimalType(19, 10); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, bigDecimal); + Assertions.assertEquals(19, calciteType.getPrecision()); + Assertions.assertEquals(10, calciteType.getScale()); + } + + @Test + void testDecimalZeroScale() { + DecimalType intDecimal = new DecimalType(10, 0); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, intDecimal); + Assertions.assertEquals(10, calciteType.getPrecision()); + Assertions.assertEquals(0, calciteType.getScale()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertEquals(0, ((DecimalType) back).getScale()); + } + + @Test + void testDecimalMinPrecision() { + DecimalType minDecimal = new DecimalType(1, 0); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, minDecimal); + Assertions.assertEquals(1, calciteType.getPrecision()); + } + + @Test + void testDateRoundTrip() { + assertRoundTrip(LocalTimeType.LOCAL_DATE_TYPE, SqlTypeName.DATE); + } + + @Test + void testTimeRoundTrip() { + assertRoundTrip(LocalTimeType.LOCAL_TIME_TYPE, SqlTypeName.TIME); + } + + @Test + void testTimestampRoundTrip() { + assertRoundTrip(LocalTimeType.LOCAL_DATE_TIME_TYPE, SqlTypeName.TIMESTAMP); + } + + @Test + void testTimestampWithTimezone() { + RelDataType calciteType = + CalciteTypeConverter.toCalciteType( + typeFactory, LocalTimeType.OFFSET_DATE_TIME_TYPE); + Assertions.assertEquals( + SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, calciteType.getSqlTypeName()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, back); + } + + @Test + void testTimeWithLocalTimezoneReverse() { + RelDataType calciteType = typeFactory.createSqlType(SqlTypeName.TIME_WITH_LOCAL_TIME_ZONE); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertEquals(LocalTimeType.LOCAL_TIME_TYPE, back); + } + + @Test + void testBytesType() { + RelDataType calciteType = + CalciteTypeConverter.toCalciteType(typeFactory, PrimitiveByteArrayType.INSTANCE); + Assertions.assertEquals(SqlTypeName.VARBINARY, calciteType.getSqlTypeName()); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertEquals(PrimitiveByteArrayType.INSTANCE, back); + } + + @Test + void testBinaryReverseMapping() { + RelDataType binaryType = typeFactory.createSqlType(SqlTypeName.BINARY); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(binaryType); + Assertions.assertEquals(PrimitiveByteArrayType.INSTANCE, back); + } + + @Test + void testCharReverseMapping() { + RelDataType charType = typeFactory.createSqlType(SqlTypeName.CHAR); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(charType); + Assertions.assertEquals(BasicType.STRING_TYPE, back); + } + + @Test + void testCharWithPrecisionReverse() { + RelDataType charType = typeFactory.createSqlType(SqlTypeName.CHAR, 50); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(charType); + Assertions.assertEquals(BasicType.STRING_TYPE, back); + } + + @Test + void testVarcharWithPrecisionReverse() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR, 255); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(varcharType); + Assertions.assertEquals(BasicType.STRING_TYPE, back); + } + + @Test + void testNullType() { + RelDataType calciteType = + CalciteTypeConverter.toCalciteType(typeFactory, BasicType.VOID_TYPE); + Assertions.assertEquals(SqlTypeName.NULL, calciteType.getSqlTypeName()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertEquals(BasicType.VOID_TYPE, back); + } + + @Test + void testStringArrayForward() { + RelDataType calciteType = + CalciteTypeConverter.toCalciteType(typeFactory, ArrayType.STRING_ARRAY_TYPE); + Assertions.assertEquals(SqlTypeName.ARRAY, calciteType.getSqlTypeName()); + } + + @Test + void testIntArrayRoundTrip() { + RelDataType calciteType = + CalciteTypeConverter.toCalciteType(typeFactory, ArrayType.INT_ARRAY_TYPE); + Assertions.assertEquals(SqlTypeName.ARRAY, calciteType.getSqlTypeName()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertInstanceOf(ArrayType.class, back); + Assertions.assertEquals(BasicType.INT_TYPE, ((ArrayType) back).getElementType()); + } + + @Test + void testBooleanArrayForward() { + assertArrayForward(ArrayType.BOOLEAN_ARRAY_TYPE); + } + + @Test + void testByteArrayForward() { + assertArrayForward(ArrayType.BYTE_ARRAY_TYPE); + } + + @Test + void testShortArrayForward() { + assertArrayForward(ArrayType.SHORT_ARRAY_TYPE); + } + + @Test + void testLongArrayForward() { + assertArrayForward(ArrayType.LONG_ARRAY_TYPE); + } + + @Test + void testFloatArrayForward() { + assertArrayForward(ArrayType.FLOAT_ARRAY_TYPE); + } + + @Test + void testDoubleArrayForward() { + assertArrayForward(ArrayType.DOUBLE_ARRAY_TYPE); + } + + @Test + void testStringArrayRoundTrip() { + RelDataType calciteType = + CalciteTypeConverter.toCalciteType(typeFactory, ArrayType.STRING_ARRAY_TYPE); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertInstanceOf(ArrayType.class, back); + Assertions.assertEquals(BasicType.STRING_TYPE, ((ArrayType) back).getElementType()); + } + + @Test + void testLongArrayRoundTrip() { + RelDataType calciteType = + CalciteTypeConverter.toCalciteType(typeFactory, ArrayType.LONG_ARRAY_TYPE); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertInstanceOf(ArrayType.class, back); + Assertions.assertEquals(BasicType.LONG_TYPE, ((ArrayType) back).getElementType()); + } + + @Test + void testDoubleArrayRoundTrip() { + RelDataType calciteType = + CalciteTypeConverter.toCalciteType(typeFactory, ArrayType.DOUBLE_ARRAY_TYPE); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertInstanceOf(ArrayType.class, back); + Assertions.assertEquals(BasicType.DOUBLE_TYPE, ((ArrayType) back).getElementType()); + } + + @Test + void testArrayNullComponentReverse() { + RelDataType arrayType = + typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.VARCHAR), -1); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(arrayType); + Assertions.assertInstanceOf(ArrayType.class, back); + } + + @Test + void testArrayOfDecimalForward() { + ArrayType arrayType = + new ArrayType<>(java.math.BigDecimal.class, new DecimalType(10, 2)); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, arrayType); + Assertions.assertEquals(SqlTypeName.ARRAY, calciteType.getSqlTypeName()); + } + + @Test + void testArrayOfDateForward() { + ArrayType arrayType = + new ArrayType<>(java.time.LocalDate.class, LocalTimeType.LOCAL_DATE_TYPE); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, arrayType); + Assertions.assertEquals(SqlTypeName.ARRAY, calciteType.getSqlTypeName()); + } + + @Test + void testMapType() { + MapType mapType = new MapType<>(BasicType.STRING_TYPE, BasicType.INT_TYPE); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, mapType); + Assertions.assertEquals(SqlTypeName.MAP, calciteType.getSqlTypeName()); + } + + @Test + void testMapTypeRoundTrip() { + MapType mapType = new MapType<>(BasicType.STRING_TYPE, BasicType.INT_TYPE); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, mapType); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertInstanceOf(MapType.class, back); + MapType backMap = (MapType) back; + Assertions.assertEquals(BasicType.STRING_TYPE, backMap.getKeyType()); + Assertions.assertEquals(BasicType.INT_TYPE, backMap.getValueType()); + } + + @Test + void testMapIntIntForward() { + MapType mapType = new MapType<>(BasicType.INT_TYPE, BasicType.INT_TYPE); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, mapType); + Assertions.assertEquals(SqlTypeName.MAP, calciteType.getSqlTypeName()); + } + + @Test + void testMapIntIntRoundTrip() { + MapType mapType = new MapType<>(BasicType.INT_TYPE, BasicType.INT_TYPE); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, mapType); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + MapType backMap = (MapType) back; + Assertions.assertEquals(BasicType.INT_TYPE, backMap.getKeyType()); + Assertions.assertEquals(BasicType.INT_TYPE, backMap.getValueType()); + } + + @Test + void testMapStringDoubleRoundTrip() { + MapType mapType = + new MapType<>(BasicType.STRING_TYPE, BasicType.DOUBLE_TYPE); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, mapType); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + MapType backMap = (MapType) back; + Assertions.assertEquals(BasicType.STRING_TYPE, backMap.getKeyType()); + Assertions.assertEquals(BasicType.DOUBLE_TYPE, backMap.getValueType()); + } + + @Test + void testMapWithComplexValue() { + SeaTunnelRowType innerRow = + new SeaTunnelRowType( + new String[] {"name"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + MapType mapType = new MapType<>(BasicType.STRING_TYPE, innerRow); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, mapType); + Assertions.assertEquals(SqlTypeName.MAP, calciteType.getSqlTypeName()); + } + + @Test + void testRowType() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"name", "age"}, + new SeaTunnelDataType[] {BasicType.STRING_TYPE, BasicType.INT_TYPE}); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, rowType); + Assertions.assertTrue(calciteType.isStruct()); + Assertions.assertEquals(2, calciteType.getFieldCount()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertInstanceOf(SeaTunnelRowType.class, back); + SeaTunnelRowType backRow = (SeaTunnelRowType) back; + Assertions.assertEquals("name", backRow.getFieldName(0)); + Assertions.assertEquals("age", backRow.getFieldName(1)); + } + + @Test + void testNestedRowType() { + SeaTunnelRowType inner = + new SeaTunnelRowType( + new String[] {"city", "zip"}, + new SeaTunnelDataType[] {BasicType.STRING_TYPE, BasicType.INT_TYPE}); + SeaTunnelRowType outer = + new SeaTunnelRowType( + new String[] {"name", "address"}, + new SeaTunnelDataType[] {BasicType.STRING_TYPE, inner}); + + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, outer); + Assertions.assertTrue(calciteType.isStruct()); + Assertions.assertTrue(calciteType.getFieldList().get(1).getType().isStruct()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + SeaTunnelRowType backOuter = (SeaTunnelRowType) back; + SeaTunnelRowType backInner = (SeaTunnelRowType) backOuter.getFieldType(1); + Assertions.assertEquals("city", backInner.getFieldName(0)); + Assertions.assertEquals("zip", backInner.getFieldName(1)); + } + + @Test + void testSingleFieldRow() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, rowType); + Assertions.assertEquals(1, calciteType.getFieldCount()); + } + + @Test + void testRowWithManyFields() { + String[] names = new String[10]; + SeaTunnelDataType[] types = new SeaTunnelDataType[10]; + for (int i = 0; i < 10; i++) { + names[i] = "field_" + i; + types[i] = BasicType.INT_TYPE; + } + SeaTunnelRowType rowType = new SeaTunnelRowType(names, types); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, rowType); + Assertions.assertEquals(10, calciteType.getFieldCount()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + SeaTunnelRowType backRow = (SeaTunnelRowType) back; + Assertions.assertEquals(10, backRow.getTotalFields()); + for (int i = 0; i < 10; i++) { + Assertions.assertEquals("field_" + i, backRow.getFieldName(i)); + } + } + + @Test + void testRowWithAllBasicTypes() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] { + "bool", "byte", "short", "int", "long", "float", "double", "str" + }, + new SeaTunnelDataType[] { + BasicType.BOOLEAN_TYPE, + BasicType.BYTE_TYPE, + BasicType.SHORT_TYPE, + BasicType.INT_TYPE, + BasicType.LONG_TYPE, + BasicType.FLOAT_TYPE, + BasicType.DOUBLE_TYPE, + BasicType.STRING_TYPE + }); + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, rowType); + Assertions.assertEquals(8, calciteType.getFieldCount()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + SeaTunnelRowType backRow = (SeaTunnelRowType) back; + Assertions.assertEquals(SqlType.BOOLEAN, backRow.getFieldType(0).getSqlType()); + Assertions.assertEquals(SqlType.TINYINT, backRow.getFieldType(1).getSqlType()); + Assertions.assertEquals(SqlType.SMALLINT, backRow.getFieldType(2).getSqlType()); + Assertions.assertEquals(SqlType.INT, backRow.getFieldType(3).getSqlType()); + Assertions.assertEquals(SqlType.BIGINT, backRow.getFieldType(4).getSqlType()); + Assertions.assertEquals(SqlType.FLOAT, backRow.getFieldType(5).getSqlType()); + Assertions.assertEquals(SqlType.DOUBLE, backRow.getFieldType(6).getSqlType()); + Assertions.assertEquals(SqlType.STRING, backRow.getFieldType(7).getSqlType()); + } + + @Test + void testDeeplyNestedRow() { + SeaTunnelRowType level3 = + new SeaTunnelRowType( + new String[] {"val"}, new SeaTunnelDataType[] {BasicType.INT_TYPE}); + SeaTunnelRowType level2 = + new SeaTunnelRowType(new String[] {"inner"}, new SeaTunnelDataType[] {level3}); + SeaTunnelRowType level1 = + new SeaTunnelRowType(new String[] {"mid"}, new SeaTunnelDataType[] {level2}); + + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, level1); + Assertions.assertTrue(calciteType.isStruct()); + Assertions.assertTrue( + calciteType + .getFieldList() + .get(0) + .getType() + .getFieldList() + .get(0) + .getType() + .isStruct()); + + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + SeaTunnelRowType backL1 = (SeaTunnelRowType) back; + SeaTunnelRowType backL2 = (SeaTunnelRowType) backL1.getFieldType(0); + SeaTunnelRowType backL3 = (SeaTunnelRowType) backL2.getFieldType(0); + Assertions.assertEquals(SqlType.INT, backL3.getFieldType(0).getSqlType()); + } + + @Test + void testVectorBinaryType() { + assertVectorForward(VectorType.VECTOR_BINARY_TYPE); + } + + @Test + void testVectorFloatType() { + assertVectorForward(VectorType.VECTOR_FLOAT_TYPE); + } + + @Test + void testVectorFloat16Type() { + assertVectorForward(VectorType.VECTOR_FLOAT16_TYPE); + } + + @Test + void testVectorBfloat16Type() { + assertVectorForward(VectorType.VECTOR_BFLOAT16_TYPE); + } + + @Test + void testVectorSparseFloatType() { + assertVectorForward(VectorType.VECTOR_SPARSE_FLOAT_TYPE); + } + + @Test + void testIntervalYear() { + assertIntervalReverse(TimeUnit.YEAR, null); + } + + @Test + void testIntervalYearMonth() { + assertIntervalReverse(TimeUnit.YEAR, TimeUnit.MONTH); + } + + @Test + void testIntervalMonth() { + assertIntervalReverse(TimeUnit.MONTH, null); + } + + @Test + void testIntervalDay() { + assertIntervalReverse(TimeUnit.DAY, null); + } + + @Test + void testIntervalDayHour() { + assertIntervalReverse(TimeUnit.DAY, TimeUnit.HOUR); + } + + @Test + void testIntervalDayMinute() { + assertIntervalReverse(TimeUnit.DAY, TimeUnit.MINUTE); + } + + @Test + void testIntervalDaySecond() { + assertIntervalReverse(TimeUnit.DAY, TimeUnit.SECOND); + } + + @Test + void testIntervalHour() { + assertIntervalReverse(TimeUnit.HOUR, null); + } + + @Test + void testIntervalHourMinute() { + assertIntervalReverse(TimeUnit.HOUR, TimeUnit.MINUTE); + } + + @Test + void testIntervalHourSecond() { + assertIntervalReverse(TimeUnit.HOUR, TimeUnit.SECOND); + } + + @Test + void testIntervalMinute() { + assertIntervalReverse(TimeUnit.MINUTE, null); + } + + @Test + void testIntervalMinuteSecond() { + assertIntervalReverse(TimeUnit.MINUTE, TimeUnit.SECOND); + } + + @Test + void testIntervalSecond() { + assertIntervalReverse(TimeUnit.SECOND, null); + } + + @Test + void testAnyReverseMapping() { + RelDataType anyType = typeFactory.createSqlType(SqlTypeName.ANY); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(anyType); + Assertions.assertEquals(BasicType.STRING_TYPE, back); + } + + @Test + void testCalciteFloatReverse() { + RelDataType floatType = typeFactory.createSqlType(SqlTypeName.FLOAT); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(floatType); + Assertions.assertEquals(BasicType.FLOAT_TYPE, back); + } + + @Test + void testCalciteRealReverse() { + RelDataType realType = typeFactory.createSqlType(SqlTypeName.REAL); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(realType); + Assertions.assertEquals(BasicType.FLOAT_TYPE, back); + } + + @Test + void testMultisetReverse() { + RelDataType multisetType = + typeFactory.createMultisetType(typeFactory.createSqlType(SqlTypeName.INTEGER), -1); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(multisetType); + Assertions.assertInstanceOf(ArrayType.class, back); + } + + @Test + void testNullableIntegerReverse() { + RelDataType nullableInt = + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.INTEGER), true); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(nullableInt); + Assertions.assertEquals(BasicType.INT_TYPE, back); + } + + @Test + void testNonNullableIntegerReverse() { + RelDataType notNullInt = + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.INTEGER), false); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(notNullInt); + Assertions.assertEquals(BasicType.INT_TYPE, back); + } + + @Test + void testNullableVarcharReverse() { + RelDataType nullableVarchar = + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(nullableVarchar); + Assertions.assertEquals(BasicType.STRING_TYPE, back); + } + + private void assertRoundTrip(SeaTunnelDataType stType, SqlTypeName expectedCalciteName) { + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, stType); + Assertions.assertEquals(expectedCalciteName, calciteType.getSqlTypeName()); + SeaTunnelDataType roundTripped = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertEquals(stType, roundTripped); + } + + private void assertArrayForward(ArrayType arrayType) { + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, arrayType); + Assertions.assertEquals(SqlTypeName.ARRAY, calciteType.getSqlTypeName()); + } + + private void assertVectorForward(SeaTunnelDataType vectorType) { + RelDataType calciteType = CalciteTypeConverter.toCalciteType(typeFactory, vectorType); + Assertions.assertEquals(SqlTypeName.VARBINARY, calciteType.getSqlTypeName()); + } + + private void assertIntervalReverse(TimeUnit start, TimeUnit end) { + SqlIntervalQualifier qualifier = new SqlIntervalQualifier(start, end, SqlParserPos.ZERO); + RelDataType calciteType = typeFactory.createSqlIntervalType(qualifier); + SeaTunnelDataType back = CalciteTypeConverter.toSeaTunnelType(calciteType); + Assertions.assertEquals(BasicType.LONG_TYPE, back); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/DesEncryptDecryptFunctionTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/DesEncryptDecryptFunctionTest.java new file mode 100644 index 000000000000..98775db35154 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/DesEncryptDecryptFunctionTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class DesEncryptDecryptFunctionTest { + + private static final String PASSWORD = "12345678"; + + @Test + void testEncryptNullPassword() { + Assertions.assertNull(DesEncryptFunction.eval(null, "seatunnel")); + } + + @Test + void testEncryptNullData() { + Assertions.assertNull(DesEncryptFunction.eval(PASSWORD, null)); + } + + @Test + void testDecryptNullPassword() { + Assertions.assertNull(DesDecryptFunction.eval(null, "seatunnel")); + } + + @Test + void testDecryptNullData() { + Assertions.assertNull(DesDecryptFunction.eval(PASSWORD, null)); + } + + @Test + void testEncryptProducesNonNullResult() { + String result = DesEncryptFunction.eval(PASSWORD, "seatunnel-transform"); + Assertions.assertNotNull(result); + Assertions.assertFalse(result.isEmpty()); + } + + @Test + void testEncryptDecryptRoundTrip() { + String original = "seatunnel-connector-v2"; + String encrypted = DesEncryptFunction.eval(PASSWORD, original); + String decrypted = DesDecryptFunction.eval(PASSWORD, encrypted); + Assertions.assertEquals(original, decrypted); + } + + @Test + void testRoundTripLongText() { + String original = "apache-seatunnel-zeta-engine-checkpoint"; + String encrypted = DesEncryptFunction.eval(PASSWORD, original); + String decrypted = DesDecryptFunction.eval(PASSWORD, encrypted); + Assertions.assertEquals(original, decrypted); + } + + @Test + void testDifferentPasswordsProduceDifferentCiphertext() { + String data = "seatunnel"; + String enc1 = DesEncryptFunction.eval("abcdefgh", data); + String enc2 = DesEncryptFunction.eval("12345678", data); + Assertions.assertNotEquals(enc1, enc2); + } + + @Test + void testEncryptDeterministic() { + String first = DesEncryptFunction.eval(PASSWORD, "seatunnel"); + String second = DesEncryptFunction.eval(PASSWORD, "seatunnel"); + Assertions.assertEquals(first, second); + } + + @Test + void testFunctionNames() { + Assertions.assertEquals("DES_ENCRYPT", new DesEncryptFunction().functionName()); + Assertions.assertEquals("DES_DECRYPT", new DesDecryptFunction().functionName()); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/MaskFunctionTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/MaskFunctionTest.java new file mode 100644 index 000000000000..ef89c7030088 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/MaskFunctionTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class MaskFunctionTest { + + @Test + void testNormalMask() { + Assertions.assertEquals("138****5678", MaskFunction.eval("13812345678", 3, 7, "*")); + } + + @Test + void testNullInput() { + Assertions.assertNull(MaskFunction.eval(null, 0, 3, "*")); + } + + @Test + void testInvalidRange() { + Assertions.assertEquals("source", MaskFunction.eval("source", 6, 3, "*")); + Assertions.assertEquals("source", MaskFunction.eval("source", -1, 3, "*")); + } + + @Test + void testCustomMaskChar() { + Assertions.assertEquals("ze##!", MaskFunction.eval("zeta!", 2, 4, "#")); + } + + @Test + void testEmptyMaskChar() { + Assertions.assertEquals("s**k", MaskFunction.eval("sink", 1, 3, "")); + } + + @Test + void testEndExceedsLength() { + Assertions.assertEquals("source", MaskFunction.eval("source", 0, 10, "*")); + } + + @Test + void testNullMaskChar() { + Assertions.assertEquals("s**k", MaskFunction.eval("sink", 1, 3, null)); + } + + @Test + void testFunctionName() { + MaskFunction fn = new MaskFunction(); + Assertions.assertEquals("MASK", fn.functionName()); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/MaskHashFunctionTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/MaskHashFunctionTest.java new file mode 100644 index 000000000000..da653d0233af --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/MaskHashFunctionTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class MaskHashFunctionTest { + + @Test + void testNullInput() { + Assertions.assertNull(MaskHashFunction.eval(null)); + } + + @Test + void testNonNullReturns64CharHex() { + String result = MaskHashFunction.eval("seatunnel-transform"); + Assertions.assertNotNull(result); + Assertions.assertEquals(64, result.length()); + Assertions.assertTrue(result.matches("[0-9a-f]{64}")); + } + + @Test + void testDeterministic() { + String first = MaskHashFunction.eval("seatunnel"); + String second = MaskHashFunction.eval("seatunnel"); + Assertions.assertEquals(first, second); + } + + @Test + void testDifferentInputsDifferentHash() { + String hash1 = MaskHashFunction.eval("connector-source"); + String hash2 = MaskHashFunction.eval("connector-sink"); + Assertions.assertNotEquals(hash1, hash2); + } + + @Test + void testEmptyString() { + String result = MaskHashFunction.eval(""); + Assertions.assertNotNull(result); + Assertions.assertEquals(64, result.length()); + } + + @Test + void testKnownSHA256() { + // SHA-256("abc") = ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad + Assertions.assertEquals( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + MaskHashFunction.eval("abc")); + } + + @Test + void testFunctionName() { + MaskHashFunction fn = new MaskHashFunction(); + Assertions.assertEquals("MASK_HASH", fn.functionName()); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/VectorUdfTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/VectorUdfTest.java new file mode 100644 index 000000000000..519a71751577 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/udf/VectorUdfTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.calcite.udf; + +import org.apache.seatunnel.common.utils.VectorUtils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +class VectorUdfTest { + + private static byte[] toBytes(Float[] floats) { + ByteBuffer buf = VectorUtils.toByteBuffer(floats); + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + return bytes; + } + + private static Float[] fromBytes(byte[] bytes) { + return VectorUtils.toFloatArray(ByteBuffer.wrap(bytes)); + } + + @Test + void testCosineDistanceIdenticalVectors() { + byte[] v = toBytes(new Float[] {1.0f, 2.0f, 3.0f}); + Double result = CosineDistanceFunction.eval(v, v); + Assertions.assertEquals(0.0, result, 1e-9); + } + + @Test + void testCosineDistanceOrthogonalVectors() { + byte[] v1 = toBytes(new Float[] {1.0f, 0.0f}); + byte[] v2 = toBytes(new Float[] {0.0f, 1.0f}); + Double result = CosineDistanceFunction.eval(v1, v2); + Assertions.assertEquals(1.0, result, 1e-9); + } + + @Test + void testCosineDistanceNull() { + Assertions.assertNull(CosineDistanceFunction.eval(null, toBytes(new Float[] {1.0f}))); + Assertions.assertNull(CosineDistanceFunction.eval(toBytes(new Float[] {1.0f}), null)); + } + + @Test + void testCosineDistanceDimensionMismatch() { + byte[] v1 = toBytes(new Float[] {1.0f, 2.0f}); + byte[] v2 = toBytes(new Float[] {1.0f, 2.0f, 3.0f}); + Assertions.assertThrows( + IllegalArgumentException.class, () -> CosineDistanceFunction.eval(v1, v2)); + } + + @Test + void testCosineDistanceFunctionName() { + Assertions.assertEquals("COSINE_DISTANCE", new CosineDistanceFunction().functionName()); + } + + @Test + void testL1Distance() { + byte[] v1 = toBytes(new Float[] {2.0f, 4.0f, 6.0f}); + byte[] v2 = toBytes(new Float[] {1.0f, 2.0f, 3.0f}); + Double result = L1DistanceFunction.eval(v1, v2); + Assertions.assertEquals(6.0, result, 1e-9); + } + + @Test + void testL1DistanceNull() { + Assertions.assertNull(L1DistanceFunction.eval(null, toBytes(new Float[] {1.0f}))); + Assertions.assertNull(L1DistanceFunction.eval(toBytes(new Float[] {1.0f}), null)); + } + + @Test + void testL1DistanceFunctionName() { + Assertions.assertEquals("L1_DISTANCE", new L1DistanceFunction().functionName()); + } + + @Test + void testL2Distance() { + byte[] v1 = toBytes(new Float[] {2.0f, 4.0f, 4.0f}); + byte[] v2 = toBytes(new Float[] {1.0f, 2.0f, 2.0f}); + Double result = L2DistanceFunction.eval(v1, v2); + Assertions.assertEquals(3.0, result, 1e-9); + } + + @Test + void testL2DistanceNull() { + Assertions.assertNull(L2DistanceFunction.eval(null, toBytes(new Float[] {1.0f}))); + Assertions.assertNull(L2DistanceFunction.eval(toBytes(new Float[] {1.0f}), null)); + } + + @Test + void testL2DistanceFunctionName() { + Assertions.assertEquals("L2_DISTANCE", new L2DistanceFunction().functionName()); + } + + @Test + void testVectorDims() { + byte[] v = toBytes(new Float[] {1.0f, 2.0f, 3.0f}); + Assertions.assertEquals(3, VectorDimsFunction.eval(v)); + } + + @Test + void testVectorDimsNull() { + Assertions.assertNull(VectorDimsFunction.eval(null)); + } + + @Test + void testVectorDimsFunctionName() { + Assertions.assertEquals("VECTOR_DIMS", new VectorDimsFunction().functionName()); + } + + @Test + void testVectorNorm() { + byte[] v = toBytes(new Float[] {1.0f, 2.0f, 2.0f}); + Double result = VectorNormFunction.eval(v); + Assertions.assertEquals(3.0, result, 1e-9); + } + + @Test + void testVectorNormNull() { + Assertions.assertNull(VectorNormFunction.eval(null)); + } + + @Test + void testVectorNormFunctionName() { + Assertions.assertEquals("VECTOR_NORM", new VectorNormFunction().functionName()); + } + + @Test + void testInnerProduct() { + byte[] v1 = toBytes(new Float[] {1.0f, 2.0f, 3.0f}); + byte[] v2 = toBytes(new Float[] {7.0f, 8.0f, 9.0f}); + Double result = InnerProductFunction.eval(v1, v2); + Assertions.assertEquals(50.0, result, 1e-9); + } + + @Test + void testInnerProductNull() { + Assertions.assertNull(InnerProductFunction.eval(null, toBytes(new Float[] {1.0f}))); + Assertions.assertNull(InnerProductFunction.eval(toBytes(new Float[] {1.0f}), null)); + } + + @Test + void testInnerProductFunctionName() { + Assertions.assertEquals("INNER_PRODUCT", new InnerProductFunction().functionName()); + } + + @Test + void testVectorReduceTruncate() { + byte[] v = toBytes(new Float[] {1.0f, 2.0f, 3.0f, 4.0f}); + byte[] result = VectorReduceFunction.eval(v, 2, "TRUNCATE"); + Float[] reduced = fromBytes(result); + Assertions.assertArrayEquals(new Float[] {1.0f, 2.0f}, reduced); + } + + @Test + void testVectorReduceNoTruncateNeeded() { + byte[] v = toBytes(new Float[] {1.0f, 2.0f}); + byte[] result = VectorReduceFunction.eval(v, 10, "TRUNCATE"); + Assertions.assertSame(v, result); + } + + @Test + void testVectorReduceRandomProjection() { + byte[] v = toBytes(new Float[] {1.0f, 2.0f, 3.0f, 4.0f}); + byte[] result = VectorReduceFunction.eval(v, 2, "RANDOM_PROJECTION"); + Float[] reduced = fromBytes(result); + Assertions.assertEquals(2, reduced.length); + } + + @Test + void testVectorReduceSparseProjection() { + byte[] v = toBytes(new Float[] {1.0f, 2.0f, 3.0f, 4.0f}); + byte[] result = VectorReduceFunction.eval(v, 2, "SPARSE_RANDOM_PROJECTION"); + Float[] reduced = fromBytes(result); + Assertions.assertEquals(2, reduced.length); + } + + @Test + void testVectorReduceNull() { + Assertions.assertNull(VectorReduceFunction.eval(null, 2, "TRUNCATE")); + byte[] v = toBytes(new Float[] {1.0f}); + Assertions.assertNull(VectorReduceFunction.eval(v, null, "TRUNCATE")); + Assertions.assertNull(VectorReduceFunction.eval(v, 2, null)); + } + + @Test + void testVectorReduceUnknownMethod() { + byte[] v = toBytes(new Float[] {1.0f, 2.0f, 3.0f, 4.0f}); + Assertions.assertThrows( + IllegalArgumentException.class, () -> VectorReduceFunction.eval(v, 2, "UNKNOWN")); + } + + @Test + void testVectorReduceFunctionName() { + Assertions.assertEquals("VECTOR_REDUCE", new VectorReduceFunction().functionName()); + } + + @Test + void testVectorNormalize() { + byte[] v = toBytes(new Float[] {3.0f, 4.0f}); + byte[] result = VectorNormalizeFunction.eval(v); + Float[] normalized = fromBytes(result); + Assertions.assertEquals(2, normalized.length); + double norm = Math.sqrt(normalized[0] * normalized[0] + normalized[1] * normalized[1]); + Assertions.assertEquals(1.0, norm, 1e-6); + } + + @Test + void testVectorNormalizeZeroVector() { + byte[] v = toBytes(new Float[] {0.0f, 0.0f}); + byte[] result = VectorNormalizeFunction.eval(v); + Assertions.assertSame(v, result); + } + + @Test + void testVectorNormalizeNull() { + Assertions.assertNull(VectorNormalizeFunction.eval(null)); + } + + @Test + void testVectorNormalizeFunctionName() { + Assertions.assertEquals("VECTOR_NORMALIZE", new VectorNormalizeFunction().functionName()); + } +} diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 707aa3ccbf8f..efc70e55f593 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -52,6 +52,7 @@ asm-9.1.jar avro-1.11.1.jar groovy-4.0.16.jar seatunnel-janino-3.0.0-SNAPSHOT-optional.jar +seatunnel-calcite-3.0.0-SNAPSHOT-optional.jar protobuf-java-util-3.25.3.jar protobuf-java-3.25.3.jar protoc-jar-3.11.4.jar From c7396b842c5496f8cfc1d8a9e406ea3d2ac437c1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 20 Jun 2026 20:24:33 +0800 Subject: [PATCH 039/375] [Test][E2E] Stabilize Db2 container readiness for JDBC IT (#11138) --- .../connectors/seatunnel/jdbc/JdbcDb2IT.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java index ee0633ee492b..29cd23855751 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java @@ -27,7 +27,7 @@ import org.testcontainers.containers.Db2Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.DockerLoggerFactory; import java.math.BigDecimal; @@ -56,8 +56,11 @@ public class JdbcDb2IT extends AbstractJdbcIT { private static final List CONFIG_FILE = Lists.newArrayList("/jdbc_db2_source_and_sink.conf"); - /** db2 in dockerhub */ - private static final String DB2_IMAGE = "ibmcom/db2"; + /** + * Keep the tag aligned with Testcontainers' validated default instead of relying on {@code + * latest}, whose startup behavior is not stable enough for CI. + */ + private static final String DB2_IMAGE = "ibmcom/db2:11.5.0.0a"; private static final int PORT = 50000; private static final int LOCAL_PORT = 50000; @@ -181,6 +184,8 @@ Pair> initTestData() { @Override protected GenericContainer initContainer() { + // The DB2 port becomes reachable before the instance can accept JDBC logins, and the + // first-time initialization regularly exceeds the default 10 minute wait on slow runners. GenericContainer container = new Db2Container(DB2_IMAGE) .withExposedPorts(PORT) @@ -190,7 +195,9 @@ protected GenericContainer initContainer() { .withUsername(DB2_USER) .withPassword(DB2_PASSWORD) .waitingFor( - Wait.forListeningPort().withStartupTimeout(Duration.ofMinutes(5))) + new LogMessageWaitStrategy() + .withRegEx(".*Setup has completed\\..*") + .withStartupTimeout(Duration.ofMinutes(20))) .withLogConsumer( new Slf4jLogConsumer(DockerLoggerFactory.getLogger(DB2_IMAGE))) .acceptLicense(); From 7601726ff6bc02da88313c725866a32c32bb4fd1 Mon Sep 17 00:00:00 2001 From: Jast Date: Sun, 21 Jun 2026 02:11:27 +0800 Subject: [PATCH 040/375] [Docs] Fix IMap persistence grammar (#11148) Co-authored-by: zhangshenghang <29418975+zhangshenghang@users.noreply.github.com> --- docs/en/engines/zeta/separated-cluster-deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/zeta/separated-cluster-deployment.md b/docs/en/engines/zeta/separated-cluster-deployment.md index 453376ef3235..8cf659363d4f 100644 --- a/docs/en/engines/zeta/separated-cluster-deployment.md +++ b/docs/en/engines/zeta/separated-cluster-deployment.md @@ -232,7 +232,7 @@ The following describes how to use the MapStore persistence configuration. For d **type** -The type of IMap persistence, currently only supports `hdfs`. +The type of IMap persistence. Currently, only `hdfs` is supported. **namespace** From 9e991df0160e657b77c75efdec7965f570f208c9 Mon Sep 17 00:00:00 2001 From: Jast Date: Sun, 21 Jun 2026 02:12:10 +0800 Subject: [PATCH 041/375] [Docs] Fix default engine grammar (#11147) Co-authored-by: zhangshenghang <29418975+zhangshenghang@users.noreply.github.com> --- docs/en/introduction/about.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/about.md b/docs/en/introduction/about.md index 0175d6626ee7..e229144970dc 100644 --- a/docs/en/introduction/about.md +++ b/docs/en/introduction/about.md @@ -62,7 +62,7 @@ The Source Connector is responsible for parallel reading and sending the data to SeaTunnel is an EtL(T) data integration tool. Therefore, in SeaTunnel, transform can only be used to perform some simple transformations on data, such as converting the data of a column to uppercase or lowercase, changing the column name, or splitting a column into multiple columns. -The default engine use by SeaTunnel is [SeaTunnel Engine](../engines/zeta/about.md). If you choose to use the Flink or Spark engine, SeaTunnel will package the Connector into a Flink or Spark program and submit it to Flink or Spark to run. +The default engine used by SeaTunnel is [SeaTunnel Engine](../engines/zeta/about.md). If you choose to use the Flink or Spark engine, SeaTunnel will package the Connector into a Flink or Spark program and submit it to Flink or Spark to run. ## Connector From 974cde277a566f799314e8e674ec75018b9df729 Mon Sep 17 00:00:00 2001 From: Jast Date: Sun, 21 Jun 2026 02:13:10 +0800 Subject: [PATCH 042/375] [Docs] Fix Pulsar source fetching typo (#11145) Co-authored-by: zhangshenghang <29418975+zhangshenghang@users.noreply.github.com> --- docs/en/connectors/source/Pulsar.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/connectors/source/Pulsar.md b/docs/en/connectors/source/Pulsar.md index f5fcfb0f94fb..23833d3595d8 100644 --- a/docs/en/connectors/source/Pulsar.md +++ b/docs/en/connectors/source/Pulsar.md @@ -117,7 +117,7 @@ The maximum time (in ms) to wait when fetching records. A longer time increases ### poll.interval [Long] -The interval time(in ms) when fetcing records. A shorter time increases throughput, but also increases CPU load. +The interval time (in ms) when fetching records. A shorter time increases throughput, but also increases CPU load. ### poll.batch.size [Integer] From 45f29f39794031fb2b3220f79b3b36a209f122d9 Mon Sep 17 00:00:00 2001 From: Jast Date: Sun, 21 Jun 2026 02:15:21 +0800 Subject: [PATCH 043/375] [Docs] Polish Pulsar sink routing mode docs (#11146) Co-authored-by: zhangshenghang <29418975+zhangshenghang@users.noreply.github.com> --- docs/en/connectors/sink/Pulsar.md | 10 +++++----- docs/zh/connectors/sink/Pulsar.md | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/connectors/sink/Pulsar.md b/docs/en/connectors/sink/Pulsar.md index 924007edfefe..636bffcc844a 100644 --- a/docs/en/connectors/sink/Pulsar.md +++ b/docs/en/connectors/sink/Pulsar.md @@ -108,11 +108,11 @@ covering all the producer parameters specified in the official Pulsar document. ### message.routing.mode [Enum] -Default routing mode for messages to partition. -Available options are SinglePartition,RoundRobinPartition. -If you choose SinglePartition, If no key is provided, The partitioned producer will randomly pick one single partition and publish all the messages into that partition, If a key is provided on the message, the partitioned producer will hash the key and assign message to a particular partition. -If you choose RoundRobinPartition, If no key is provided, the producer will publish messages across all partitions in round-robin fashion to achieve maximum throughput. -Please note that round-robin is not done per individual message but rather it's set to the same boundary of batching delay, to ensure batching is effective. +Default routing mode for partitioned messages. +Available options are `SinglePartition` and `RoundRobinPartition`. +When `SinglePartition` is selected and no key is provided, the partitioned producer randomly selects one partition and publishes all messages to that partition. When a key is provided, the producer hashes the key and sends the message to the selected partition. +When `RoundRobinPartition` is selected and no key is provided, the producer publishes messages across all partitions in round-robin order to achieve maximum throughput. +Round-robin routing is applied at the batching delay boundary rather than per individual message, so batching remains effective. ### partition_key_fields [String] diff --git a/docs/zh/connectors/sink/Pulsar.md b/docs/zh/connectors/sink/Pulsar.md index dbd517ee9f38..955effa165b8 100644 --- a/docs/zh/connectors/sink/Pulsar.md +++ b/docs/zh/connectors/sink/Pulsar.md @@ -92,9 +92,9 @@ Pulsar 服务的 Service URL 提供程序。要使用客户端库连接到 Pulsa ### message.routing.mode [Enum] -要分区的消息的默认路由模式。可用选项包括 SinglePartition、RoundRobinPartition。 -如果选择 SinglePartition，如果未提供密钥，分区生产者将随机选择一个分区并将所有消息发布到该分区中，如果消息上提供了密钥，则分区生产者将对密钥进行哈希处理并将消息分配给特定分区。 -如果选择 RoundRobinPartition，则如果未提供密钥，则生产者将以循环方式跨所有分区发布消息，以实现最大吞吐量。请注意，轮询不是按单个消息完成的，而是设置为相同的批处理延迟边界，以确保批处理有效。 +分区消息的默认路由模式。可用选项包括 `SinglePartition` 和 `RoundRobinPartition`。 +选择 `SinglePartition` 且未提供 key 时，分区生产者会随机选择一个分区，并将所有消息发布到该分区。提供 key 时，生产者会对 key 做哈希，并将消息发送到对应分区。 +选择 `RoundRobinPartition` 且未提供 key 时，生产者会以轮询方式将消息发布到所有分区，以获得最大吞吐量。请注意，轮询不是逐条消息执行，而是在批处理延迟边界上执行，以确保批处理仍然有效。 ### partition_key_fields [String] From 690609ae36bd72f21cb084e873e43c13d4593b5e Mon Sep 17 00:00:00 2001 From: Jast Date: Sun, 21 Jun 2026 02:16:22 +0800 Subject: [PATCH 044/375] [Docs] Fix zh JVM tuning guide link (#11135) --- docs/zh/engines/zeta/tuning-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/engines/zeta/tuning-guide.md b/docs/zh/engines/zeta/tuning-guide.md index bfe495206dd6..63561169c21e 100644 --- a/docs/zh/engines/zeta/tuning-guide.md +++ b/docs/zh/engines/zeta/tuning-guide.md @@ -7,7 +7,7 @@ sidebar_position: 15 本文为大家介绍 SeaTunnel Engine 的调优方法，帮助用户根据实际需求优化 SeaTunnel Engine 的性能和稳定性。 阅读次篇前请知晓，当前指南结合的是大部分用户的真实使用情况总结而成，可能并不适用于所有场景，用户可以根据实际情况进行调整。 -SeaTunnel Engine 是基于 [JVM] (https://zh.wikipedia.org/wiki/Java%E8%99%9A%E6%8B%9F%E6%9C%BA) 运行的数据集成引擎，所以 JVM 部分的调优对 SeaTunnel Engine 同样适用，这里就不再赘述。 +SeaTunnel Engine 是基于 [JVM](https://zh.wikipedia.org/wiki/Java%E8%99%9A%E6%8B%9F%E6%9C%BA) 运行的数据集成引擎，所以 JVM 部分的调优对 SeaTunnel Engine 同样适用，这里就不再赘述。 ## 集群响应缓慢或假死 From 9093e3ec37baeebbbd9d9d1500d7b321412c15ce Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 21 Jun 2026 02:39:25 +0800 Subject: [PATCH 045/375] [Test][E2E] Wait for SFTP continuous job exit after cancel (#11143) --- .../e2e/connector/file/fstp/SftpFileIT.java | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java index 658cfaab0082..e6941d0de6bf 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java @@ -343,14 +343,7 @@ public void testSftpBinaryUpdateModeContinuousDiscoveryDistcp(TestContainer cont Container.ExecResult cancelResult = container.cancelJob(jobId); Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr()); - - Container.ExecResult execResult; - try { - execResult = jobFuture.get(120, TimeUnit.SECONDS); - } catch (Exception e) { - throw new RuntimeException("Wait continuous job exit failed.", e); - } - Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + waitContinuousJobExit(container, jobId, jobFuture); } finally { deleteFileFromContainer(SFTP_CONTAINER_HOME + "/tmp/seatunnel/continuous"); } @@ -402,14 +395,7 @@ public void testSftpBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan( Container.ExecResult cancelResult = container.cancelJob(jobId); Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr()); - - Container.ExecResult execResult; - try { - execResult = jobFuture.get(120, TimeUnit.SECONDS); - } catch (Exception e) { - throw new RuntimeException("Wait continuous job exit failed.", e); - } - Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + waitContinuousJobExit(container, jobId, jobFuture); } finally { deleteFileFromContainer(SFTP_CONTAINER_HOME + "/tmp/seatunnel/continuous"); } @@ -583,6 +569,31 @@ private boolean isSftpFileExists(String containerPath) return result.getExitCode() == 0; } + /** + * Wait for the continuous discovery job to enter the terminal canceled state and for the + * asynchronous execute call to finish its post-job thread cleanup. + */ + private void waitContinuousJobExit( + TestContainer container, + String jobId, + CompletableFuture jobFuture) { + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .untilAsserted( + () -> Assertions.assertEquals("CANCELED", container.getJobStatus(jobId))); + Awaitility.await() + .atMost(180, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until(jobFuture::isDone); + try { + Container.ExecResult execResult = jobFuture.get(30, TimeUnit.SECONDS); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + } catch (Exception e) { + throw new RuntimeException("Wait continuous job exit failed.", e); + } + } + private void waitUntilContainerTimeAfter(long epochSeconds) { Awaitility.await() .atMost(10, TimeUnit.SECONDS) From b7d3f1515f015dd2bba498ddcd6218a69ccfdade Mon Sep 17 00:00:00 2001 From: Doyeon Kim <132787602+dybyte@users.noreply.github.com> Date: Sun, 21 Jun 2026 11:48:54 +0900 Subject: [PATCH 046/375] [Improve][Zeta] Add ReportMetricsOperation observability exports (#11104) --- docs/en/engines/zeta/telemetry.md | 9 ++ docs/zh/engines/zeta/telemetry.md | 9 ++ .../engine/server/TaskExecutionService.java | 72 +++++++++++- .../metrics/ExportsInstanceInitializer.java | 3 + .../entity/ReportMetricsOperationStats.java | 39 +++++++ .../ReportMetricsOperationExports.java | 90 ++++++++++++++ ...elemetryCollectorCoordinatorGuardTest.java | 110 ++++++++++++++++++ 7 files changed, 329 insertions(+), 3 deletions(-) create mode 100644 seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/ReportMetricsOperationStats.java create mode 100644 seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/ReportMetricsOperationExports.java diff --git a/docs/en/engines/zeta/telemetry.md b/docs/en/engines/zeta/telemetry.md index a749785a1864..7c91348a1db3 100644 --- a/docs/en/engines/zeta/telemetry.md +++ b/docs/en/engines/zeta/telemetry.md @@ -146,6 +146,15 @@ engine_state_store_connector_jar_total_references{backend="hazelcast"} | job_thread_pool_task_total | Counter | **address**, server instance address,for example: "127.0.0.1:5801" | The taskCount of seatunnel coordinator job's executor cached thread pool | | job_thread_pool_rejection_total | Counter | **address**, server instance address,for example: "127.0.0.1:5801" | The rejectionCount of seatunnel coordinator job's executor cached thread pool | | +### Report Metrics Operation + +| MetricName | Type | Labels | DESCRIPTION | +|---------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------| +| report_metrics_operation_total | Counter | **address**, worker instance address,for example: "127.0.0.1:5801". **result**, one of "success" "failure" "interrupted" | The total number of `ReportMetricsOperation` invocations sent by a worker | +| report_metrics_operation_last_payload_task_count | Gauge | **address**, worker instance address,for example: "127.0.0.1:5801" | The number of task metrics included in the most recent `ReportMetricsOperation` payload sent by a worker | +| report_metrics_operation_last_invocation_latency_ms | Gauge | **address**, worker instance address,for example: "127.0.0.1:5801" | The most recent worker-side `ReportMetricsOperation` reporting latency in milliseconds, including local metrics collection and worker-to-master invocation | +| report_metrics_operation_max_invocation_latency_ms | Gauge | **address**, worker instance address,for example: "127.0.0.1:5801" | The maximum observed worker-side `ReportMetricsOperation` reporting latency in milliseconds since the worker started, including local metrics collection and worker-to-master invocation | + ### Job info detail | MetricName | Type | Labels | DESCRIPTION | diff --git a/docs/zh/engines/zeta/telemetry.md b/docs/zh/engines/zeta/telemetry.md index 0da7c4e1b889..e90a320be483 100644 --- a/docs/zh/engines/zeta/telemetry.md +++ b/docs/zh/engines/zeta/telemetry.md @@ -145,6 +145,15 @@ engine_state_store_connector_jar_total_references{backend="hazelcast"} | job_thread_pool_task_total | Counter | **address**，服务器实例地址，例如："127.0.0.1:5801" | seatunnel 协调器作业执行器缓存线程池的总任务数 | | job_thread_pool_rejection_total | Counter | **address**，服务器实例地址，例如："127.0.0.1:5801" | seatunnel 协调器作业执行器缓存线程池的拒绝任务总数 | +### ReportMetricsOperation 指标 + +| MetricName | Type | Labels | 描述 | +|---------------------------------------------------|---------|--------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| report_metrics_operation_total | Counter | **address**，worker 实例地址，例如："127.0.0.1:5801"。**result**，取值包括："success" "failure" "interrupted" | worker 发送的 `ReportMetricsOperation` 调用总次数 | +| report_metrics_operation_last_payload_task_count | Gauge | **address**，worker 实例地址，例如："127.0.0.1:5801" | 最近一次 `ReportMetricsOperation` payload 中包含的 task metrics 数量 | +| report_metrics_operation_last_invocation_latency_ms | Gauge | **address**，worker 实例地址，例如："127.0.0.1:5801" | worker 侧最近一次 `ReportMetricsOperation` 上报耗时，单位为毫秒，包含本地 metrics 收集和 worker 到 master 的调用时间 | +| report_metrics_operation_max_invocation_latency_ms | Gauge | **address**，worker 实例地址，例如："127.0.0.1:5801" | worker 启动以来观测到的 `ReportMetricsOperation` 最大上报耗时，单位为毫秒，包含本地 metrics 收集和 worker 到 master 的调用时间 | + ### 作业信息详细 | MetricName | Type | Labels | 描述 | diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java index a7b02766dc3a..5e011d53914a 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java @@ -53,6 +53,7 @@ import org.apache.seatunnel.engine.server.task.TaskGroupImmutableInformation; import org.apache.seatunnel.engine.server.task.operation.NotifyTaskStatusOperation; import org.apache.seatunnel.engine.server.task.operation.ReportMetricsOperation; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.ReportMetricsOperationStats; import org.apache.commons.collections4.CollectionUtils; @@ -96,6 +97,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -196,6 +198,13 @@ public class TaskExecutionService implements DynamicMetricsProvider { /** SeaTunnel configuration for this engine. */ private final SeaTunnelConfig seaTunnelConfig; + // Track worker-side metrics reporting cost without changing the report path semantics. + private final AtomicLong reportMetricsOperationSuccessCount = new AtomicLong(); + private final AtomicLong reportMetricsOperationFailureCount = new AtomicLong(); + private final AtomicLong reportMetricsOperationInterruptedCount = new AtomicLong(); + private final AtomicLong reportMetricsOperationLastPayloadTaskCount = new AtomicLong(); + private final AtomicLong reportMetricsOperationLastInvocationLatencyMs = new AtomicLong(); + private final AtomicLong reportMetricsOperationMaxInvocationLatencyMs = new AtomicLong(); /** Scheduled executor for periodic tasks like metrics backup. */ private final ScheduledExecutorService scheduledExecutorService; @@ -753,26 +762,83 @@ private void updateMetricsContextInImap() { return; } + long invocationStartNanos = System.nanoTime(); + HashMap localMetricsMap = collectLocalMetricsMap(); + int payloadTaskCount = localMetricsMap.size(); InvocationFuture invoke = nodeEngine .getOperationService() .createInvocationBuilder( SeaTunnelServer.SERVICE_NAME, - new ReportMetricsOperation(collectLocalMetricsMap()), + new ReportMetricsOperation(localMetricsMap), nodeEngine.getMasterAddress()) .invoke(); try { invoke.get(); + recordReportMetricsOperationSuccess( + payloadTaskCount, elapsedMillisSince(invocationStartNanos)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - logger.severe("update metrics context stopped due to thread interruption.", e); + long elapsedMillis = elapsedMillisSince(invocationStartNanos); + recordReportMetricsOperationInterruption(payloadTaskCount, elapsedMillis); + logger.severe( + String.format( + "update metrics context stopped due to thread interruption, " + + "payloadTaskCount=%d, invocationLatencyMs=%d.", + payloadTaskCount, elapsedMillis), + e); } catch (Exception e) { - logger.severe("failed to update metrics", e); + long elapsedMillis = elapsedMillisSince(invocationStartNanos); + recordReportMetricsOperationFailure(payloadTaskCount, elapsedMillis); + logger.severe( + String.format( + "failed to update metrics, payloadTaskCount=%d, " + + "invocationLatencyMs=%d.", + payloadTaskCount, elapsedMillis), + e); } this.printTaskExecutionRuntimeInfo(); } + private void recordReportMetricsOperationSuccess(int payloadTaskCount, long elapsedMillis) { + updateReportMetricsOperationObservability(payloadTaskCount, elapsedMillis); + reportMetricsOperationSuccessCount.incrementAndGet(); + } + + private void recordReportMetricsOperationFailure(int payloadTaskCount, long elapsedMillis) { + updateReportMetricsOperationObservability(payloadTaskCount, elapsedMillis); + reportMetricsOperationFailureCount.incrementAndGet(); + } + + private void recordReportMetricsOperationInterruption( + int payloadTaskCount, long elapsedMillis) { + updateReportMetricsOperationObservability(payloadTaskCount, elapsedMillis); + reportMetricsOperationInterruptedCount.incrementAndGet(); + } + + private void updateReportMetricsOperationObservability( + int payloadTaskCount, long elapsedMillis) { + reportMetricsOperationLastPayloadTaskCount.set(payloadTaskCount); + reportMetricsOperationLastInvocationLatencyMs.set(elapsedMillis); + reportMetricsOperationMaxInvocationLatencyMs.accumulateAndGet(elapsedMillis, Math::max); + } + + private long elapsedMillisSince(long startNanos) { + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); + } + + /** Returns the latest worker-side ReportMetricsOperation observability snapshot. */ + public ReportMetricsOperationStats getReportMetricsOperationStats() { + return new ReportMetricsOperationStats( + reportMetricsOperationSuccessCount.get(), + reportMetricsOperationFailureCount.get(), + reportMetricsOperationInterruptedCount.get(), + reportMetricsOperationLastPayloadTaskCount.get(), + reportMetricsOperationLastInvocationLatencyMs.get(), + reportMetricsOperationMaxInvocationLatencyMs.get()); + } + private HashMap collectLocalMetricsMap() { Map contextMap = new HashMap<>(); contextMap.putAll(finishedExecutionContexts); diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/ExportsInstanceInitializer.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/ExportsInstanceInitializer.java index d4b6cba6ec40..4b759aba5f78 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/ExportsInstanceInitializer.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/ExportsInstanceInitializer.java @@ -23,6 +23,7 @@ import org.apache.seatunnel.engine.server.telemetry.metrics.exports.JobMetricExports; import org.apache.seatunnel.engine.server.telemetry.metrics.exports.JobThreadPoolStatusExports; import org.apache.seatunnel.engine.server.telemetry.metrics.exports.NodeMetricExports; +import org.apache.seatunnel.engine.server.telemetry.metrics.exports.ReportMetricsOperationExports; import com.hazelcast.instance.impl.Node; import io.prometheus.client.CollectorRegistry; @@ -47,6 +48,8 @@ public static synchronized void init(Node node) { new JobThreadPoolStatusExports(node).register(collectorRegistry); // Node metrics new NodeMetricExports(node).register(collectorRegistry); + // ReportMetricsOperation metrics + new ReportMetricsOperationExports(node).register(collectorRegistry); // Engine state store metrics new EngineStateStoreMetricExports(node).register(collectorRegistry); // Engine state store logical metrics diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/ReportMetricsOperationStats.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/ReportMetricsOperationStats.java new file mode 100644 index 000000000000..ddd80fe89d6f --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/ReportMetricsOperationStats.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** Snapshot of worker-side ReportMetricsOperation observability state. */ +@Data +@AllArgsConstructor +public class ReportMetricsOperationStats { + /** Total successful report invocations sent by this worker. */ + private long successCount; + /** Total failed report invocations sent by this worker. */ + private long failureCount; + /** Total interrupted report invocations sent by this worker. */ + private long interruptedCount; + /** Task metric count in the most recent report payload. */ + private long lastPayloadTaskCount; + /** Most recent worker-side reporting latency in milliseconds. */ + private long lastInvocationLatencyMs; + /** Maximum observed worker-side reporting latency in milliseconds. */ + private long maxInvocationLatencyMs; +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/ReportMetricsOperationExports.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/ReportMetricsOperationExports.java new file mode 100644 index 000000000000..0f46c647652a --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/ReportMetricsOperationExports.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics.exports; + +import org.apache.seatunnel.engine.server.TaskExecutionService; +import org.apache.seatunnel.engine.server.telemetry.metrics.AbstractCollector; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.ReportMetricsOperationStats; + +import com.hazelcast.instance.impl.Node; +import io.prometheus.client.CounterMetricFamily; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.List; + +public class ReportMetricsOperationExports extends AbstractCollector { + + public ReportMetricsOperationExports(Node node) { + super(node); + } + + @Override + public List collect() { + List mfs = new ArrayList(); + TaskExecutionService taskExecutionService = getServer().getTaskExecutionService(); + if (taskExecutionService == null) { + return mfs; + } + + String address = localAddress(); + ReportMetricsOperationStats stats = taskExecutionService.getReportMetricsOperationStats(); + + CounterMetricFamily totalMetricFamily = + new CounterMetricFamily( + "report_metrics_operation", + "The total number of ReportMetricsOperation invocations sent by a worker", + clusterLabelNames(ADDRESS, "result")); + totalMetricFamily.addMetric(labelValues(address, "success"), stats.getSuccessCount()); + totalMetricFamily.addMetric(labelValues(address, "failure"), stats.getFailureCount()); + totalMetricFamily.addMetric( + labelValues(address, "interrupted"), stats.getInterruptedCount()); + mfs.add(totalMetricFamily); + + GaugeMetricFamily payloadMetricFamily = + new GaugeMetricFamily( + "report_metrics_operation_last_payload_task_count", + "The number of task metrics included in the most recent " + + "ReportMetricsOperation payload sent by a worker", + clusterLabelNames(ADDRESS)); + payloadMetricFamily.addMetric(labelValues(address), stats.getLastPayloadTaskCount()); + mfs.add(payloadMetricFamily); + + GaugeMetricFamily lastLatencyMetricFamily = + new GaugeMetricFamily( + "report_metrics_operation_last_invocation_latency_ms", + "The most recent worker-side ReportMetricsOperation reporting latency " + + "in milliseconds, including local metrics collection and " + + "worker-to-master invocation", + clusterLabelNames(ADDRESS)); + lastLatencyMetricFamily.addMetric(labelValues(address), stats.getLastInvocationLatencyMs()); + mfs.add(lastLatencyMetricFamily); + + GaugeMetricFamily maxLatencyMetricFamily = + new GaugeMetricFamily( + "report_metrics_operation_max_invocation_latency_ms", + "The maximum observed worker-side ReportMetricsOperation reporting " + + "latency in milliseconds since the worker started, " + + "including local metrics collection and worker-to-master " + + "invocation", + clusterLabelNames(ADDRESS)); + maxLatencyMetricFamily.addMetric(labelValues(address), stats.getMaxInvocationLatencyMs()); + mfs.add(maxLatencyMetricFamily); + return mfs; + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/TelemetryCollectorCoordinatorGuardTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/TelemetryCollectorCoordinatorGuardTest.java index c41a923580f6..03edb148a494 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/TelemetryCollectorCoordinatorGuardTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/TelemetryCollectorCoordinatorGuardTest.java @@ -19,11 +19,14 @@ import org.apache.seatunnel.engine.server.CoordinatorService; import org.apache.seatunnel.engine.server.SeaTunnelServer; +import org.apache.seatunnel.engine.server.TaskExecutionService; import org.apache.seatunnel.engine.server.telemetry.metrics.entity.JobCounter; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.ReportMetricsOperationStats; import org.apache.seatunnel.engine.server.telemetry.metrics.entity.ThreadPoolStatus; import org.apache.seatunnel.engine.server.telemetry.metrics.exports.ClusterMetricExports; import org.apache.seatunnel.engine.server.telemetry.metrics.exports.JobMetricExports; import org.apache.seatunnel.engine.server.telemetry.metrics.exports.JobThreadPoolStatusExports; +import org.apache.seatunnel.engine.server.telemetry.metrics.exports.ReportMetricsOperationExports; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -181,6 +184,76 @@ void testJobThreadPoolStatusExportsReturnsEmptyWhenNotMaster() { Mockito.verify(mockServer, Mockito.never()).isCoordinatorActive(); } + @Test + void testReportMetricsOperationExportsReturnsEmptyWhenTaskExecutionServiceMissing() { + Mockito.when(mockServer.getTaskExecutionService()).thenReturn(null); + + ReportMetricsOperationExports exports = new ReportMetricsOperationExports(mockNode); + List result = exports.collect(); + + Assertions.assertTrue( + result.isEmpty(), + "collect() must return empty when task execution service is unavailable"); + } + + @Test + void testReportMetricsOperationExportsReturnsMetricsWhenTaskExecutionServiceAvailable() { + TaskExecutionService taskExecutionService = Mockito.mock(TaskExecutionService.class); + Mockito.when(mockServer.getTaskExecutionService()).thenReturn(taskExecutionService); + Mockito.when(taskExecutionService.getReportMetricsOperationStats()) + .thenReturn(new ReportMetricsOperationStats(3L, 1L, 1L, 9L, 15L, 27L)); + + ReportMetricsOperationExports exports = new ReportMetricsOperationExports(mockNode); + List result = exports.collect(); + + Assertions.assertFalse( + result.isEmpty(), + "collect() must return metrics when task execution service exists"); + Collector.MetricFamilySamples totalMetric = + result.stream() + .filter(s -> "report_metrics_operation".equals(s.name)) + .findFirst() + .orElse(null); + Assertions.assertNotNull(totalMetric); + Assertions.assertEquals(3, totalMetric.samples.size()); + assertMetricSample(totalMetric, "report_metrics_operation_total", "success", 3D); + assertMetricSample(totalMetric, "report_metrics_operation_total", "failure", 1D); + assertMetricSample(totalMetric, "report_metrics_operation_total", "interrupted", 1D); + + Collector.MetricFamilySamples payloadMetric = + result.stream() + .filter( + s -> + "report_metrics_operation_last_payload_task_count" + .equals(s.name)) + .findFirst() + .orElse(null); + Assertions.assertNotNull(payloadMetric); + assertSingleMetricSample(payloadMetric, 9D); + + Collector.MetricFamilySamples lastLatencyMetric = + result.stream() + .filter( + s -> + "report_metrics_operation_last_invocation_latency_ms" + .equals(s.name)) + .findFirst() + .orElse(null); + Assertions.assertNotNull(lastLatencyMetric); + assertSingleMetricSample(lastLatencyMetric, 15D); + + Collector.MetricFamilySamples maxLatencyMetric = + result.stream() + .filter( + s -> + "report_metrics_operation_max_invocation_latency_ms" + .equals(s.name)) + .findFirst() + .orElse(null); + Assertions.assertNotNull(maxLatencyMetric); + assertSingleMetricSample(maxLatencyMetric, 27D); + } + // ------------------------------------------------------------------------- // ClusterMetricExports // ------------------------------------------------------------------------- @@ -237,4 +310,41 @@ void testClusterMetricExportsSkipsClusterInfoAndLogsWarningWhenMasterAddressUnre Mockito.eq("Skip cluster_info metric: unable to resolve master address"), Mockito.any(UnknownHostException.class)); } + + private void assertMetricSample( + Collector.MetricFamilySamples metricFamilySamples, + String expectedSampleName, + String result, + double expectedValue) { + Collector.MetricFamilySamples.Sample sample = + metricFamilySamples.samples.stream() + .filter( + s -> { + if (!expectedSampleName.equals(s.name)) { + return false; + } + int resultLabelIndex = s.labelNames.indexOf("result"); + return resultLabelIndex >= 0 + && result.equals(s.labelValues.get(resultLabelIndex)); + }) + .findFirst() + .orElse(null); + Assertions.assertNotNull(sample); + assertAddressLabel(sample); + Assertions.assertEquals(expectedValue, sample.value); + } + + private void assertSingleMetricSample( + Collector.MetricFamilySamples metricFamilySamples, double expectedValue) { + Assertions.assertEquals(1, metricFamilySamples.samples.size()); + Collector.MetricFamilySamples.Sample sample = metricFamilySamples.samples.get(0); + assertAddressLabel(sample); + Assertions.assertEquals(expectedValue, sample.value); + } + + private void assertAddressLabel(Collector.MetricFamilySamples.Sample sample) { + int addressLabelIndex = sample.labelNames.indexOf("address"); + Assertions.assertTrue(addressLabelIndex >= 0, "metric sample must contain 'address' label"); + Assertions.assertEquals("127.0.0.1:5801", sample.labelValues.get(addressLabelIndex)); + } } From df28704d9290c0b9ffdb973efb62362128491a93 Mon Sep 17 00:00:00 2001 From: Jast Date: Sun, 21 Jun 2026 12:49:40 +0800 Subject: [PATCH 047/375] [Docs] Update CNCF landscape license link (#11149) --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 28f5385f5bbd..d1c315b69f3f 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Refer to this [Setup](https://seatunnel.apache.org/docs/developer/setup) for com - Twitter: [ASFSeaTunnel on Twitter](https://twitter.com/ASFSeaTunnel) ## Landscapes -SeaTunnel enriches the [CNCF CLOUD NATIVE Landscape](https://landscape.cncf.io/?landscape=observability-and-analysis&license=apache-license-2-0). +SeaTunnel enriches the [CNCF CLOUD NATIVE Landscape](https://landscape.cncf.io/?landscape=observability-and-analysis&license=Apache+License+2.0). ## License [Apache 2.0 License](LICENSE) @@ -93,4 +93,3 @@ More information, please refer to [FAQ](https://seatunnel.apache.org/docs/faq). ### 4. How can I contribute to SeaTunnel? We welcome contributions! Please refer to our [Contribution Guidelines](https://seatunnel.apache.org/docs/developer/coding-guide) for details. - From b5fd7601bd577b2ff2e482a920943258b43694d4 Mon Sep 17 00:00:00 2001 From: David Zollo Date: Sun, 21 Jun 2026 13:53:50 +0800 Subject: [PATCH 048/375] [Docs] Add first-job entry and getting started recipes (#11058) Co-authored-by: Daniel <48329107+DanielLeens@users.noreply.github.com> --- .../locally/quick-start-seatunnel-engine.md | 3 +- .../locally/run-your-first-job.md | 99 +++++++++ .../recipes/file-to-starrocks.md | 145 +++++++++++++ .../getting-started/recipes/http-to-jdbc.md | 126 +++++++++++ docs/en/getting-started/recipes/jdbc-to-s3.md | 137 ++++++++++++ .../recipes/kafka-to-iceberg.md | 138 ++++++++++++ .../recipes/multi-table-cdc.md | 204 ++++++++++++++++++ .../recipes/mysql-cdc-to-doris.md | 177 +++++++++++++++ docs/en/introduction/about.md | 26 ++- docs/sidebars.js | 29 +++ .../locally/quick-start-seatunnel-engine.md | 3 +- .../locally/run-your-first-job.md | 99 +++++++++ .../recipes/file-to-starrocks.md | 145 +++++++++++++ .../getting-started/recipes/http-to-jdbc.md | 126 +++++++++++ docs/zh/getting-started/recipes/jdbc-to-s3.md | 137 ++++++++++++ .../recipes/kafka-to-iceberg.md | 138 ++++++++++++ .../recipes/multi-table-cdc.md | 204 ++++++++++++++++++ .../recipes/mysql-cdc-to-doris.md | 177 +++++++++++++++ docs/zh/introduction/about.md | 26 ++- 19 files changed, 2135 insertions(+), 4 deletions(-) create mode 100644 docs/en/getting-started/locally/run-your-first-job.md create mode 100644 docs/en/getting-started/recipes/file-to-starrocks.md create mode 100644 docs/en/getting-started/recipes/http-to-jdbc.md create mode 100644 docs/en/getting-started/recipes/jdbc-to-s3.md create mode 100644 docs/en/getting-started/recipes/kafka-to-iceberg.md create mode 100644 docs/en/getting-started/recipes/multi-table-cdc.md create mode 100644 docs/en/getting-started/recipes/mysql-cdc-to-doris.md create mode 100644 docs/zh/getting-started/locally/run-your-first-job.md create mode 100644 docs/zh/getting-started/recipes/file-to-starrocks.md create mode 100644 docs/zh/getting-started/recipes/http-to-jdbc.md create mode 100644 docs/zh/getting-started/recipes/jdbc-to-s3.md create mode 100644 docs/zh/getting-started/recipes/kafka-to-iceberg.md create mode 100644 docs/zh/getting-started/recipes/multi-table-cdc.md create mode 100644 docs/zh/getting-started/recipes/mysql-cdc-to-doris.md diff --git a/docs/en/getting-started/locally/quick-start-seatunnel-engine.md b/docs/en/getting-started/locally/quick-start-seatunnel-engine.md index 54adbaf3a63a..6c4eff10e9f6 100644 --- a/docs/en/getting-started/locally/quick-start-seatunnel-engine.md +++ b/docs/en/getting-started/locally/quick-start-seatunnel-engine.md @@ -109,7 +109,7 @@ is a sign to determine whether the command ran successfully or not. The SeaTunnel console will print some logs as below: ```shell -2022-12-19 11:01:45,417 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - output rowType: name, age +2022-12-19 11:01:45,417 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - output rowType: new_name, age 2022-12-19 11:01:46,489 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - subtaskIndex=0 rowIndex=1: SeaTunnelRow#tableId=-1 SeaTunnelRow#kind=INSERT: CpiOd, 8520946 2022-12-19 11:01:46,490 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - subtaskIndex=0 rowIndex=2: SeaTunnelRow#tableId=-1 SeaTunnelRow#kind=INSERT: eQqTs, 1256802974 2022-12-19 11:01:46,490 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - subtaskIndex=0 rowIndex=3: SeaTunnelRow#tableId=-1 SeaTunnelRow#kind=INSERT: UsRgO, 2053193072 @@ -245,6 +245,7 @@ Recommendation: - Start with [Getting Started Overview](../overview.md) if you want a guided reading path across deployment, quick start, and configuration. - Use [Job Configuration Guide](../job-configuration-guide.md) when you are ready to replace the sample source and sink with real connectors. +- If you want runnable source-to-sink examples next, continue with [MySQL CDC to Doris](../recipes/mysql-cdc-to-doris.md), [JDBC to S3](../recipes/jdbc-to-s3.md), [Kafka to Iceberg](../recipes/kafka-to-iceberg.md), [Http to JDBC](../recipes/http-to-jdbc.md), [File to StarRocks](../recipes/file-to-starrocks.md), or [Multi-table CDC](../recipes/multi-table-cdc.md). - Start writing your own config file, choose the [connector](../../connectors/source) you want to use, and configure the parameters according to the connector documentation. - If you want to deploy a multi-node SeaTunnel Engine cluster, continue with [SeaTunnel Engine(Zeta) Deployment](../../engines/zeta/deployment.md). - See [SeaTunnel Engine(Zeta)](../../engines/zeta/about.md) if you want to learn more about SeaTunnel Engine. diff --git a/docs/en/getting-started/locally/run-your-first-job.md b/docs/en/getting-started/locally/run-your-first-job.md new file mode 100644 index 000000000000..9bde8145bf72 --- /dev/null +++ b/docs/en/getting-started/locally/run-your-first-job.md @@ -0,0 +1,99 @@ +--- +sidebar_position: 1 +title: Run your first job +--- + +# Run your first job + +This page gives you the shortest path to a successful SeaTunnel run. The example stays fully local, does not require MySQL, Kafka, or object storage, and helps you confirm that your installation, config parsing, and execution engine are all working. + +## Step 1: Finish local deployment + +Complete [Deployment](deployment.md) first and make sure `bin/seatunnel.sh` is available in your SeaTunnel home directory. + +## Step 2: Install only the plugins this sample needs + +Follow [Deployment > Download The Connector Plugins](deployment.md#download-the-connector-plugins), then keep only `connector-fake` and `connector-console` in `config/plugin_config`: + +```plugin_config +--seatunnel-connectors-- +connector-fake +connector-console +--end-- +``` + +Install the plugins and confirm they were downloaded into `${SEATUNNEL_HOME}/connectors`: + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(fake|console)' +``` + +## Step 3: Use a minimal job + +Save the following config as `config/v2.batch.config.template` or another local file: + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + row.num = 16 + schema = { + fields { + name = "string" + age = "int" + } + } + } +} + +transform { + FieldMapper { + plugin_input = "fake" + plugin_output = "fake1" + field_mapper = { + age = age + name = new_name + } + } +} + +sink { + Console { + plugin_input = "fake1" + } +} +``` + +## Step 4: Run it in local mode + +```shell +cd "apache-seatunnel-${version}" +./bin/seatunnel.sh --config ./config/v2.batch.config.template -m local +``` + +## Expected validation result + +- The process starts successfully without connector loading errors. +- The console prints an `output rowType` line for the mapped fields. +- The console prints 16 rows from `ConsoleSinkWriter`. +- The batch job exits successfully after all rows are written. + +If this works, your basic local path is healthy and you can move on to real pipelines. + +## Next step + +- For the full local walkthrough, continue with [Quick Start With SeaTunnel Engine](quick-start-seatunnel-engine.md). +- For runnable source-to-sink examples, start with these recipes: + - [MySQL CDC to Doris](../recipes/mysql-cdc-to-doris.md) + - [JDBC to S3](../recipes/jdbc-to-s3.md) + - [Kafka to Iceberg](../recipes/kafka-to-iceberg.md) + - [Http to JDBC](../recipes/http-to-jdbc.md) + - [File to StarRocks](../recipes/file-to-starrocks.md) + - [Multi-table CDC](../recipes/multi-table-cdc.md) diff --git a/docs/en/getting-started/recipes/file-to-starrocks.md b/docs/en/getting-started/recipes/file-to-starrocks.md new file mode 100644 index 000000000000..dfa08d8fcfd3 --- /dev/null +++ b/docs/en/getting-started/recipes/file-to-starrocks.md @@ -0,0 +1,145 @@ +--- +sidebar_position: 5 +title: File to StarRocks +--- + +# File to StarRocks + +Use this recipe when you want to import local CSV or text files into StarRocks for fast analytical queries. + +## Prerequisites + +1. Finish [Run your first job](../locally/run-your-first-job.md). + +2. Install the plugins required by this recipe. Follow [Deployment > Download The Connector Plugins](../locally/deployment.md#download-the-connector-plugins), then keep only the plugins below in `config/plugin_config`: + +```plugin_config +--seatunnel-connectors-- +connector-file-local +connector-starrocks +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(file-local|starrocks)' +``` + +3. Put the MySQL JDBC driver required by the StarRocks sink into `${SEATUNNEL_HOME}/lib`, then confirm the jar is visible: + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +``` + +4. Prepare the local input file and make sure the SeaTunnel process can read it: + +```bash +mkdir -p /tmp/seatunnel/input +cat <<'EOF' > /tmp/seatunnel/input/customers.csv +id,name,city,updated_at +1001,Alice,Shanghai,2026-06-12 10:00:00 +1002,Bob,Beijing,2026-06-12 10:05:00 +1003,Carol,Hangzhou,2026-06-12 10:10:00 +EOF +``` + +5. Create the target database and table in StarRocks before running the job. + +## Minimal configuration + +This example reads a local CSV file with a header line and writes the rows to an existing StarRocks primary-key table. + +Prepare the target table first: + +```sql +CREATE DATABASE IF NOT EXISTS sync_demo; + +CREATE TABLE IF NOT EXISTS sync_demo.customers ( + id BIGINT NOT NULL, + name STRING, + city STRING, + updated_at DATETIME +) +ENGINE=OLAP +PRIMARY KEY(id) +DISTRIBUTED BY HASH(id) +PROPERTIES ( + "replication_num" = "1" +); +``` + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + LocalFile { + plugin_output = "customers_file" + path = "/tmp/seatunnel/input/customers.csv" + file_format_type = "csv" + csv_use_header_line = true + schema = { + fields { + id = bigint + name = string + city = string + updated_at = timestamp + } + } + } +} + +sink { + StarRocks { + plugin_input = "customers_file" + nodeUrls = ["starrocks-fe:8030"] + base-url = "jdbc:mysql://starrocks-fe:9030/sync_demo" + username = "root" + password = "" + database = "sync_demo" + table = "customers" + batch_max_rows = 1000 + schema_save_mode = "IGNORE" + starrocks.config = { + format = "JSON" + strip_outer_array = true + } + } +} +``` + +## Run the job + +Save the config as `config/file-to-starrocks.conf`, then run SeaTunnel in local mode: + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/file-to-starrocks.conf -m local +``` + +## Validation result + +1. Run the job and confirm it finishes without StarRocks stream load errors. +2. Check the target table in StarRocks. + +```sql +SELECT COUNT(*) FROM sync_demo.customers; +SELECT id, name, city, updated_at FROM sync_demo.customers ORDER BY id; +``` + +If the imported rows in StarRocks match the file content, the pipeline is working. + +## Common pitfalls + +- `base-url` is missing even though `nodeUrls` is configured. +- The file has a header row, but `csv_use_header_line = true` is not set. +- The source schema does not match the file delimiter or timestamp format. +- The target table was not created before the job. This recipe uses `schema_save_mode = "IGNORE"` because the local file source does not provide primary-key metadata for StarRocks auto DDL. + +## Related docs + +- [LocalFile source](../../connectors/source/LocalFile.md) +- [StarRocks sink](../../connectors/sink/StarRocks.md) diff --git a/docs/en/getting-started/recipes/http-to-jdbc.md b/docs/en/getting-started/recipes/http-to-jdbc.md new file mode 100644 index 000000000000..45bfb660be07 --- /dev/null +++ b/docs/en/getting-started/recipes/http-to-jdbc.md @@ -0,0 +1,126 @@ +--- +sidebar_position: 4 +title: Http to JDBC +--- + +# Http to JDBC + +Use this recipe when you want to pull structured data from an HTTP API and store the result in a relational database. + +## Prerequisites + +1. Finish [Run your first job](../locally/run-your-first-job.md). + +2. Install the plugins required by this recipe. Follow [Deployment > Download The Connector Plugins](../locally/deployment.md#download-the-connector-plugins), then keep only the plugins below in `config/plugin_config`: + +```plugin_config +--seatunnel-connectors-- +connector-http-base +connector-jdbc +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(http-base|jdbc)' +``` + +3. Put the target database JDBC driver into `${SEATUNNEL_HOME}/lib`, then confirm the jar is visible: + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'postgresql' +``` + +4. Inspect the HTTP response before running the job. The sample endpoint from the [Http source](../../connectors/source/Http.md) should return a JSON body that contains top-level fields such as `c_string` and `c_int`: + +```bash +curl http://mockserver:1080/example/http +``` + +If your real API nests the useful records under another field, define `json_field` or `content_field` before you continue. + +5. Prepare the PostgreSQL target database and grant the sink user permission to create tables in `public`, because this recipe uses `generate_sink_sql = true`: + +```sql +CREATE USER test WITH PASSWORD 'test'; +CREATE DATABASE test OWNER test; +``` + +Reconnect to database `test`, then run: + +```sql +GRANT USAGE, CREATE ON SCHEMA public TO test; +``` + +## Minimal configuration + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Http { + plugin_output = "http_orders" + url = "http://mockserver:1080/example/http" + method = "GET" + format = "json" + schema = { + fields { + c_string = string + c_int = int + } + } + } +} + +sink { + Jdbc { + plugin_input = "http_orders" + driver = "org.postgresql.Driver" + url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" + username = "test" + password = "test" + generate_sink_sql = true + database = "test" + table = "public.http_orders" + primary_keys = ["c_string"] + batch_size = 100 + } +} +``` + +## Run the job + +Save the config as `config/http-to-jdbc.conf`, then run SeaTunnel in local mode: + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/http-to-jdbc.conf -m local +``` + +## Validation result + +1. Run the job and confirm there are no HTTP parse or JDBC DDL errors. +2. Query the target table and compare the row count with the API response. + +```sql +SELECT COUNT(*) FROM public.http_orders; +SELECT c_string, c_int FROM public.http_orders ORDER BY c_string; +``` + +If the rows in the target table match the HTTP response, the pipeline is working. With the default mock response, you should see the same `c_string` and `c_int` values you saw in `curl`. + +## Common pitfalls + +- The response body is JSON, but the configured schema does not match the actual field names or types. +- The API data is nested, but `content_field` or `json_field` is not configured. +- Pagination or rate limits exist on the source API, but the job treats it as a single-page endpoint. +- The JDBC sink auto-creates a table, but the chosen primary key does not uniquely identify records. + +## Related docs + +- [Http source](../../connectors/source/Http.md) +- [JDBC sink](../../connectors/sink/Jdbc.md) diff --git a/docs/en/getting-started/recipes/jdbc-to-s3.md b/docs/en/getting-started/recipes/jdbc-to-s3.md new file mode 100644 index 000000000000..161b2c51c569 --- /dev/null +++ b/docs/en/getting-started/recipes/jdbc-to-s3.md @@ -0,0 +1,137 @@ +--- +sidebar_position: 2 +title: JDBC to S3 +--- + +# JDBC to S3 + +Use this recipe when you want to export table data from a relational database to an S3-compatible object store. + +## Prerequisites + +1. Finish [Run your first job](../locally/run-your-first-job.md). + +2. Install the plugins required by this recipe. Follow [Deployment > Download The Connector Plugins](../locally/deployment.md#download-the-connector-plugins), then keep only the plugins below in `config/plugin_config`: + +```plugin_config +--seatunnel-connectors-- +connector-jdbc +connector-file-s3 +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(jdbc|file-s3)' +``` + +3. Put the source database JDBC driver into `${SEATUNNEL_HOME}/lib` for SeaTunnel Zeta, then confirm the jar is visible: + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +``` + +4. Put `hadoop-aws` and the AWS SDK bundle required by the S3 connector into `${SEATUNNEL_HOME}/lib`, then confirm both jars are present: + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'hadoop-aws|aws-java-sdk-bundle' +``` + +5. Prepare the source table in the relational database. This example uses MySQL and exports two rows from `analytics.orders`: + +```sql +CREATE DATABASE IF NOT EXISTS analytics; + +CREATE TABLE IF NOT EXISTS analytics.orders ( + id BIGINT PRIMARY KEY, + customer_id BIGINT, + total_amount DECIMAL(16, 2), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO analytics.orders (id, customer_id, total_amount, updated_at) VALUES + (5001, 101, 19.99, NOW()), + (5002, 102, 29.99, NOW()); +``` + +6. Prepare an S3 bucket and credentials with write permission. The example below assumes the bucket already exists and that `access_key` and `secret_key` can write to `s3://company-data-lake/seatunnel/orders/`. + +## Minimal configuration + +This example exports a query result from MySQL to S3 in JSON lines format so that the output is easy to inspect. + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + plugin_output = "orders_jdbc" + url = "jdbc:mysql://mysql:3306/analytics" + driver = "com.mysql.cj.jdbc.Driver" + username = "root" + password = "password" + query = "select id, customer_id, total_amount, updated_at from orders" + } +} + +sink { + S3File { + plugin_input = "orders_jdbc" + bucket = "s3a://company-data-lake" + path = "/seatunnel/orders/" + fs.s3a.endpoint = "s3.us-east-1.amazonaws.com" + fs.s3a.aws.credentials.provider = "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" + access_key = "your-access-key" + secret_key = "your-secret-key" + file_format_type = "json" + row_delimiter = "\n" + custom_filename = true + file_name_expression = "orders" + filename_extension = "json" + single_file_mode = true + is_enable_transaction = false + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + data_save_mode = "APPEND_DATA" + } +} +``` + +## Run the job + +Save the config as `config/jdbc-to-s3.conf`, then run SeaTunnel in local mode: + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/jdbc-to-s3.conf -m local +``` + +## Validation result + +1. Run the source query directly in the database and record the expected row count. +2. Start the SeaTunnel job. +3. Verify that new objects are written to the expected S3 prefix. + +```bash +aws s3 ls s3://company-data-lake/seatunnel/orders/ --recursive +aws s3 cp s3://company-data-lake/seatunnel/orders/orders.json - | head +``` + +If objects are created under the target prefix and the exported content matches the source query, the pipeline is working. + +## Common pitfalls + +- The JDBC driver is available on your workstation but not under `${SEATUNNEL_HOME}/lib`. +- `bucket` and `path` are mixed up. Keep the bucket in `bucket` and the prefix in `path`. +- The credential provider does not match the authentication method you configured. +- Large tables are exported through one unbounded query without filtering or partitioning. +- Fixed filenames are only safe for this single-file tutorial. If you enable transactions again, keep `${transactionId}` in `file_name_expression`. +- The target endpoint is S3-compatible, but the `fs.s3a.endpoint` value still points to AWS. + +## Related docs + +- [JDBC source](../../connectors/source/Jdbc.md) +- [S3File sink](../../connectors/sink/S3File.md) diff --git a/docs/en/getting-started/recipes/kafka-to-iceberg.md b/docs/en/getting-started/recipes/kafka-to-iceberg.md new file mode 100644 index 000000000000..ab50c867584b --- /dev/null +++ b/docs/en/getting-started/recipes/kafka-to-iceberg.md @@ -0,0 +1,138 @@ +--- +sidebar_position: 3 +title: Kafka to Iceberg +--- + +# Kafka to Iceberg + +Use this recipe when you want to land streaming events from Kafka into an Iceberg table for downstream analytics. + +## Prerequisites + +1. Finish [Run your first job](../locally/run-your-first-job.md). + +2. Install the plugins required by this recipe. Follow [Deployment > Download The Connector Plugins](../locally/deployment.md#download-the-connector-plugins), then keep only the plugins below in `config/plugin_config`: + +```plugin_config +--seatunnel-connectors-- +connector-kafka +connector-iceberg +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(kafka|iceberg)' +``` + +3. If you use Flink or Spark, add the Iceberg dependencies required by your environment, including `hive-exec` and `libfb303` when needed. + +4. Choose an empty, writable Iceberg warehouse path. This tutorial uses a local Hadoop catalog at `file:///tmp/seatunnel/iceberg/warehouse-demo`: + +```bash +mkdir -p /tmp/seatunnel/iceberg/warehouse-demo +``` + +5. Create the Kafka topic and produce a few JSON messages before starting the job: + +```bash +kafka-topics.sh \ + --create \ + --if-not-exists \ + --topic orders \ + --bootstrap-server kafka:9092 \ + --partitions 1 \ + --replication-factor 1 + +kafka-console-producer.sh --topic orders --bootstrap-server kafka:9092 <<'EOF' +{"id":1001,"customer_id":2001,"total_amount":19.99,"event_date":"2026-06-12"} +{"id":1002,"customer_id":2002,"total_amount":29.99,"event_date":"2026-06-12"} +EOF +``` + +## Minimal configuration + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + Kafka { + plugin_output = "orders_kafka" + topic = "orders" + bootstrap.servers = "kafka:9092" + consumer.group = "seatunnel-orders" + start_mode = "earliest" + format = "json" + schema = { + fields { + id = bigint + customer_id = bigint + total_amount = "decimal(16, 2)" + event_date = string + } + } + } +} + +sink { + Iceberg { + plugin_input = "orders_kafka" + catalog_name = "seatunnel_demo" + namespace = "lakehouse" + table = "orders" + iceberg.catalog.config = { + type = "hadoop" + warehouse = "file:///tmp/seatunnel/iceberg/warehouse-demo" + } + iceberg.table.primary-keys = "id" + iceberg.table.partition-keys = "event_date" + iceberg.table.upsert-mode-enabled = true + iceberg.table.schema-evolution-enabled = true + case_sensitive = true + } +} +``` + +## Run the job + +Save the config as `config/kafka-to-iceberg.conf`, then start SeaTunnel in local mode: + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/kafka-to-iceberg.conf -m local +``` + +Keep the job running while Kafka messages are being consumed, because this is a streaming pipeline. + +## Validation result + +1. Verify that Iceberg metadata and data files appear under the warehouse path. +2. Query the Iceberg table with Spark, Trino, or another Iceberg-compatible engine. + +```bash +ls /tmp/seatunnel/iceberg/warehouse-demo/lakehouse/orders +spark-sql \ + --conf spark.sql.catalog.seatunnel_demo=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.seatunnel_demo.type=hadoop \ + --conf spark.sql.catalog.seatunnel_demo.warehouse=file:///tmp/seatunnel/iceberg/warehouse-demo \ + -e "SELECT COUNT(*) FROM seatunnel_demo.lakehouse.orders" +``` + +If the table can be queried and the row count matches the Kafka messages you produced, the pipeline is working. With the two sample messages above, the count should be `2`. + +## Common pitfalls + +- JSON messages in Kafka do not match the schema defined in the source block. +- Checkpointing is disabled in a streaming pipeline, which weakens restart and consistency behavior. +- The Iceberg catalog type is correct, but the warehouse path is not writable by the engine process. +- Upsert mode is enabled even though the incoming records do not have stable primary keys. + +## Related docs + +- [Kafka source](../../connectors/source/Kafka.md) +- [Iceberg sink](../../connectors/sink/Iceberg.md) diff --git a/docs/en/getting-started/recipes/multi-table-cdc.md b/docs/en/getting-started/recipes/multi-table-cdc.md new file mode 100644 index 000000000000..9f49d3521e7c --- /dev/null +++ b/docs/en/getting-started/recipes/multi-table-cdc.md @@ -0,0 +1,204 @@ +--- +sidebar_position: 6 +title: Multi-table CDC +--- + +# Multi-table CDC + +Use this recipe when you want one SeaTunnel job to capture changes from multiple upstream tables and route each table to its own downstream table automatically. + +## Prerequisites + +1. Finish [Run your first job](../locally/run-your-first-job.md). + +2. Install the plugins required by this recipe. Follow [Deployment > Download The Connector Plugins](../locally/deployment.md#download-the-connector-plugins), then keep only the plugins below in `config/plugin_config`: + +```plugin_config +--seatunnel-connectors-- +connector-cdc-mysql +connector-jdbc +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(cdc-mysql|jdbc)' +``` + +3. If you use SeaTunnel Zeta, place both the MySQL JDBC driver and the PostgreSQL JDBC driver into `${SEATUNNEL_HOME}/lib`, then confirm they are visible: + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector|postgresql' +``` + +4. Prepare the MySQL source tables. Each upstream table should have a stable primary key because this recipe routes CDC changes to downstream upsert tables automatically. + +```sql +CREATE DATABASE IF NOT EXISTS inventory; + +CREATE TABLE IF NOT EXISTS inventory.orders ( + id BIGINT PRIMARY KEY, + order_status VARCHAR(32), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS inventory.customers ( + id BIGINT PRIMARY KEY, + customer_name VARCHAR(64), + city VARCHAR(64), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS inventory.products ( + id BIGINT PRIMARY KEY, + product_name VARCHAR(64), + unit_price DECIMAL(10, 2), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +INSERT INTO inventory.orders (id, order_status, updated_at) VALUES + (2001, 'CREATED', NOW()); + +INSERT INTO inventory.customers (id, customer_name, city, updated_at) VALUES + (3001, 'Alice', 'Shanghai', NOW()); + +INSERT INTO inventory.products (id, product_name, unit_price, updated_at) VALUES + (4001, 'Keyboard', 99.00, NOW()); +``` + +5. Create the MySQL CDC user and grant the required privileges: + +```sql +CREATE USER IF NOT EXISTS 'st_user_source'@'%' IDENTIFIED BY 'mysqlpw'; +GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT +ON *.* TO 'st_user_source'@'%'; +FLUSH PRIVILEGES; +``` + +6. Verify that MySQL binlog is ready: + +```sql +SHOW VARIABLES WHERE variable_name IN ('log_bin', 'binlog_format', 'binlog_row_image'); +``` + +The expected values are `log_bin = ON`, `binlog_format = ROW`, and `binlog_row_image = FULL`. + +7. Prepare the PostgreSQL target database and grant the sink user permission to create tables in `public`: + +```sql +CREATE USER st_user_sink WITH PASSWORD 'pgpw'; +CREATE DATABASE sync_demo; +GRANT ALL PRIVILEGES ON DATABASE sync_demo TO st_user_sink; +``` + +Reconnect to `sync_demo`, then run: + +```sql +GRANT USAGE, CREATE ON SCHEMA public TO st_user_sink; +``` + +This recipe uses `generate_sink_sql = true`, so SeaTunnel will create tables such as `public.st_orders` and `public.st_customers` automatically on the first run. + +## Minimal configuration + +This example reads multiple MySQL tables through one `table-pattern` and writes them to PostgreSQL tables named `st_`. + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + MySQL-CDC { + plugin_output = "mysql_multi" + startup.mode = "initial" + server-id = 5652 + username = "st_user_source" + password = "mysqlpw" + database-pattern = "inventory" + table-pattern = "inventory\\.(orders|customers|products)" + url = "jdbc:mysql://mysql:3306/inventory" + } +} + +sink { + Jdbc { + plugin_input = "mysql_multi" + driver = "org.postgresql.Driver" + url = "jdbc:postgresql://postgresql:5432/sync_demo" + username = "st_user_sink" + password = "pgpw" + generate_sink_sql = true + database = "sync_demo" + table = "public.st_${table_name}" + primary_keys = ["${primary_key}"] + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + data_save_mode = "APPEND_DATA" + } +} +``` + +## Run the job + +Save the config as `config/multi-table-cdc.conf`, then start SeaTunnel in local mode: + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/multi-table-cdc.conf -m local +``` + +Keep the job running while you verify new changes from MySQL, because this is a streaming CDC pipeline. + +## Validation result + +1. Start the job and let the initial snapshot finish. +2. Confirm that SeaTunnel created multiple target tables in PostgreSQL: + +```sql +SELECT table_name +FROM information_schema.tables +WHERE table_schema = 'public' AND table_name LIKE 'st_%' +ORDER BY table_name; +``` + +3. Apply changes on each MySQL source table: + +```sql +INSERT INTO inventory.orders (id, order_status, updated_at) +VALUES (2002, 'PAID', NOW()); + +UPDATE inventory.customers +SET city = 'Hangzhou', updated_at = NOW() +WHERE id = 3001; + +INSERT INTO inventory.products (id, product_name, unit_price, updated_at) +VALUES (4002, 'Mouse', 59.00, NOW()); +``` + +4. Verify that each downstream table received only its own source data: + +```sql +SELECT id, order_status FROM public.st_orders ORDER BY id; +SELECT id, customer_name, city FROM public.st_customers ORDER BY id; +SELECT id, product_name, unit_price FROM public.st_products ORDER BY id; +``` + +If each upstream table is routed to its own target table and changes continue to flow, the multi-table CDC pipeline is working. + +## Common pitfalls + +- The regular expression in `table-pattern` is not escaped correctly. In HOCON, `.` usually needs `\\.` when you mean a literal dot. +- MySQL binlog or CDC user privileges are incomplete, so the job can read the snapshot but cannot continue reading incremental changes. +- Placeholder-based sink routing is not configured, so multiple source tables are written into one target table accidentally. +- The PostgreSQL sink user can connect to the database but does not have `CREATE` permission on schema `public`. +- Upstream tables do not have primary keys, but the sink is configured as if upsert semantics were available. +- The downstream naming convention is valid for one database but invalid for another because schema and table placeholders are used differently. + +## Related docs + +- [MySQL CDC source](../../connectors/source/MySQL-CDC.md) +- [JDBC sink](../../connectors/sink/Jdbc.md) +- [Multi-table synchronization architecture](../../architecture/features/multi-table.md) diff --git a/docs/en/getting-started/recipes/mysql-cdc-to-doris.md b/docs/en/getting-started/recipes/mysql-cdc-to-doris.md new file mode 100644 index 000000000000..e5e2fab1b92f --- /dev/null +++ b/docs/en/getting-started/recipes/mysql-cdc-to-doris.md @@ -0,0 +1,177 @@ +--- +sidebar_position: 1 +title: MySQL CDC to Doris +--- + +# MySQL CDC to Doris + +Use this recipe when you want to capture row-level changes from MySQL and keep a Doris table updated continuously. + +## Prerequisites + +1. Finish [Run your first job](../locally/run-your-first-job.md) and make sure local execution works. + +2. Install the plugins required by this recipe. Follow [Deployment > Download The Connector Plugins](../locally/deployment.md#download-the-connector-plugins), then keep only the plugins below in `config/plugin_config`: + +```plugin_config +--seatunnel-connectors-- +connector-cdc-mysql +connector-doris +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(cdc-mysql|doris)' +``` + +3. If you use SeaTunnel Zeta, download the MySQL JDBC driver and place it in `${SEATUNNEL_HOME}/lib`, then confirm the jar is visible: + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +``` + +If you use Flink or Spark instead of Zeta, put the same driver jar into the engine plugin directory that your runtime loads. + +4. Prepare the MySQL source table. This recipe relies on a stable primary key so that updates and deletes can be replayed correctly downstream. + +```sql +CREATE DATABASE IF NOT EXISTS inventory; + +CREATE TABLE IF NOT EXISTS inventory.orders ( + id BIGINT PRIMARY KEY, + order_status VARCHAR(32), + amount DECIMAL(10, 2), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +INSERT INTO inventory.orders (id, order_status, amount, updated_at) VALUES + (1001, 'CREATED', 19.99, NOW()), + (1002, 'CREATED', 29.99, NOW()); +``` + +5. Create the MySQL CDC user and grant the same privileges required by the [MySQL CDC source](../../connectors/source/MySQL-CDC.md): + +```sql +CREATE USER IF NOT EXISTS 'st_user_source'@'%' IDENTIFIED BY 'mysqlpw'; +GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT +ON *.* TO 'st_user_source'@'%'; +FLUSH PRIVILEGES; +``` + +6. Verify that MySQL binlog is ready for CDC: + +```sql +SHOW VARIABLES WHERE variable_name IN ('log_bin', 'binlog_format', 'binlog_row_image'); +``` + +The expected values are `log_bin = ON`, `binlog_format = ROW`, and `binlog_row_image = FULL`. +If they are not set yet, update `my.cnf` and restart MySQL: + +```ini +[mysqld] +server-id = 223344 +log_bin = mysql-bin +binlog_format = ROW +binlog_row_image = FULL +``` + +7. Prepare the Doris target database. This recipe keeps `schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST"`, so SeaTunnel will create `sync_demo.orders` automatically from the MySQL primary-key metadata on first startup. + +```sql +CREATE DATABASE IF NOT EXISTS sync_demo; +``` + +## Minimal configuration + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + MySQL-CDC { + plugin_output = "orders_cdc" + parallelism = 1 + startup.mode = "initial" + server-id = 5652 + username = "st_user_source" + password = "mysqlpw" + table-names = ["inventory.orders"] + url = "jdbc:mysql://mysql:3306/inventory" + } +} + +sink { + Doris { + plugin_input = "orders_cdc" + fenodes = "doris-fe:8030" + username = "root" + password = "" + database = "sync_demo" + table = "orders" + sink.label-prefix = "orders-cdc" + sink.enable-delete = true + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + doris.config = { + format = "csv" + column_separator = "," + } + } +} +``` + +## Run the job + +Save the config as `config/mysql-cdc-to-doris.conf`, then start SeaTunnel in local mode: + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/mysql-cdc-to-doris.conf -m local +``` + +Keep the job running while you execute the validation SQL below, because this is a streaming CDC pipeline. + +## Validation result + +1. Start the job and wait for the initial snapshot to finish. +2. Change the source rows in MySQL: + +```sql +INSERT INTO inventory.orders (id, order_status, amount, updated_at) +VALUES (1003, 'CREATED', 39.99, NOW()); + +UPDATE inventory.orders +SET order_status = 'PAID', updated_at = NOW() +WHERE id = 1001; + +DELETE FROM inventory.orders +WHERE id = 1002; +``` + +3. Query Doris and confirm that the latest state is visible there: + +```sql +SELECT COUNT(*) FROM sync_demo.orders; +SELECT id, order_status, amount FROM sync_demo.orders ORDER BY id; +``` + +You should now see row `1001` with status `PAID`, row `1003` with status `CREATED`, and no remaining row `1002`. If inserts, updates, and deletes from MySQL are reflected in Doris, the pipeline is working. + +## Common pitfalls + +- MySQL binlog is not enabled or is not using `ROW` format. +- The CDC user is missing replication privileges. +- The `server-id` conflicts with another MySQL replica or another CDC job. +- `sink.label-prefix` is reused across multiple running jobs, which can cause Doris stream load conflicts. +- Delete propagation is enabled, but the Doris table model does not support the expected delete behavior. +- The source table has no stable primary key, so Doris auto-create and downstream upsert behavior are not deterministic. + +## Related docs + +- [MySQL CDC source](../../connectors/source/MySQL-CDC.md) +- [Doris sink](../../connectors/sink/Doris.md) +- [SeaTunnel Engine quick start](../locally/quick-start-seatunnel-engine.md) diff --git a/docs/en/introduction/about.md b/docs/en/introduction/about.md index e229144970dc..99d51a5cf456 100644 --- a/docs/en/introduction/about.md +++ b/docs/en/introduction/about.md @@ -11,6 +11,30 @@ SeaTunnel is a multimodal, ultra-high-performance, distributed data integration If you are new to SeaTunnel, use this short reading path: + + - [Getting Started Overview](../getting-started/overview.md) for the shortest path into the docs - [Quick Start With SeaTunnel Engine](../getting-started/locally/quick-start-seatunnel-engine.md) for the first local run - [Job Configuration Guide](../getting-started/job-configuration-guide.md) for writing real jobs @@ -87,4 +111,4 @@ SeaTunnel enriches the , age +2022-12-19 11:01:45,417 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - output rowType: new_name, age 2022-12-19 11:01:46,489 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - subtaskIndex=0 rowIndex=1: SeaTunnelRow#tableId=-1 SeaTunnelRow#kind=INSERT: CpiOd, 8520946 2022-12-19 11:01:46,490 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - subtaskIndex=0 rowIndex=2: SeaTunnelRow#tableId=-1 SeaTunnelRow#kind=INSERT: eQqTs, 1256802974 2022-12-19 11:01:46,490 INFO org.apache.seatunnel.connectors.seatunnel.console.sink.ConsoleSinkWriter - subtaskIndex=0 rowIndex=3: SeaTunnelRow#tableId=-1 SeaTunnelRow#kind=INSERT: UsRgO, 2053193072 @@ -243,6 +243,7 @@ Total Failed Count : 0 - 如果你想先建立整体路径感，可以返回阅读[快速入门总览](../overview.md)。 - 当你准备把示例 Source 和 Sink 替换成真实连接器时，建议继续阅读[作业配置指南](../job-configuration-guide.md)。 +- 如果你下一步就想看可直接照着改的源端到目标端示例，可以继续看 [MySQL CDC 到 Doris](../recipes/mysql-cdc-to-doris.md)、[JDBC 到 S3](../recipes/jdbc-to-s3.md)、[Kafka 到 Iceberg](../recipes/kafka-to-iceberg.md)、[Http 到 JDBC](../recipes/http-to-jdbc.md)、[File 到 StarRocks](../recipes/file-to-starrocks.md) 和 [多表 CDC](../recipes/multi-table-cdc.md)。 - 开始编写您自己的配置文件，选择您想要使用的[连接器](../../connectors/source)，并根据连接器的文档配置参数。 - 如果您要部署多节点 SeaTunnel Engine 集群，请继续阅读[SeaTunnel Engine(Zeta) 安装部署](../../engines/zeta/deployment.md)。 - 如果您想进一步了解 SeaTunnel Engine，请参阅[SeaTunnel引擎](../../engines/zeta/about.md)。 diff --git a/docs/zh/getting-started/locally/run-your-first-job.md b/docs/zh/getting-started/locally/run-your-first-job.md new file mode 100644 index 000000000000..aaadfadf00e4 --- /dev/null +++ b/docs/zh/getting-started/locally/run-your-first-job.md @@ -0,0 +1,99 @@ +--- +sidebar_position: 1 +title: 跑第一个任务 +--- + +# 跑第一个任务 + +这一页只解决一件事：用最短路径把 SeaTunnel 真正跑起来。这个示例完全本地运行，不依赖 MySQL、Kafka 或对象存储，适合先确认安装、配置解析和执行引擎都正常。 + +## 步骤 1：先完成本地部署 + +先完成 [部署](deployment.md)，并确认 SeaTunnel 目录下已经有 `bin/seatunnel.sh`。 + +## 步骤 2：只安装这篇示例真正需要的插件 + +先看 [部署 > 下载连接器插件](deployment.md#下载连接器插件)，然后把 `config/plugin_config` 收敛成下面两个插件： + +```plugin_config +--seatunnel-connectors-- +connector-fake +connector-console +--end-- +``` + +接着执行安装命令，并确认插件已经下载到 `${SEATUNNEL_HOME}/connectors`： + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(fake|console)' +``` + +## 步骤 3：使用最小可运行配置 + +把下面的配置保存为 `config/v2.batch.config.template`，或者保存为你自己的本地配置文件： + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + row.num = 16 + schema = { + fields { + name = "string" + age = "int" + } + } + } +} + +transform { + FieldMapper { + plugin_input = "fake" + plugin_output = "fake1" + field_mapper = { + age = age + name = new_name + } + } +} + +sink { + Console { + plugin_input = "fake1" + } +} +``` + +## 步骤 4：用本地模式运行 + +```shell +cd "apache-seatunnel-${version}" +./bin/seatunnel.sh --config ./config/v2.batch.config.template -m local +``` + +## 验证结果 + +- 任务可以正常启动，没有 connector 加载错误。 +- 控制台会打印映射后字段的 `output rowType` 行。 +- 控制台会打印 16 行 `ConsoleSinkWriter` 输出。 +- 批任务在写完全部数据后正常退出。 + +如果这里已经跑通，说明本地基础链路是正常的，后面就可以切到真实数据源和真实目标端。 + +## 下一步 + +- 如果你想看完整本地链路，请继续看 [SeaTunnel 引擎快速开始](quick-start-seatunnel-engine.md)。 +- 如果你想直接看真实源端到目标端示例，请从下面这些教程开始： + - [MySQL CDC 到 Doris](../recipes/mysql-cdc-to-doris.md) + - [JDBC 到 S3](../recipes/jdbc-to-s3.md) + - [Kafka 到 Iceberg](../recipes/kafka-to-iceberg.md) + - [Http 到 JDBC](../recipes/http-to-jdbc.md) + - [File 到 StarRocks](../recipes/file-to-starrocks.md) + - [多表 CDC](../recipes/multi-table-cdc.md) diff --git a/docs/zh/getting-started/recipes/file-to-starrocks.md b/docs/zh/getting-started/recipes/file-to-starrocks.md new file mode 100644 index 000000000000..167356cf9920 --- /dev/null +++ b/docs/zh/getting-started/recipes/file-to-starrocks.md @@ -0,0 +1,145 @@ +--- +sidebar_position: 5 +title: File 到 StarRocks +--- + +# File 到 StarRocks + +当你想把本地 CSV 或文本文件导入 StarRocks，供后续高性能分析查询使用时，可以使用这条链路。 + +## 前置条件 + +1. 先完成 [跑第一个任务](../locally/run-your-first-job.md)。 + +2. 安装这条链路需要的插件。先看 [部署 > 下载连接器插件](../locally/deployment.md#下载连接器插件)，然后把 `config/plugin_config` 改成下面这样： + +```plugin_config +--seatunnel-connectors-- +connector-file-local +connector-starrocks +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(file-local|starrocks)' +``` + +3. 把 StarRocks sink 依赖的 MySQL JDBC 驱动放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经落盘： + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +``` + +4. 先准备本地输入文件，并确保 SeaTunnel 进程能读到它： + +```bash +mkdir -p /tmp/seatunnel/input +cat <<'EOF' > /tmp/seatunnel/input/customers.csv +id,name,city,updated_at +1001,Alice,Shanghai,2026-06-12 10:00:00 +1002,Bob,Beijing,2026-06-12 10:05:00 +1003,Carol,Hangzhou,2026-06-12 10:10:00 +EOF +``` + +5. 运行任务前，先在 StarRocks 中创建好目标库和目标表。 + +## 最小配置 + +下面的示例读取一个带表头的本地 CSV 文件，并把数据写入已经存在的 StarRocks 主键表。 + +先创建目标表： + +```sql +CREATE DATABASE IF NOT EXISTS sync_demo; + +CREATE TABLE IF NOT EXISTS sync_demo.customers ( + id BIGINT NOT NULL, + name STRING, + city STRING, + updated_at DATETIME +) +ENGINE=OLAP +PRIMARY KEY(id) +DISTRIBUTED BY HASH(id) +PROPERTIES ( + "replication_num" = "1" +); +``` + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + LocalFile { + plugin_output = "customers_file" + path = "/tmp/seatunnel/input/customers.csv" + file_format_type = "csv" + csv_use_header_line = true + schema = { + fields { + id = bigint + name = string + city = string + updated_at = timestamp + } + } + } +} + +sink { + StarRocks { + plugin_input = "customers_file" + nodeUrls = ["starrocks-fe:8030"] + base-url = "jdbc:mysql://starrocks-fe:9030/sync_demo" + username = "root" + password = "" + database = "sync_demo" + table = "customers" + batch_max_rows = 1000 + schema_save_mode = "IGNORE" + starrocks.config = { + format = "JSON" + strip_outer_array = true + } + } +} +``` + +## 运行任务 + +把配置保存为 `config/file-to-starrocks.conf`，然后用本地模式运行 SeaTunnel： + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/file-to-starrocks.conf -m local +``` + +## 验证结果 + +1. 运行任务，确认没有 StarRocks stream load 错误。 +2. 在 StarRocks 中查询目标表。 + +```sql +SELECT COUNT(*) FROM sync_demo.customers; +SELECT id, name, city, updated_at FROM sync_demo.customers ORDER BY id; +``` + +如果 StarRocks 中的导入结果和文件内容一致，这条链路就是通的。 + +## 常见坑 + +- 配了 `nodeUrls`，但漏了 `base-url`。 +- 文件带表头，但没有设置 `csv_use_header_line = true`。 +- 源文件 schema、分隔符、时间格式和实际文件内容不一致。 +- 运行前没有先创建目标表。本教程使用 `schema_save_mode = "IGNORE"`，因为本地文件源不会提供 StarRocks 自动建表需要的主键元数据。 + +## 相关文档 + +- [LocalFile Source](../../connectors/source/LocalFile.md) +- [StarRocks Sink](../../connectors/sink/StarRocks.md) diff --git a/docs/zh/getting-started/recipes/http-to-jdbc.md b/docs/zh/getting-started/recipes/http-to-jdbc.md new file mode 100644 index 000000000000..420d02ee18c0 --- /dev/null +++ b/docs/zh/getting-started/recipes/http-to-jdbc.md @@ -0,0 +1,126 @@ +--- +sidebar_position: 4 +title: Http 到 JDBC +--- + +# Http 到 JDBC + +当你想从 HTTP API 拉取结构化数据，并把结果落到关系型数据库中时，可以使用这条链路。 + +## 前置条件 + +1. 先完成 [跑第一个任务](../locally/run-your-first-job.md)。 + +2. 安装这条链路需要的插件。先看 [部署 > 下载连接器插件](../locally/deployment.md#下载连接器插件)，然后把 `config/plugin_config` 改成下面这样： + +```plugin_config +--seatunnel-connectors-- +connector-http-base +connector-jdbc +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(http-base|jdbc)' +``` + +3. 把目标数据库 JDBC 驱动放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经落盘： + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'postgresql' +``` + +4. 运行任务前，先看一眼 HTTP 返回内容。这里直接使用 [Http Source](../../connectors/source/Http.md) 里的示例接口，返回 JSON 顶层应该能看到 `c_string` 和 `c_int` 这些字段： + +```bash +curl http://mockserver:1080/example/http +``` + +如果你的真实接口把有效数据包在更深层字段里，就要先补 `json_field` 或 `content_field`，否则别急着运行。 + +5. 先准备 PostgreSQL 目标库，并给 sink 用户授予在 `public` schema 自动建表的权限，因为这篇教程使用了 `generate_sink_sql = true`： + +```sql +CREATE USER test WITH PASSWORD 'test'; +CREATE DATABASE test OWNER test; +``` + +重新连接到 `test` 库以后，再执行： + +```sql +GRANT USAGE, CREATE ON SCHEMA public TO test; +``` + +## 最小配置 + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Http { + plugin_output = "http_orders" + url = "http://mockserver:1080/example/http" + method = "GET" + format = "json" + schema = { + fields { + c_string = string + c_int = int + } + } + } +} + +sink { + Jdbc { + plugin_input = "http_orders" + driver = "org.postgresql.Driver" + url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" + username = "test" + password = "test" + generate_sink_sql = true + database = "test" + table = "public.http_orders" + primary_keys = ["c_string"] + batch_size = 100 + } +} +``` + +## 运行任务 + +把配置保存为 `config/http-to-jdbc.conf`，然后用本地模式运行 SeaTunnel： + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/http-to-jdbc.conf -m local +``` + +## 验证结果 + +1. 运行任务，确认没有 HTTP 解析错误和 JDBC DDL 错误。 +2. 查询目标表，核对行数和 API 返回结果。 + +```sql +SELECT COUNT(*) FROM public.http_orders; +SELECT c_string, c_int FROM public.http_orders ORDER BY c_string; +``` + +如果目标表里的数据和 HTTP 返回内容一致，这条链路就是通的。使用默认 mock 返回时，查询结果里应该能看到和 `curl` 输出一致的 `c_string`、`c_int` 值。 + +## 常见坑 + +- 返回体是 JSON，但 schema 中字段名或字段类型写错了。 +- API 数据是嵌套结构，但没有配置 `content_field` 或 `json_field`。 +- 源接口有分页或限流，但作业按单页接口处理。 +- JDBC sink 虽然自动建表了，但你选的主键并不能真正唯一标识一条记录。 + +## 相关文档 + +- [Http Source](../../connectors/source/Http.md) +- [JDBC Sink](../../connectors/sink/Jdbc.md) diff --git a/docs/zh/getting-started/recipes/jdbc-to-s3.md b/docs/zh/getting-started/recipes/jdbc-to-s3.md new file mode 100644 index 000000000000..2ea7a42e79ef --- /dev/null +++ b/docs/zh/getting-started/recipes/jdbc-to-s3.md @@ -0,0 +1,137 @@ +--- +sidebar_position: 2 +title: JDBC 到 S3 +--- + +# JDBC 到 S3 + +当你想把关系型数据库里的表数据批量导出到 S3 或兼容 S3 的对象存储时，可以使用这条链路。 + +## 前置条件 + +1. 先完成 [跑第一个任务](../locally/run-your-first-job.md)。 + +2. 安装这条链路需要的插件。先看 [部署 > 下载连接器插件](../locally/deployment.md#下载连接器插件)，然后把 `config/plugin_config` 改成下面这样： + +```plugin_config +--seatunnel-connectors-- +connector-jdbc +connector-file-s3 +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(jdbc|file-s3)' +``` + +3. 把源端数据库 JDBC 驱动放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经落盘： + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +``` + +4. 把 S3 连接器依赖的 `hadoop-aws` 和 AWS SDK bundle 也放进 `${SEATUNNEL_HOME}/lib`，然后确认这两个依赖都能看到： + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'hadoop-aws|aws-java-sdk-bundle' +``` + +5. 先准备源端数据库表。这个示例使用 MySQL，并从 `analytics.orders` 导出两条数据： + +```sql +CREATE DATABASE IF NOT EXISTS analytics; + +CREATE TABLE IF NOT EXISTS analytics.orders ( + id BIGINT PRIMARY KEY, + customer_id BIGINT, + total_amount DECIMAL(16, 2), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO analytics.orders (id, customer_id, total_amount, updated_at) VALUES + (5001, 101, 19.99, NOW()), + (5002, 102, 29.99, NOW()); +``` + +6. 准备好可写入的 S3 bucket 和访问凭据。下面的示例默认目标 bucket 已存在，而且 `access_key` 与 `secret_key` 对 `s3://company-data-lake/seatunnel/orders/` 有写权限。 + +## 最小配置 + +这个示例把 MySQL 查询结果导出成 JSON Lines，便于直接检查输出内容。 + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + plugin_output = "orders_jdbc" + url = "jdbc:mysql://mysql:3306/analytics" + driver = "com.mysql.cj.jdbc.Driver" + username = "root" + password = "password" + query = "select id, customer_id, total_amount, updated_at from orders" + } +} + +sink { + S3File { + plugin_input = "orders_jdbc" + bucket = "s3a://company-data-lake" + path = "/seatunnel/orders/" + fs.s3a.endpoint = "s3.us-east-1.amazonaws.com" + fs.s3a.aws.credentials.provider = "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" + access_key = "your-access-key" + secret_key = "your-secret-key" + file_format_type = "json" + row_delimiter = "\n" + custom_filename = true + file_name_expression = "orders" + filename_extension = "json" + single_file_mode = true + is_enable_transaction = false + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + data_save_mode = "APPEND_DATA" + } +} +``` + +## 运行任务 + +把配置保存为 `config/jdbc-to-s3.conf`，然后用本地模式运行 SeaTunnel： + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/jdbc-to-s3.conf -m local +``` + +## 验证结果 + +1. 先在数据库里直接执行源查询，记录预期行数。 +2. 启动 SeaTunnel 作业。 +3. 检查目标 S3 前缀下是否生成了新对象。 + +```bash +aws s3 ls s3://company-data-lake/seatunnel/orders/ --recursive +aws s3 cp s3://company-data-lake/seatunnel/orders/orders.json - | head +``` + +如果目标前缀下生成了对象，且内容和源查询结果一致，这条链路就是通的。 + +## 常见坑 + +- JDBC 驱动在本机有，但没有放进 `${SEATUNNEL_HOME}/lib`。 +- `bucket` 和 `path` 写反了。`bucket` 写桶，`path` 写桶内前缀。 +- 凭据提供器和你实际配置的认证方式不匹配。 +- 大表直接跑一个无边界 `query`，没有做过滤或分片。 +- 固定文件名只适合这个单文件教程。如果重新开启事务，`file_name_expression` 里必须保留 `${transactionId}`。 +- 目标是兼容 S3 的对象存储，但 `fs.s3a.endpoint` 还在指向 AWS 默认地址。 + +## 相关文档 + +- [JDBC Source](../../connectors/source/Jdbc.md) +- [S3File Sink](../../connectors/sink/S3File.md) diff --git a/docs/zh/getting-started/recipes/kafka-to-iceberg.md b/docs/zh/getting-started/recipes/kafka-to-iceberg.md new file mode 100644 index 000000000000..bb056feba8cb --- /dev/null +++ b/docs/zh/getting-started/recipes/kafka-to-iceberg.md @@ -0,0 +1,138 @@ +--- +sidebar_position: 3 +title: Kafka 到 Iceberg +--- + +# Kafka 到 Iceberg + +当你想把 Kafka 里的流式事件落到 Iceberg 表中，供后续分析查询使用时，可以使用这条链路。 + +## 前置条件 + +1. 先完成 [跑第一个任务](../locally/run-your-first-job.md)。 + +2. 安装这条链路需要的插件。先看 [部署 > 下载连接器插件](../locally/deployment.md#下载连接器插件)，然后把 `config/plugin_config` 改成下面这样： + +```plugin_config +--seatunnel-connectors-- +connector-kafka +connector-iceberg +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(kafka|iceberg)' +``` + +3. 如果你使用 Flink 或 Spark，请补齐 Iceberg 在对应环境里需要的依赖，例如 `hive-exec` 和 `libfb303`。 + +4. 先选一个空的、当前进程可写的 Iceberg warehouse 目录。这篇教程使用本地 Hadoop catalog，对应路径是 `file:///tmp/seatunnel/iceberg/warehouse-demo`： + +```bash +mkdir -p /tmp/seatunnel/iceberg/warehouse-demo +``` + +5. 在启动任务前，先创建 Kafka topic 并写入几条 JSON 消息： + +```bash +kafka-topics.sh \ + --create \ + --if-not-exists \ + --topic orders \ + --bootstrap-server kafka:9092 \ + --partitions 1 \ + --replication-factor 1 + +kafka-console-producer.sh --topic orders --bootstrap-server kafka:9092 <<'EOF' +{"id":1001,"customer_id":2001,"total_amount":19.99,"event_date":"2026-06-12"} +{"id":1002,"customer_id":2002,"total_amount":29.99,"event_date":"2026-06-12"} +EOF +``` + +## 最小配置 + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + Kafka { + plugin_output = "orders_kafka" + topic = "orders" + bootstrap.servers = "kafka:9092" + consumer.group = "seatunnel-orders" + start_mode = "earliest" + format = "json" + schema = { + fields { + id = bigint + customer_id = bigint + total_amount = "decimal(16, 2)" + event_date = string + } + } + } +} + +sink { + Iceberg { + plugin_input = "orders_kafka" + catalog_name = "seatunnel_demo" + namespace = "lakehouse" + table = "orders" + iceberg.catalog.config = { + type = "hadoop" + warehouse = "file:///tmp/seatunnel/iceberg/warehouse-demo" + } + iceberg.table.primary-keys = "id" + iceberg.table.partition-keys = "event_date" + iceberg.table.upsert-mode-enabled = true + iceberg.table.schema-evolution-enabled = true + case_sensitive = true + } +} +``` + +## 运行任务 + +把配置保存为 `config/kafka-to-iceberg.conf`，然后用本地模式启动 SeaTunnel： + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/kafka-to-iceberg.conf -m local +``` + +这是一条流式任务，所以 Kafka 消息被消费和落表时，任务需要保持运行中。 + +## 验证结果 + +1. 检查 warehouse 路径下是否生成了 Iceberg 元数据和数据文件。 +2. 使用 Spark、Trino 或其他 Iceberg 兼容引擎查询表。 + +```bash +ls /tmp/seatunnel/iceberg/warehouse-demo/lakehouse/orders +spark-sql \ + --conf spark.sql.catalog.seatunnel_demo=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.seatunnel_demo.type=hadoop \ + --conf spark.sql.catalog.seatunnel_demo.warehouse=file:///tmp/seatunnel/iceberg/warehouse-demo \ + -e "SELECT COUNT(*) FROM seatunnel_demo.lakehouse.orders" +``` + +如果表可以正常查询，且行数和你写入 Kafka 的消息数量一致，这条链路就是通的。按照上面两条样例消息，最终行数应该是 `2`。 + +## 常见坑 + +- Kafka 中的 JSON 消息结构和 source 里定义的 schema 不一致。 +- 流作业没有开启 checkpoint，导致重启和一致性行为变弱。 +- Iceberg catalog 类型对了，但 warehouse 路径对当前引擎进程不可写。 +- 开启了 upsert 模式，但消息并没有稳定主键。 + +## 相关文档 + +- [Kafka Source](../../connectors/source/Kafka.md) +- [Iceberg Sink](../../connectors/sink/Iceberg.md) diff --git a/docs/zh/getting-started/recipes/multi-table-cdc.md b/docs/zh/getting-started/recipes/multi-table-cdc.md new file mode 100644 index 000000000000..0a5e30c1cc52 --- /dev/null +++ b/docs/zh/getting-started/recipes/multi-table-cdc.md @@ -0,0 +1,204 @@ +--- +sidebar_position: 6 +title: 多表 CDC +--- + +# 多表 CDC + +当你想用一个 SeaTunnel 作业同时采集多个上游表的变更，并自动把每张表路由到各自下游表时，可以使用这条链路。 + +## 前置条件 + +1. 先完成 [跑第一个任务](../locally/run-your-first-job.md)。 + +2. 安装这条链路需要的插件。先看 [部署 > 下载连接器插件](../locally/deployment.md#下载连接器插件)，然后把 `config/plugin_config` 改成下面这样： + +```plugin_config +--seatunnel-connectors-- +connector-cdc-mysql +connector-jdbc +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(cdc-mysql|jdbc)' +``` + +3. 如果你用的是 SeaTunnel Zeta，再把 MySQL JDBC 驱动和 PostgreSQL JDBC 驱动都放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经能看到： + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector|postgresql' +``` + +4. 先准备 MySQL 源表。因为这条链路会自动把 CDC 事件路由到下游 upsert 表，所以每张上游表都要有稳定主键。 + +```sql +CREATE DATABASE IF NOT EXISTS inventory; + +CREATE TABLE IF NOT EXISTS inventory.orders ( + id BIGINT PRIMARY KEY, + order_status VARCHAR(32), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS inventory.customers ( + id BIGINT PRIMARY KEY, + customer_name VARCHAR(64), + city VARCHAR(64), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS inventory.products ( + id BIGINT PRIMARY KEY, + product_name VARCHAR(64), + unit_price DECIMAL(10, 2), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +INSERT INTO inventory.orders (id, order_status, updated_at) VALUES + (2001, 'CREATED', NOW()); + +INSERT INTO inventory.customers (id, customer_name, city, updated_at) VALUES + (3001, 'Alice', 'Shanghai', NOW()); + +INSERT INTO inventory.products (id, product_name, unit_price, updated_at) VALUES + (4001, 'Keyboard', 99.00, NOW()); +``` + +5. 创建 MySQL CDC 用户并授权： + +```sql +CREATE USER IF NOT EXISTS 'st_user_source'@'%' IDENTIFIED BY 'mysqlpw'; +GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT +ON *.* TO 'st_user_source'@'%'; +FLUSH PRIVILEGES; +``` + +6. 检查 MySQL binlog 是否满足 CDC 要求： + +```sql +SHOW VARIABLES WHERE variable_name IN ('log_bin', 'binlog_format', 'binlog_row_image'); +``` + +期望值是 `log_bin = ON`、`binlog_format = ROW`、`binlog_row_image = FULL`。 + +7. 准备 PostgreSQL 目标库，并给 sink 用户授予在 `public` schema 自动建表的权限： + +```sql +CREATE USER st_user_sink WITH PASSWORD 'pgpw'; +CREATE DATABASE sync_demo; +GRANT ALL PRIVILEGES ON DATABASE sync_demo TO st_user_sink; +``` + +重新连到 `sync_demo` 以后，再执行： + +```sql +GRANT USAGE, CREATE ON SCHEMA public TO st_user_sink; +``` + +这篇教程使用 `generate_sink_sql = true`，所以第一次运行时 SeaTunnel 会自动创建 `public.st_orders`、`public.st_customers` 这类目标表。 + +## 最小配置 + +下面这个示例通过一个 `table-pattern` 同时读取多张 MySQL 表，并把它们分别写入 PostgreSQL 中名为 `st_<上游表名>` 的目标表。 + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + MySQL-CDC { + plugin_output = "mysql_multi" + startup.mode = "initial" + server-id = 5652 + username = "st_user_source" + password = "mysqlpw" + database-pattern = "inventory" + table-pattern = "inventory\\.(orders|customers|products)" + url = "jdbc:mysql://mysql:3306/inventory" + } +} + +sink { + Jdbc { + plugin_input = "mysql_multi" + driver = "org.postgresql.Driver" + url = "jdbc:postgresql://postgresql:5432/sync_demo" + username = "st_user_sink" + password = "pgpw" + generate_sink_sql = true + database = "sync_demo" + table = "public.st_${table_name}" + primary_keys = ["${primary_key}"] + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + data_save_mode = "APPEND_DATA" + } +} +``` + +## 运行任务 + +把配置保存为 `config/multi-table-cdc.conf`，然后用本地模式启动 SeaTunnel： + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/multi-table-cdc.conf -m local +``` + +这是一条流式 CDC 作业，所以执行下面的验证步骤时，任务需要保持运行中。 + +## 验证结果 + +1. 启动作业，等首轮快照完成。 +2. 先确认 PostgreSQL 里已经自动创建出多张目标表： + +```sql +SELECT table_name +FROM information_schema.tables +WHERE table_schema = 'public' AND table_name LIKE 'st_%' +ORDER BY table_name; +``` + +3. 分别在 MySQL 源表执行几条变更： + +```sql +INSERT INTO inventory.orders (id, order_status, updated_at) +VALUES (2002, 'PAID', NOW()); + +UPDATE inventory.customers +SET city = 'Hangzhou', updated_at = NOW() +WHERE id = 3001; + +INSERT INTO inventory.products (id, product_name, unit_price, updated_at) +VALUES (4002, 'Mouse', 59.00, NOW()); +``` + +4. 检查每张目标表是否只接收到自己的数据： + +```sql +SELECT id, order_status FROM public.st_orders ORDER BY id; +SELECT id, customer_name, city FROM public.st_customers ORDER BY id; +SELECT id, product_name, unit_price FROM public.st_products ORDER BY id; +``` + +如果每张上游表都能进入对应的下游表，并且后续变更还能持续同步，这条多表 CDC 链路就是通的。 + +## 常见坑 + +- `table-pattern` 的正则没有转义好。在 HOCON 里，字面量 `.` 通常要写成 `\\.`。 +- MySQL binlog 或 CDC 用户权限不完整，导致任务只能读快照，后续增量读不出来。 +- 没有配置基于占位符的 sink 路由，结果多张源表被写进了同一张目标表。 +- PostgreSQL sink 用户虽然能连库，但没有 `public` schema 的 `CREATE` 权限，自动建表会失败。 +- 上游表没有主键，但下游配置却按 upsert 语义来用。 +- 不同数据库对 schema 和 table 占位符的命名规则不同，直接照搬会失败。 + +## 相关文档 + +- [MySQL CDC Source](../../connectors/source/MySQL-CDC.md) +- [JDBC Sink](../../connectors/sink/Jdbc.md) +- [多表同步架构](../../architecture/features/multi-table.md) diff --git a/docs/zh/getting-started/recipes/mysql-cdc-to-doris.md b/docs/zh/getting-started/recipes/mysql-cdc-to-doris.md new file mode 100644 index 000000000000..c3f0f32d6bda --- /dev/null +++ b/docs/zh/getting-started/recipes/mysql-cdc-to-doris.md @@ -0,0 +1,177 @@ +--- +sidebar_position: 1 +title: MySQL CDC 到 Doris +--- + +# MySQL CDC 到 Doris + +当你想把 MySQL 的行级变更持续同步到 Doris，并让 Doris 始终保持最新状态时，可以直接使用这条链路。 + +## 前置条件 + +1. 先完成 [跑第一个任务](../locally/run-your-first-job.md)，确认本地基础链路正常。 + +2. 安装这条链路需要的插件。先看 [部署 > 下载连接器插件](../locally/deployment.md#下载连接器插件)，然后把 `config/plugin_config` 改成下面这样： + +```plugin_config +--seatunnel-connectors-- +connector-cdc-mysql +connector-doris +--end-- +``` + +```bash +cd "${SEATUNNEL_HOME}" +sh bin/install-plugin.sh +ls connectors | rg 'connector-(cdc-mysql|doris)' +``` + +3. 如果你用的是 SeaTunnel Zeta，再把 MySQL JDBC 驱动放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经落盘： + +```bash +ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +``` + +如果你用的是 Flink 或 Spark，就把同一个驱动 jar 放到对应引擎实际加载的插件目录里。 + +4. 先准备 MySQL 源表。这条教程依赖稳定主键，这样下游才能正确回放更新和删除事件。 + +```sql +CREATE DATABASE IF NOT EXISTS inventory; + +CREATE TABLE IF NOT EXISTS inventory.orders ( + id BIGINT PRIMARY KEY, + order_status VARCHAR(32), + amount DECIMAL(10, 2), + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +INSERT INTO inventory.orders (id, order_status, amount, updated_at) VALUES + (1001, 'CREATED', 19.99, NOW()), + (1002, 'CREATED', 29.99, NOW()); +``` + +5. 按照 [MySQL CDC Source](../../connectors/source/MySQL-CDC.md) 的要求创建 CDC 用户并授权： + +```sql +CREATE USER IF NOT EXISTS 'st_user_source'@'%' IDENTIFIED BY 'mysqlpw'; +GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT +ON *.* TO 'st_user_source'@'%'; +FLUSH PRIVILEGES; +``` + +6. 检查 MySQL binlog 是否已经满足 CDC 要求： + +```sql +SHOW VARIABLES WHERE variable_name IN ('log_bin', 'binlog_format', 'binlog_row_image'); +``` + +期望值是 `log_bin = ON`、`binlog_format = ROW`、`binlog_row_image = FULL`。 +如果还没有配置好，就修改 `my.cnf` 并重启 MySQL： + +```ini +[mysqld] +server-id = 223344 +log_bin = mysql-bin +binlog_format = ROW +binlog_row_image = FULL +``` + +7. 先准备 Doris 目标库。这篇教程保留 `schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST"`，所以第一次启动时会用 MySQL 主键信息自动创建 `sync_demo.orders`。 + +```sql +CREATE DATABASE IF NOT EXISTS sync_demo; +``` + +## 最小配置 + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + MySQL-CDC { + plugin_output = "orders_cdc" + parallelism = 1 + startup.mode = "initial" + server-id = 5652 + username = "st_user_source" + password = "mysqlpw" + table-names = ["inventory.orders"] + url = "jdbc:mysql://mysql:3306/inventory" + } +} + +sink { + Doris { + plugin_input = "orders_cdc" + fenodes = "doris-fe:8030" + username = "root" + password = "" + database = "sync_demo" + table = "orders" + sink.label-prefix = "orders-cdc" + sink.enable-delete = true + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + doris.config = { + format = "csv" + column_separator = "," + } + } +} +``` + +## 运行任务 + +把配置保存为 `config/mysql-cdc-to-doris.conf`，然后用本地模式启动 SeaTunnel： + +```bash +cd "${SEATUNNEL_HOME}" +./bin/seatunnel.sh --config ./config/mysql-cdc-to-doris.conf -m local +``` + +这是一条流式 CDC 作业，所以执行下面的验证 SQL 时，任务需要保持运行中。 + +## 验证结果 + +1. 启动作业，等首轮快照完成。 +2. 在 MySQL 中执行下面几条变更： + +```sql +INSERT INTO inventory.orders (id, order_status, amount, updated_at) +VALUES (1003, 'CREATED', 39.99, NOW()); + +UPDATE inventory.orders +SET order_status = 'PAID', updated_at = NOW() +WHERE id = 1001; + +DELETE FROM inventory.orders +WHERE id = 1002; +``` + +3. 在 Doris 中查询最新结果： + +```sql +SELECT COUNT(*) FROM sync_demo.orders; +SELECT id, order_status, amount FROM sync_demo.orders ORDER BY id; +``` + +此时你应该能看到 `1001` 变成 `PAID`，`1003` 成功插入，`1002` 已经消失。如果 MySQL 的新增、更新、删除都能体现在 Doris 里，这条链路就是通的。 + +## 常见坑 + +- MySQL 没开 binlog，或者 binlog 不是 `ROW` 格式。 +- CDC 用户缺少复制相关权限。 +- `server-id` 和其他 MySQL 副本或 CDC 作业冲突。 +- 多个运行中的任务复用了同一个 `sink.label-prefix`，导致 Doris stream load 冲突。 +- 开启了删除同步，但 Doris 目标表模型不支持预期的删除行为。 +- 源表没有稳定主键，导致 Doris 自动建表和下游 upsert 结果都不稳定。 + +## 相关文档 + +- [MySQL CDC Source](../../connectors/source/MySQL-CDC.md) +- [Doris Sink](../../connectors/sink/Doris.md) +- [SeaTunnel 引擎快速开始](../locally/quick-start-seatunnel-engine.md) diff --git a/docs/zh/introduction/about.md b/docs/zh/introduction/about.md index 5ed6c05f31b4..89c2d6073cb9 100644 --- a/docs/zh/introduction/about.md +++ b/docs/zh/introduction/about.md @@ -11,6 +11,30 @@ SeaTunnel是一个多模态、超高性能、分布式的海量数据集成工 如果你是第一次接触 SeaTunnel，建议按下面路径进入文档： + + - [快速入门总览](../getting-started/overview.md)，先建立整体路径 - [SeaTunnel 引擎快速开始](../getting-started/locally/quick-start-seatunnel-engine.md)，先跑通第一个本地任务 - [作业配置指南](../getting-started/job-configuration-guide.md)，开始编写真实作业 @@ -87,4 +111,4 @@ SeaTunnel 丰富了> ownedSlotProfilesIMap; - private IMap> metricsImap; - private IMap pendingPipelineCleanupIMap; private IMap pendingJobCleanupIMap; @@ -229,25 +223,13 @@ public class CoordinatorService { private final AtomicBoolean coordinatorServiceCleared = new AtomicBoolean(false); - private final AtomicLong runningJobMetricsPartitionKeyCount = new AtomicLong(); - - private final AtomicLong runningJobMetricsTaskContextCount = new AtomicLong(); - - private final Object runningJobMetricsStatsLock = new Object(); - - private final Set runningJobMetricsDirtyKeys = ConcurrentHashMap.newKeySet(); - - private final Map runningJobMetricsStatsByJobId = new HashMap<>(); - - private final AtomicBoolean runningJobMetricsInitializing = new AtomicBoolean(false); - - private volatile UUID runningJobMetricsListenerId; - public CoordinatorService( @NonNull NodeEngineImpl nodeEngine, @NonNull SeaTunnelServer seaTunnelServer, + @NonNull SeaTunnelEngineContext engineContext, EngineConfig engineConfig) { this.nodeEngine = nodeEngine; + this.engineContext = engineContext; this.engineConfig = engineConfig; this.logger = nodeEngine.getLogger(getClass()); this.executorService = createCoordinatorExecutor(); @@ -538,8 +520,6 @@ private void initCoordinatorService() { nodeEngine.getHazelcastInstance().getMap(Constant.IMAP_STATE_TIMESTAMPS); ownedSlotProfilesIMap = nodeEngine.getHazelcastInstance().getMap(Constant.IMAP_OWNED_SLOT_PROFILES); - metricsImap = nodeEngine.getHazelcastInstance().getMap(Constant.IMAP_RUNNING_JOB_METRICS); - initRunningJobMetricsStoreStats(); pendingPipelineCleanupIMap = nodeEngine.getHazelcastInstance().getMap(Constant.IMAP_PENDING_PIPELINE_CLEANUP); pendingJobCleanupIMap = @@ -1124,7 +1104,6 @@ public synchronized void clearCoordinatorService() { if (!coordinatorServiceCleared.compareAndSet(false, true)) { return; } - removeRunningJobMetricsListener(); // interrupt all JobMaster runningJobMasterMap.values().forEach(JobMaster::interrupt); if (isWaitStrategy) { @@ -1834,7 +1813,6 @@ public void shutdown() { if (pipelineCleanupScheduler != null) { pipelineCleanupScheduler.shutdown(); } - removeRunningJobMetricsListener(); clearCoordinatorService(); awaitSchedulerTermination("master active listener", masterActiveListener); awaitSchedulerTermination("pipeline cleanup scheduler", pipelineCleanupScheduler); @@ -2097,11 +2075,11 @@ public int getPendingJobCount() { } public long getRunningJobMetricsPartitionKeyCount() { - return runningJobMetricsPartitionKeyCount.get(); + return getMetricsSnapshotStateStore().activePartitionKeyCount(); } public long getRunningJobMetricsTaskContextCount() { - return runningJobMetricsTaskContextCount.get(); + return getMetricsSnapshotStateStore().size(); } public EngineConfig getEngineConfig() { @@ -2109,8 +2087,8 @@ public EngineConfig getEngineConfig() { } @VisibleForTesting - protected IMap> getMetricsImap() { - return metricsImap; + protected MetricsSnapshotStateStore getMetricsSnapshotStateStore() { + return engineContext.getStateStores().metricsSnapshotStore(); } @VisibleForTesting @@ -2128,135 +2106,6 @@ void runPendingJobCleanupOnce() { } } - private void initRunningJobMetricsStoreStats() { - removeRunningJobMetricsListener(); - runningJobMetricsListenerId = - metricsImap.addEntryListener(new RunningJobMetricsEntryListener(), true); - runningJobMetricsInitializing.set(true); - runningJobMetricsDirtyKeys.clear(); - - Map snapshotStats = new HashMap<>(); - metricsImap.forEach( - (partitionKey, metrics) -> - snapshotStats.put(partitionKey, toRunningJobMetricsStats(metrics))); - - Set dirtyKeys = new HashSet<>(runningJobMetricsDirtyKeys); - for (Long partitionKey : dirtyKeys) { - snapshotStats.put( - partitionKey, toRunningJobMetricsStats(metricsImap.get(partitionKey))); - } - - synchronized (runningJobMetricsStatsLock) { - runningJobMetricsStatsByJobId.clear(); - runningJobMetricsPartitionKeyCount.set(0L); - runningJobMetricsTaskContextCount.set(0L); - snapshotStats.forEach(this::replaceRunningJobMetricsStatsLocked); - runningJobMetricsInitializing.set(false); - - Set postSnapshotDirtyKeys = new HashSet<>(runningJobMetricsDirtyKeys); - runningJobMetricsDirtyKeys.clear(); - for (Long partitionKey : postSnapshotDirtyKeys) { - replaceRunningJobMetricsStatsLocked( - partitionKey, toRunningJobMetricsStats(metricsImap.get(partitionKey))); - } - } - } - - private RunningJobMetricsStats toRunningJobMetricsStats( - Map metrics) { - if (metrics == null || metrics.isEmpty()) { - return RunningJobMetricsStats.EMPTY; - } - return new RunningJobMetricsStats(1L, metrics.size()); - } - - private void replaceRunningJobMetricsStatsLocked( - Long partitionKey, RunningJobMetricsStats stats) { - RunningJobMetricsStats currentStats = runningJobMetricsStatsByJobId.get(partitionKey); - if (currentStats != null) { - runningJobMetricsPartitionKeyCount.addAndGet(-currentStats.partitionKeyCount); - runningJobMetricsTaskContextCount.addAndGet(-currentStats.taskContextCount); - } - - if (stats.isEmpty()) { - runningJobMetricsStatsByJobId.remove(partitionKey); - return; - } - - runningJobMetricsStatsByJobId.put(partitionKey, stats); - runningJobMetricsPartitionKeyCount.addAndGet(stats.partitionKeyCount); - runningJobMetricsTaskContextCount.addAndGet(stats.taskContextCount); - } - - private void removeRunningJobMetricsListener() { - if (metricsImap != null && runningJobMetricsListenerId != null) { - metricsImap.removeEntryListener(runningJobMetricsListenerId); - runningJobMetricsListenerId = null; - } - runningJobMetricsInitializing.set(false); - runningJobMetricsDirtyKeys.clear(); - synchronized (runningJobMetricsStatsLock) { - runningJobMetricsStatsByJobId.clear(); - } - runningJobMetricsPartitionKeyCount.set(0L); - runningJobMetricsTaskContextCount.set(0L); - } - - private final class RunningJobMetricsEntryListener - implements EntryAddedListener>, - EntryUpdatedListener>, - EntryRemovedListener> { - - @Override - public void entryAdded( - EntryEvent> event) { - replaceRunningJobMetricsStats(event.getKey(), event.getValue()); - } - - @Override - public void entryUpdated( - EntryEvent> event) { - replaceRunningJobMetricsStats(event.getKey(), event.getValue()); - } - - @Override - public void entryRemoved( - EntryEvent> event) { - replaceRunningJobMetricsStats(event.getKey(), null); - } - } - - private void replaceRunningJobMetricsStats( - Long partitionKey, Map metrics) { - if (runningJobMetricsInitializing.get()) { - runningJobMetricsDirtyKeys.add(partitionKey); - return; - } - synchronized (runningJobMetricsStatsLock) { - if (runningJobMetricsInitializing.get()) { - runningJobMetricsDirtyKeys.add(partitionKey); - return; - } - replaceRunningJobMetricsStatsLocked(partitionKey, toRunningJobMetricsStats(metrics)); - } - } - - private static final class RunningJobMetricsStats { - private static final RunningJobMetricsStats EMPTY = new RunningJobMetricsStats(0L, 0L); - - private final long partitionKeyCount; - private final long taskContextCount; - - private RunningJobMetricsStats(long partitionKeyCount, long taskContextCount) { - this.partitionKeyCount = partitionKeyCount; - this.taskContextCount = taskContextCount; - } - - private boolean isEmpty() { - return partitionKeyCount == 0L && taskContextCount == 0L; - } - } - @VisibleForTesting public PeekBlockingQueue getPendingJobQueue() { return pendingJobQueue; diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java index c7a362c53f3e..37f39b03f816 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java @@ -27,6 +27,10 @@ import org.apache.seatunnel.engine.core.classloader.ClassLoaderService; import org.apache.seatunnel.engine.core.classloader.DefaultClassLoaderService; import org.apache.seatunnel.engine.server.checkpoint.monitor.CheckpointMonitorService; +import org.apache.seatunnel.engine.server.common.SeaTunnelEngineContext; +import org.apache.seatunnel.engine.server.common.statestore.EngineStateStores; +import org.apache.seatunnel.engine.server.common.statestore.hazelcast.HazelcastEngineStateStores; +import org.apache.seatunnel.engine.server.common.statestore.metrics.MetricsSnapshotStateStore; import org.apache.seatunnel.engine.server.dag.physical.PipelineLocation; import org.apache.seatunnel.engine.server.execution.ExecutionState; import org.apache.seatunnel.engine.server.execution.TaskGroupLocation; @@ -58,14 +62,11 @@ import lombok.extern.slf4j.Slf4j; import java.sql.DriverManager; -import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; @Slf4j public class SeaTunnelServer @@ -87,6 +88,7 @@ public class SeaTunnelServer public static final String SERVICE_NAME = "st:impl:seaTunnelServer"; + @Getter private SeaTunnelEngineContext engineContext; private NodeEngineImpl nodeEngine; private final LiveOperationRegistry liveOperationRegistry; @@ -146,6 +148,12 @@ public void init(NodeEngine engine, Properties hzProperties) { // TODO Determine whether to execute there method on the master node according to the deploy // type + EngineStateStores stateStores = + new HazelcastEngineStateStores( + nodeEngine, + seaTunnelConfig.getEngineConfig().getJobMetricsPartitionCount()); + this.engineContext = SeaTunnelEngineContext.builder(stateStores).build(); + classLoaderService = new DefaultClassLoaderService( seaTunnelConfig.getEngineConfig().isClassloaderCacheMode(), nodeEngine); @@ -191,10 +199,11 @@ public void init(NodeEngine engine, Properties hzProperties) { private void startMaster() { checkpointService = new CheckpointService(seaTunnelConfig.getEngineConfig().getCheckpointConfig()); - checkpointMonitorService = new CheckpointMonitorService(nodeEngine, 32); + checkpointMonitorService = new CheckpointMonitorService(engineContext, 32); monitorService = Executors.newSingleThreadScheduledExecutor(); coordinatorService = - new CoordinatorService(nodeEngine, this, seaTunnelConfig.getEngineConfig()); + new CoordinatorService( + nodeEngine, this, engineContext, seaTunnelConfig.getEngineConfig()); monitorService.scheduleAtFixedRate( this::printExecutionInfo, 0, @@ -204,7 +213,8 @@ private void startMaster() { private void startWorker() { taskExecutionService = - new TaskExecutionService(classLoaderService, nodeEngine, eventService); + new TaskExecutionService( + classLoaderService, nodeEngine, engineContext, eventService); nodeEngine.getMetricsRegistry().registerDynamicMetricsProvider(taskExecutionService); taskExecutionService.start(); getSlotService(); @@ -243,6 +253,7 @@ public void shutdown(boolean terminate) { } MetadataProviderManager.closeProviders(); + engineContext.close(); } @Override @@ -376,78 +387,15 @@ private void printExecutionInfo() { } public void updateMetrics(Map localMap) { - if (localMap == null || localMap.isEmpty()) { - return; - } - int partitionCount = seaTunnelConfig.getEngineConfig().getJobMetricsPartitionCount(); - - IMap> metricsImap = - getNodeEngine().getHazelcastInstance().getMap(Constant.IMAP_RUNNING_JOB_METRICS); - - Map> partitioned = new HashMap<>(); - localMap.forEach( - (key, value) -> { - long partition = getMetricsImapPartition(key, partitionCount); - partitioned.computeIfAbsent(partition, k -> new HashMap<>()).put(key, value); - }); - - partitioned - .entrySet() - .parallelStream() - .forEach( - entry -> { - metricsImap.compute( - entry.getKey(), - (k, oldVal) -> { - if (oldVal == null) oldVal = new HashMap<>(); - oldVal.putAll(entry.getValue()); - return oldVal; - }); - }); + MetricsSnapshotStateStore metricsSnapshotStateStore = + engineContext.getStateStores().metricsSnapshotStore(); + metricsSnapshotStateStore.merge(localMap); } public void removeMetrics(PipelineLocation pipelineLocation) { - IMap> metricsImap = - getNodeEngine().getHazelcastInstance().getMap(Constant.IMAP_RUNNING_JOB_METRICS); - - Map> partitionedTasks = new HashMap<>(); - for (Map.Entry> entry : - metricsImap.entrySet()) { - long partition = entry.getKey(); - List tasksToRemove = - entry.getValue().keySet().stream() - .filter( - t -> - t.getTaskGroupLocation() - .getPipelineLocation() - .equals(pipelineLocation)) - .collect(Collectors.toList()); - if (!tasksToRemove.isEmpty()) { - partitionedTasks.put(partition, tasksToRemove); - } - } - - partitionedTasks - .entrySet() - .parallelStream() - .forEach( - entry -> { - long partition = entry.getKey(); - List tasks = entry.getValue(); - metricsImap.compute( - partition, - (k, oldVal) -> { - if (oldVal != null) { - tasks.forEach(oldVal::remove); - if (oldVal.isEmpty()) return null; - } - return oldVal; - }); - }); - } - - public static long getMetricsImapPartition(TaskLocation key, int partitionCount) { - return (key.hashCode() & 0x7FFFFFFF) % partitionCount; + MetricsSnapshotStateStore metricsSnapshotStateStore = + engineContext.getStateStores().metricsSnapshotStore(); + metricsSnapshotStateStore.removePipeline(pipelineLocation); } public boolean isCoordinatorActive() { diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java index 5e011d53914a..b3bc49dd6dce 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java @@ -33,6 +33,7 @@ import org.apache.seatunnel.engine.common.utils.concurrent.CompletableFuture; import org.apache.seatunnel.engine.core.classloader.ClassLoaderService; import org.apache.seatunnel.engine.core.job.ConnectorJarIdentifier; +import org.apache.seatunnel.engine.server.common.SeaTunnelEngineContext; import org.apache.seatunnel.engine.server.exception.TaskGroupContextNotFoundException; import org.apache.seatunnel.engine.server.execution.ExecutionState; import org.apache.seatunnel.engine.server.execution.ProgressState; @@ -147,6 +148,9 @@ public class TaskExecutionService implements DynamicMetricsProvider { /** The NodeEngine implementation for this Hazelcast node. */ private final NodeEngineImpl nodeEngine; + /** Shared engine context exposing state-store abstractions. */ + private final SeaTunnelEngineContext engineContext; + /** Service for managing class loaders for connector jars. */ private final ClassLoaderService classLoaderService; @@ -225,10 +229,12 @@ public class TaskExecutionService implements DynamicMetricsProvider { public TaskExecutionService( ClassLoaderService classLoaderService, NodeEngineImpl nodeEngine, + SeaTunnelEngineContext engineContext, EventService eventService) { seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); this.hzInstanceName = nodeEngine.getHazelcastInstance().getName(); this.nodeEngine = nodeEngine; + this.engineContext = engineContext; this.classLoaderService = classLoaderService; this.logger = nodeEngine.getLoggingService().getLogger(TaskExecutionService.class); @@ -538,7 +544,8 @@ public PassiveCompletableFuture deployLocalTask( .peek( task -> { TaskExecutionContext taskExecutionContext = - new TaskExecutionContext(task, nodeEngine, this); + new TaskExecutionContext( + task, nodeEngine, engineContext, this); task.setTaskExecutionContext(taskExecutionContext); taskExecutionContextMap.put( task.getTaskID(), taskExecutionContext); diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/monitor/CheckpointMonitorService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/monitor/CheckpointMonitorService.java index 44ec00b85755..eb5b92553b82 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/monitor/CheckpointMonitorService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/monitor/CheckpointMonitorService.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.shade.com.google.common.base.Strings; -import org.apache.seatunnel.engine.common.Constant; import org.apache.seatunnel.engine.core.checkpoint.CheckpointHistoryEntry; import org.apache.seatunnel.engine.core.checkpoint.CheckpointInfo; import org.apache.seatunnel.engine.core.checkpoint.CheckpointOverview; @@ -31,61 +30,41 @@ import org.apache.seatunnel.engine.server.checkpoint.CompletedCheckpoint; import org.apache.seatunnel.engine.server.checkpoint.SubtaskStatistics; import org.apache.seatunnel.engine.server.checkpoint.TaskStatistics; +import org.apache.seatunnel.engine.server.common.SeaTunnelEngineContext; +import org.apache.seatunnel.engine.server.common.statestore.checkpoint.CheckpointOverviewStateStore; -import com.hazelcast.core.EntryEvent; -import com.hazelcast.map.IMap; -import com.hazelcast.map.listener.EntryAddedListener; -import com.hazelcast.map.listener.EntryExpiredListener; -import com.hazelcast.map.listener.EntryRemovedListener; -import com.hazelcast.map.listener.EntryUpdatedListener; -import com.hazelcast.spi.impl.NodeEngine; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Collectors; @Slf4j public class CheckpointMonitorService { - private final NodeEngine nodeEngine; - private volatile IMap overviewMap; + private final SeaTunnelEngineContext engineContext; private final int maxHistorySize; - private final AtomicLong overviewJobCount = new AtomicLong(); - private final AtomicLong inProgressCheckpointCount = new AtomicLong(); - private final AtomicLong retainedHistoryCount = new AtomicLong(); - private final Object overviewStatsLock = new Object(); - private final Set overviewDirtyJobIds = ConcurrentHashMap.newKeySet(); - private final Map overviewStatsByJobId = new HashMap<>(); - private final AtomicBoolean overviewStatsInitializing = new AtomicBoolean(false); - private volatile UUID overviewListenerId; - public CheckpointMonitorService(NodeEngine nodeEngine, int maxHistorySize) { - this.nodeEngine = nodeEngine; + private volatile CheckpointOverviewStateStore overviewMap; + + public CheckpointMonitorService(SeaTunnelEngineContext engineContext, int maxHistorySize) { + this.engineContext = engineContext; this.maxHistorySize = maxHistorySize; } - private IMap getOverviewMap() { + private CheckpointOverviewStateStore getOverviewMap() { if (overviewMap == null) { synchronized (this) { if (overviewMap == null) { overviewMap = - nodeEngine - .getHazelcastInstance() - .getMap(Constant.IMAP_CHECKPOINT_MONITOR); - initOverviewStats(); + engineContext + .getStateStores() + .auxiliary() + .checkpointOverviewStateStore(); } } } @@ -259,186 +238,26 @@ public void clearInProgress(long jobId, int pipelineId) { } public long getOverviewJobCount() { - return overviewJobCount.get(); + return getOverviewMap().getOverviewJobCount(); } public long getInProgressCheckpointCount() { - return inProgressCheckpointCount.get(); + return getOverviewMap().getInProgressCheckpointCount(); } public long getRetainedHistoryCount() { - return retainedHistoryCount.get(); + return getOverviewMap().getRetainedHistoryCount(); } private void updateOverview( long jobId, int pipelineId, Consumer consumer) { - getOverviewMap() - .compute( - jobId, - (id, overview) -> { - CheckpointOverview snapshot = - overview == null ? new CheckpointOverview(jobId) : overview; - PipelineCheckpointOverview pipeline = - snapshot.getOrCreatePipeline(pipelineId); - consumer.accept(pipeline); - snapshot.setUpdatedAt(System.currentTimeMillis()); - return snapshot; - }); + getOverviewMap().updateOverview(jobId, pipelineId, consumer); } private void removeInProgressIfExists(PipelineCheckpointOverview pipeline, long checkpointId) { pipeline.getInProgress().removeIf(cp -> cp.getCheckpointId() == checkpointId); } - private void initOverviewStats() { - removeOverviewListener(); - overviewListenerId = - overviewMap.addEntryListener(new CheckpointOverviewEntryListener(), true); - overviewStatsInitializing.set(true); - overviewDirtyJobIds.clear(); - - Map snapshotStats = new HashMap<>(); - overviewMap.forEach( - (jobId, overview) -> snapshotStats.put(jobId, toOverviewStats(overview))); - - Set dirtyJobIds = new HashSet<>(overviewDirtyJobIds); - for (Long jobId : dirtyJobIds) { - snapshotStats.put(jobId, toOverviewStats(overviewMap.get(jobId))); - } - - synchronized (overviewStatsLock) { - overviewStatsByJobId.clear(); - overviewJobCount.set(0L); - inProgressCheckpointCount.set(0L); - retainedHistoryCount.set(0L); - snapshotStats.forEach(this::replaceOverviewStatsLocked); - overviewStatsInitializing.set(false); - - Set postSnapshotDirtyJobIds = new HashSet<>(overviewDirtyJobIds); - overviewDirtyJobIds.clear(); - for (Long jobId : postSnapshotDirtyJobIds) { - replaceOverviewStatsLocked(jobId, toOverviewStats(overviewMap.get(jobId))); - } - } - } - - private void removeOverviewListener() { - if (overviewMap != null && overviewListenerId != null) { - overviewMap.removeEntryListener(overviewListenerId); - overviewListenerId = null; - } - overviewStatsInitializing.set(false); - overviewDirtyJobIds.clear(); - synchronized (overviewStatsLock) { - overviewStatsByJobId.clear(); - } - overviewJobCount.set(0L); - inProgressCheckpointCount.set(0L); - retainedHistoryCount.set(0L); - } - - private CheckpointOverviewStats toOverviewStats(CheckpointOverview overview) { - if (overview == null) { - return CheckpointOverviewStats.EMPTY; - } - return new CheckpointOverviewStats( - 1L, getInProgressCount(overview), getHistoryCount(overview)); - } - - private void replaceOverviewStatsLocked(Long jobId, CheckpointOverviewStats stats) { - CheckpointOverviewStats currentStats = overviewStatsByJobId.get(jobId); - if (currentStats != null) { - overviewJobCount.addAndGet(-currentStats.jobCount); - inProgressCheckpointCount.addAndGet(-currentStats.inProgressCheckpointCount); - retainedHistoryCount.addAndGet(-currentStats.retainedHistoryCount); - } - - if (stats.isEmpty()) { - overviewStatsByJobId.remove(jobId); - return; - } - - overviewStatsByJobId.put(jobId, stats); - overviewJobCount.addAndGet(stats.jobCount); - inProgressCheckpointCount.addAndGet(stats.inProgressCheckpointCount); - retainedHistoryCount.addAndGet(stats.retainedHistoryCount); - } - - private long getInProgressCount(CheckpointOverview overview) { - return overview.getPipelines().values().stream() - .filter(Objects::nonNull) - .mapToLong(pipelineOverview -> pipelineOverview.getInProgress().size()) - .sum(); - } - - private long getHistoryCount(CheckpointOverview overview) { - return overview.getPipelines().values().stream() - .filter(Objects::nonNull) - .mapToLong(pipelineOverview -> pipelineOverview.getHistory().size()) - .sum(); - } - - private final class CheckpointOverviewEntryListener - implements EntryAddedListener, - EntryUpdatedListener, - EntryRemovedListener, - EntryExpiredListener { - - @Override - public void entryAdded(EntryEvent event) { - replaceOverviewStats(event.getKey(), event.getValue()); - } - - @Override - public void entryUpdated(EntryEvent event) { - replaceOverviewStats(event.getKey(), event.getValue()); - } - - @Override - public void entryRemoved(EntryEvent event) { - replaceOverviewStats(event.getKey(), null); - } - - @Override - public void entryExpired(EntryEvent event) { - replaceOverviewStats(event.getKey(), null); - } - } - - private void replaceOverviewStats(Long jobId, CheckpointOverview overview) { - if (overviewStatsInitializing.get()) { - overviewDirtyJobIds.add(jobId); - return; - } - synchronized (overviewStatsLock) { - if (overviewStatsInitializing.get()) { - overviewDirtyJobIds.add(jobId); - return; - } - replaceOverviewStatsLocked(jobId, toOverviewStats(overview)); - } - } - - private static final class CheckpointOverviewStats { - private static final CheckpointOverviewStats EMPTY = - new CheckpointOverviewStats(0L, 0L, 0L); - - private final long jobCount; - private final long inProgressCheckpointCount; - private final long retainedHistoryCount; - - private CheckpointOverviewStats( - long jobCount, long inProgressCheckpointCount, long retainedHistoryCount) { - this.jobCount = jobCount; - this.inProgressCheckpointCount = inProgressCheckpointCount; - this.retainedHistoryCount = retainedHistoryCount; - } - - private boolean isEmpty() { - return jobCount == 0L && inProgressCheckpointCount == 0L && retainedHistoryCount == 0L; - } - } - public static long calculateStateSize(CompletedCheckpoint checkpoint) { return checkpoint.getTaskStatistics().values().stream() .map(TaskStatistics::getSubtaskStats) diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/SeaTunnelEngineContext.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/SeaTunnelEngineContext.java new file mode 100644 index 000000000000..18128fcab4c8 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/SeaTunnelEngineContext.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common; + +import org.apache.seatunnel.engine.server.common.statestore.EngineStateStores; + +import java.util.Objects; + +/** + * Shared engine context propagated inside the engine. + * + *

This context is intended to reduce direct propagation of infrastructure-specific runtime + * objects into engine services. At the current stage, it only exposes state store bundles and + * serves as a small entry point for state-related abstractions. + */ +public final class SeaTunnelEngineContext implements AutoCloseable { + + private final EngineStateStores stateStores; + + private SeaTunnelEngineContext(Builder builder) { + this.stateStores = builder.stateStores; + } + + public static Builder builder(EngineStateStores stateStores) { + return new Builder(stateStores); + } + + /** + * Returns the grouped engine state stores. + * + * @return engine state stores + */ + public EngineStateStores getStateStores() { + return stateStores; + } + + @Override + public void close() { + stateStores.close(); + } + + /** Builder for {@link SeaTunnelEngineContext}. */ + public static final class Builder { + private final EngineStateStores stateStores; + + private Builder(EngineStateStores stateStores) { + this.stateStores = Objects.requireNonNull(stateStores, "stateStores"); + } + + public SeaTunnelEngineContext build() { + return new SeaTunnelEngineContext(this); + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/AuthoritativeStateStores.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/AuthoritativeStateStores.java new file mode 100644 index 000000000000..1e14e769a949 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/AuthoritativeStateStores.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore; + +/** + * Bundle of authoritative control state that a new leader must trust during leader handoff. + * + *

This layer groups states that are likely to live on top of a consensus layer. The actual + * storage backend may still be a local store, but the responsibility for deciding what is + * authoritative belongs more strongly to this group. + */ +public interface AuthoritativeStateStores extends AutoCloseable { + + /** + * Releases resources owned by authoritative stores. + * + *

Implementations that only wrap non-closeable stores may keep the default no-op behavior. + */ + @Override + default void close() { + // no-op + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/AuxiliaryStateStores.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/AuxiliaryStateStores.java new file mode 100644 index 000000000000..af7c8bdae589 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/AuxiliaryStateStores.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore; + +import org.apache.seatunnel.engine.server.common.statestore.checkpoint.CheckpointOverviewStateStore; +import org.apache.seatunnel.engine.server.common.statestore.metrics.MetricsSnapshotStateStore; + +/** + * Bundle of state that is closer to observability, recent history, or cleanup than to failover + * correctness. + * + *

This group collects states that can often tolerate more staleness, or for which limited loss + * is less likely to break system correctness immediately. + */ +public interface AuxiliaryStateStores extends AutoCloseable { + /** + * Returns the store for runtime task metrics snapshots. + * + * @return metrics snapshot store + */ + MetricsSnapshotStateStore metricsSnapshotStore(); + + /** + * Returns the store for checkpoint overviews. + * + * @return checkpoint overview state store + */ + CheckpointOverviewStateStore checkpointOverviewStateStore(); + + /** + * Releases resources owned by auxiliary stores. + * + *

Implementations that only wrap non-closeable stores may keep the default no-op behavior. + */ + @Override + default void close() { + // no-op + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/DefaultAuthoritativeStateStores.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/DefaultAuthoritativeStateStores.java new file mode 100644 index 000000000000..f658f31d39e6 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/DefaultAuthoritativeStateStores.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore; + +/** Default immutable implementation of {@link AuthoritativeStateStores}. */ +public class DefaultAuthoritativeStateStores implements AuthoritativeStateStores {} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/DefaultAuxiliaryStateStores.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/DefaultAuxiliaryStateStores.java new file mode 100644 index 000000000000..87017e8f3e89 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/DefaultAuxiliaryStateStores.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore; + +import org.apache.seatunnel.engine.server.common.statestore.checkpoint.CheckpointOverviewStateStore; +import org.apache.seatunnel.engine.server.common.statestore.metrics.MetricsSnapshotStateStore; + +import java.util.Objects; + +/** Default immutable implementation of {@link AuxiliaryStateStores}. */ +public class DefaultAuxiliaryStateStores implements AuxiliaryStateStores { + private final MetricsSnapshotStateStore metricsSnapshotStore; + private final CheckpointOverviewStateStore checkpointOverviewStateStore; + + public DefaultAuxiliaryStateStores( + MetricsSnapshotStateStore metricsSnapshotStore, + CheckpointOverviewStateStore checkpointOverviewStateStore) { + this.metricsSnapshotStore = + Objects.requireNonNull(metricsSnapshotStore, "metricsSnapshotStore"); + this.checkpointOverviewStateStore = + Objects.requireNonNull( + checkpointOverviewStateStore, "checkpointOverviewStateStore"); + } + + @Override + public MetricsSnapshotStateStore metricsSnapshotStore() { + return metricsSnapshotStore; + } + + @Override + public CheckpointOverviewStateStore checkpointOverviewStateStore() { + return checkpointOverviewStateStore; + } + + @Override + public void close() { + closeIfPossible(checkpointOverviewStateStore); + closeIfPossible(metricsSnapshotStore); + } + + private void closeIfPossible(Object store) { + if (!(store instanceof AutoCloseable)) { + return; + } + try { + ((AutoCloseable) store).close(); + } catch (Exception e) { + throw new IllegalStateException("Failed to close auxiliary state store", e); + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/EngineStateStoreNames.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/EngineStateStoreNames.java new file mode 100644 index 000000000000..3ed6dfd244e3 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/EngineStateStoreNames.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore; + +import lombok.Getter; + +/** + * Bundle of map names required to build a Hazelcast-based {@link EngineStateStores} implementation. + */ +@Getter +public class EngineStateStoreNames { + private EngineStateStoreNames() {} + + public static final String RUNNING_JOB_METRICS = "engine_runningJobMetrics"; + public static final String CHECKPOINT_MONITOR = "engine_checkpoint_monitor"; +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/EngineStateStores.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/EngineStateStores.java new file mode 100644 index 000000000000..e86f2061264d --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/EngineStateStores.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore; + +import org.apache.seatunnel.engine.server.common.statestore.checkpoint.CheckpointOverviewStateStore; +import org.apache.seatunnel.engine.server.common.statestore.metrics.MetricsSnapshotStateStore; + +/** + * Top-level bundle of state stores used directly by the engine. + * + *

This is the high-level port that lets engine code depend on state semantics rather than + * concrete implementations such as {@code HazelcastRuntimeStateStore}. + * + *

Implementation construction is expected to stay in Hazelcast/RocksDB-specific providers, while + * engine code depends only on this interface. + */ +public interface EngineStateStores extends AutoCloseable { + /** + * Returns the bundle of control state that must be treated as authoritative during leader + * handoff. + * + * @return authoritative state stores + */ + AuthoritativeStateStores authoritative(); + + /** + * Returns the bundle of auxiliary state used mainly for observability, recent history, or + * cleanup. + * + * @return auxiliary state stores + */ + AuxiliaryStateStores auxiliary(); + + /** + * Returns the store for runtime task metrics snapshots. + * + * @return metrics snapshot store + */ + default MetricsSnapshotStateStore metricsSnapshotStore() { + return auxiliary().metricsSnapshotStore(); + } + + /** + * Returns the store for checkpoint overviews. + * + * @return checkpoint overview state store + */ + default CheckpointOverviewStateStore checkpointOverviewStateStore() { + return auxiliary().checkpointOverviewStateStore(); + } + + /** + * Releases resources owned by the stores. + * + *

Hazelcast implementations are usually no-op, while RocksDB implementations use this to + * close the underlying database resources. + */ + @Override + void close(); +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/IterableStateStore.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/IterableStateStore.java new file mode 100644 index 000000000000..0ead0ef613b3 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/IterableStateStore.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +/** + * Backend storage capability for full iteration. + * + *

Operations that read all values can have very different costs depending on the backend, so + * this capability is separated from the base {@link StateStore}. + * + *

This is better treated as a backend SPI used internally by higher-level stores such as history + * or metrics stores, rather than as an engine-facing contract. + * + * @param key type + * @param value type + */ +public interface IterableStateStore extends StateStore { + + /** @return all stored entries */ + Set> entrySet(); + + /** @return all stored values */ + Collection values(); + + /** + * Returns whether the store is empty. + * + * @return {@code true} if the store is empty + */ + boolean isEmpty(); + + /** + * Returns the total number of entries. + * + * @return entry count + */ + int size(); +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/StateStore.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/StateStore.java new file mode 100644 index 000000000000..60c09deabb59 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/StateStore.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore; + +/** + * Low-level backend storage SPI. + * + *

This is not meant to be the public state contract used directly by engine code. Instead, it is + * the minimal key-value storage contract used to implement real backends such as RocksDB-backed + * stores. + * + *

Rather than copying the full Hazelcast {@code IMap} surface, it keeps only the minimum + * operations that remain meaningful when switching between backend implementations. + * + * @param key type + * @param value type + */ +public interface StateStore { + + /** + * Retrieves the value for the given key. + * + * @param key key to look up + * @return stored value, or {@code null} if absent + */ + V get(K key); + + /** + * Stores a value for the given key. + * + * @param key key to store + * @param value value to store + */ + void put(K key, V value); + + /** + * Stores a value only when the key is currently absent. + * + * @param key key to store + * @param value value to store + * @return existing value if present, otherwise {@code null} + */ + V putIfAbsent(K key, V value); + + /** + * Removes the value for the given key. + * + * @param key key to remove + */ + void remove(K key); + + /** + * Returns whether the key exists. + * + * @param key key to check + * @return {@code true} if the key exists + */ + boolean containsKey(K key); + + /** + * Returns whether the store is empty. + * + * @return {@code true} if the store is empty + */ + boolean isEmpty(); + + /** + * Returns the number of key-value pairs in the store. + * + * @return size of the store + */ + int size(); + + /** + * Returns the default value when the key is absent. + * + * @param key key to look up + * @param defaultValue default value + * @return stored value or the default value + */ + default V getOrDefault(K key, V defaultValue) { + V value = get(key); + return value != null ? value : defaultValue; + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/CheckpointOverviewStateStore.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/CheckpointOverviewStateStore.java new file mode 100644 index 000000000000..342024ccfe23 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/CheckpointOverviewStateStore.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore.checkpoint; + +import org.apache.seatunnel.engine.core.checkpoint.CheckpointOverview; +import org.apache.seatunnel.engine.core.checkpoint.PipelineCheckpointOverview; +import org.apache.seatunnel.engine.server.common.statestore.StateStore; + +import java.util.function.Consumer; + +/** + * Store for checkpoint monitoring overview state. + * + *

This store keeps checkpoint overview data used by monitoring and REST queries. It is + * operational and observability-oriented state rather than core failover state. + */ +public interface CheckpointOverviewStateStore extends StateStore { + + /** + * Updates the overview for the given job and pipeline. + * + *

If no overview exists for the job, a new one is created. If no pipeline overview exists, + * it is created before applying the updater. + * + * @param jobId job identifier + * @param pipelineId pipeline identifier + * @param updater pipeline-level updater + */ + void updateOverview(long jobId, int pipelineId, Consumer updater); + + /** + * Returns the number of jobs currently tracked by the overview store. + * + * @return tracked job count + */ + long getOverviewJobCount(); + + /** + * Returns the total number of in-progress checkpoints across all tracked jobs and pipelines. + * + * @return in-progress checkpoint count + */ + long getInProgressCheckpointCount(); + + /** + * Returns the total number of retained checkpoint history entries across all tracked jobs and + * pipelines. + * + * @return retained checkpoint history entry count + */ + long getRetainedHistoryCount(); +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/hazelcast/HazelcastCheckpointOverviewStateStore.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/hazelcast/HazelcastCheckpointOverviewStateStore.java new file mode 100644 index 000000000000..0a7b9482be8d --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/hazelcast/HazelcastCheckpointOverviewStateStore.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore.checkpoint.hazelcast; + +import org.apache.seatunnel.engine.core.checkpoint.CheckpointOverview; +import org.apache.seatunnel.engine.core.checkpoint.PipelineCheckpointOverview; +import org.apache.seatunnel.engine.server.common.statestore.checkpoint.CheckpointOverviewStateStore; + +import com.hazelcast.core.EntryEvent; +import com.hazelcast.map.IMap; +import com.hazelcast.map.listener.EntryAddedListener; +import com.hazelcast.map.listener.EntryExpiredListener; +import com.hazelcast.map.listener.EntryRemovedListener; +import com.hazelcast.map.listener.EntryUpdatedListener; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +/** Hazelcast-backed implementation of {@link CheckpointOverviewStateStore}. */ +public class HazelcastCheckpointOverviewStateStore + implements CheckpointOverviewStateStore, AutoCloseable { + + private final IMap overviewMap; + private final AtomicLong overviewJobCount = new AtomicLong(); + private final AtomicLong inProgressCheckpointCount = new AtomicLong(); + private final AtomicLong retainedHistoryCount = new AtomicLong(); + private final Object overviewStatsLock = new Object(); + private final Set dirtyJobIds = ConcurrentHashMap.newKeySet(); + private final Map statsByJobId = new HashMap<>(); + private final AtomicBoolean statsInitializing = new AtomicBoolean(false); + private volatile UUID overviewListenerId; + + public HazelcastCheckpointOverviewStateStore(IMap overviewMap) { + this.overviewMap = Objects.requireNonNull(overviewMap, "overviewMap"); + initStats(); + } + + @Override + public CheckpointOverview get(Long jobId) { + return overviewMap.get(jobId); + } + + @Override + public void put(Long jobId, CheckpointOverview overview) { + overviewMap.put(jobId, overview); + } + + @Override + public CheckpointOverview putIfAbsent(Long jobId, CheckpointOverview overview) { + return overviewMap.putIfAbsent(jobId, overview); + } + + @Override + public void remove(Long jobId) { + overviewMap.remove(jobId); + } + + @Override + public boolean containsKey(Long jobId) { + return overviewMap.containsKey(jobId); + } + + @Override + public boolean isEmpty() { + return overviewJobCount.get() == 0L; + } + + @Override + public int size() { + return Math.toIntExact(overviewJobCount.get()); + } + + @Override + public void updateOverview( + long jobId, int pipelineId, Consumer updater) { + overviewMap.compute( + jobId, + (id, overview) -> { + CheckpointOverview snapshot = + overview == null ? new CheckpointOverview(jobId) : overview; + PipelineCheckpointOverview pipeline = snapshot.getOrCreatePipeline(pipelineId); + updater.accept(pipeline); + snapshot.setUpdatedAt(System.currentTimeMillis()); + return snapshot; + }); + } + + @Override + public long getOverviewJobCount() { + return overviewJobCount.get(); + } + + @Override + public long getInProgressCheckpointCount() { + return inProgressCheckpointCount.get(); + } + + @Override + public long getRetainedHistoryCount() { + return retainedHistoryCount.get(); + } + + @Override + public void close() { + removeOverviewListener(); + } + + private void initStats() { + removeOverviewListener(); + overviewListenerId = + overviewMap.addEntryListener(new CheckpointOverviewEntryListener(), true); + statsInitializing.set(true); + dirtyJobIds.clear(); + + Map snapshotStats = new HashMap<>(); + overviewMap.forEach( + (jobId, overview) -> snapshotStats.put(jobId, toOverviewStats(overview))); + + Set dirtyJobIdsDuringSnapshot = new HashSet<>(dirtyJobIds); + for (Long jobId : dirtyJobIdsDuringSnapshot) { + snapshotStats.put(jobId, toOverviewStats(overviewMap.get(jobId))); + } + + synchronized (overviewStatsLock) { + statsByJobId.clear(); + overviewJobCount.set(0L); + inProgressCheckpointCount.set(0L); + retainedHistoryCount.set(0L); + snapshotStats.forEach(this::replaceOverviewStatsLocked); + statsInitializing.set(false); + + Set postSnapshotDirtyJobIds = new HashSet<>(dirtyJobIds); + dirtyJobIds.clear(); + for (Long jobId : postSnapshotDirtyJobIds) { + replaceOverviewStatsLocked(jobId, toOverviewStats(overviewMap.get(jobId))); + } + } + } + + private void removeOverviewListener() { + if (overviewListenerId != null) { + overviewMap.removeEntryListener(overviewListenerId); + overviewListenerId = null; + } + statsInitializing.set(false); + dirtyJobIds.clear(); + synchronized (overviewStatsLock) { + statsByJobId.clear(); + } + overviewJobCount.set(0L); + inProgressCheckpointCount.set(0L); + retainedHistoryCount.set(0L); + } + + private CheckpointOverviewStats toOverviewStats(CheckpointOverview overview) { + if (overview == null) { + return CheckpointOverviewStats.EMPTY; + } + return new CheckpointOverviewStats( + 1L, getInProgressCount(overview), getHistoryCount(overview)); + } + + private void replaceOverviewStatsLocked(Long jobId, CheckpointOverviewStats stats) { + CheckpointOverviewStats currentStats = statsByJobId.get(jobId); + if (currentStats != null) { + overviewJobCount.addAndGet(-currentStats.jobCount); + inProgressCheckpointCount.addAndGet(-currentStats.inProgressCheckpointCount); + retainedHistoryCount.addAndGet(-currentStats.retainedHistoryCount); + } + + if (stats.isEmpty()) { + statsByJobId.remove(jobId); + return; + } + + statsByJobId.put(jobId, stats); + overviewJobCount.addAndGet(stats.jobCount); + inProgressCheckpointCount.addAndGet(stats.inProgressCheckpointCount); + retainedHistoryCount.addAndGet(stats.retainedHistoryCount); + } + + private long getInProgressCount(CheckpointOverview overview) { + return overview.getPipelines().values().stream() + .filter(Objects::nonNull) + .mapToLong(pipelineOverview -> pipelineOverview.getInProgress().size()) + .sum(); + } + + private long getHistoryCount(CheckpointOverview overview) { + return overview.getPipelines().values().stream() + .filter(Objects::nonNull) + .map(PipelineCheckpointOverview::getHistory) + .filter(Objects::nonNull) + .mapToLong(Collection::size) + .sum(); + } + + private void replaceOverviewStats(Long jobId, CheckpointOverview overview) { + if (statsInitializing.get()) { + dirtyJobIds.add(jobId); + return; + } + synchronized (overviewStatsLock) { + if (statsInitializing.get()) { + dirtyJobIds.add(jobId); + return; + } + replaceOverviewStatsLocked(jobId, toOverviewStats(overview)); + } + } + + private final class CheckpointOverviewEntryListener + implements EntryAddedListener, + EntryUpdatedListener, + EntryRemovedListener, + EntryExpiredListener { + + @Override + public void entryAdded(EntryEvent event) { + replaceOverviewStats(event.getKey(), event.getValue()); + } + + @Override + public void entryUpdated(EntryEvent event) { + replaceOverviewStats(event.getKey(), event.getValue()); + } + + @Override + public void entryRemoved(EntryEvent event) { + replaceOverviewStats(event.getKey(), null); + } + + @Override + public void entryExpired(EntryEvent event) { + replaceOverviewStats(event.getKey(), null); + } + } + + private static final class CheckpointOverviewStats { + private static final CheckpointOverviewStats EMPTY = + new CheckpointOverviewStats(0L, 0L, 0L); + + private final long jobCount; + private final long inProgressCheckpointCount; + private final long retainedHistoryCount; + + private CheckpointOverviewStats( + long jobCount, long inProgressCheckpointCount, long retainedHistoryCount) { + this.jobCount = jobCount; + this.inProgressCheckpointCount = inProgressCheckpointCount; + this.retainedHistoryCount = retainedHistoryCount; + } + + private boolean isEmpty() { + return jobCount == 0L && inProgressCheckpointCount == 0L && retainedHistoryCount == 0L; + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/hazelcast/HazelcastEngineStateStores.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/hazelcast/HazelcastEngineStateStores.java new file mode 100644 index 000000000000..6e5a3a9bdee7 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/hazelcast/HazelcastEngineStateStores.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore.hazelcast; + +import org.apache.seatunnel.engine.server.common.statestore.AuthoritativeStateStores; +import org.apache.seatunnel.engine.server.common.statestore.AuxiliaryStateStores; +import org.apache.seatunnel.engine.server.common.statestore.DefaultAuthoritativeStateStores; +import org.apache.seatunnel.engine.server.common.statestore.DefaultAuxiliaryStateStores; +import org.apache.seatunnel.engine.server.common.statestore.EngineStateStores; +import org.apache.seatunnel.engine.server.common.statestore.checkpoint.CheckpointOverviewStateStore; +import org.apache.seatunnel.engine.server.common.statestore.checkpoint.hazelcast.HazelcastCheckpointOverviewStateStore; +import org.apache.seatunnel.engine.server.common.statestore.metrics.MetricsSnapshotStateStore; +import org.apache.seatunnel.engine.server.common.statestore.metrics.hazelcast.HazelcastMetricsSnapshotStateStore; + +import com.hazelcast.spi.impl.NodeEngine; + +import java.util.Objects; + +import static org.apache.seatunnel.engine.server.common.statestore.EngineStateStoreNames.CHECKPOINT_MONITOR; +import static org.apache.seatunnel.engine.server.common.statestore.EngineStateStoreNames.RUNNING_JOB_METRICS; + +/** + * {@link EngineStateStores} implementation backed by Hazelcast. + * + *

Engine code is not expected to reference this implementation directly. It is intended to be + * created only during bootstrap and injected through interfaces. + */ +public class HazelcastEngineStateStores implements EngineStateStores { + + private final NodeEngine nodeEngine; + private final int metricsPartitionCount; + private volatile AuthoritativeStateStores authoritativeStateStores; + private volatile AuxiliaryStateStores auxiliaryStateStores; + + public HazelcastEngineStateStores(NodeEngine nodeEngine, int metricsPartitionCount) { + Objects.requireNonNull(nodeEngine, "nodeEngine"); + this.nodeEngine = nodeEngine; + this.metricsPartitionCount = metricsPartitionCount; + } + + private void ensureInitialized() { + if (authoritativeStateStores != null && auxiliaryStateStores != null) { + return; + } + synchronized (this) { + if (authoritativeStateStores != null && auxiliaryStateStores != null) { + return; + } + + MetricsSnapshotStateStore metricsSnapshotStore = + new HazelcastMetricsSnapshotStateStore( + nodeEngine.getHazelcastInstance().getMap(RUNNING_JOB_METRICS), + metricsPartitionCount); + CheckpointOverviewStateStore checkpointOverviewStateStore = + new HazelcastCheckpointOverviewStateStore( + nodeEngine.getHazelcastInstance().getMap(CHECKPOINT_MONITOR)); + this.authoritativeStateStores = new DefaultAuthoritativeStateStores(); + this.auxiliaryStateStores = + new DefaultAuxiliaryStateStores( + metricsSnapshotStore, checkpointOverviewStateStore); + } + } + + @Override + public AuthoritativeStateStores authoritative() { + ensureInitialized(); + return authoritativeStateStores; + } + + @Override + public AuxiliaryStateStores auxiliary() { + ensureInitialized(); + return auxiliaryStateStores; + } + + @Override + public void close() { + if (auxiliaryStateStores != null) { + auxiliaryStateStores.close(); + } + if (authoritativeStateStores != null) { + authoritativeStateStores.close(); + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/metrics/MetricsSnapshotStateStore.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/metrics/MetricsSnapshotStateStore.java new file mode 100644 index 000000000000..03f01ec5ef7f --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/metrics/MetricsSnapshotStateStore.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore.metrics; + +import org.apache.seatunnel.engine.server.common.statestore.StateStore; +import org.apache.seatunnel.engine.server.dag.physical.PipelineLocation; +import org.apache.seatunnel.engine.server.execution.TaskLocation; +import org.apache.seatunnel.engine.server.metrics.SeaTunnelMetricsContext; + +import java.util.Collections; +import java.util.Map; + +/** + * Store for runtime task metrics snapshots. + * + *

Here, {@code merge()} does not mean merging the internals of {@link SeaTunnelMetricsContext}. + * It means batch-applying the latest snapshots for multiple tasks. + * + *

In the current SeaTunnel metrics reporting flow, each task sends its latest {@link + * SeaTunnelMetricsContext} to the coordinator, and the coordinator replaces the snapshot by task + * key. The Hazelcast implementation uses {@code compute()} internally to safely update the outer + * partition map, while the RocksDB implementation uses task location directly as the key, so + * task-level overwrite preserves the same storage meaning. Even when a backend uses internal + * partition buckets, the exposed {@link StateStore} contract remains task-based, so {@link #size()} + * must report the number of task snapshots rather than backend-specific buckets. + */ +public interface MetricsSnapshotStateStore + extends StateStore { + + /** + * Applies metrics snapshots for multiple tasks in a single batch. + * + * @param snapshot snapshot batch to apply + */ + void merge(Map snapshot); + + /** + * Stores a single task snapshot by delegating to {@link #merge(Map)}. + * + * @param taskLocation task location to store + * @param metricsContext metrics snapshot to store + */ + @Override + default void put(TaskLocation taskLocation, SeaTunnelMetricsContext metricsContext) { + merge(Collections.singletonMap(taskLocation, metricsContext)); + } + + /** + * Checks whether a task snapshot exists. + * + * @param taskLocation task location to check + * @return {@code true} if a snapshot exists for the task + */ + @Override + default boolean containsKey(TaskLocation taskLocation) { + return get(taskLocation) != null; + } + + /** + * Conditional insertion is intentionally not exposed for metrics snapshots because the current + * engine model treats them as latest-snapshot overwrites. + * + * @param taskLocation task location to store + * @param metricsContext metrics snapshot to store + * @return never returns normally + */ + @Override + default SeaTunnelMetricsContext putIfAbsent( + TaskLocation taskLocation, SeaTunnelMetricsContext metricsContext) { + throw new UnsupportedOperationException( + "Metrics snapshots are updated through merge semantics rather than putIfAbsent."); + } + + /** + * Removes all task metrics belonging to a specific pipeline. + * + * @param pipelineLocation pipeline location to remove + */ + void removePipeline(PipelineLocation pipelineLocation); + + /** + * Checks whether any task snapshot exists for a specific pipeline. + * + * @param pipelineLocation pipeline location to check + * @return {@code true} if any snapshot exists for the pipeline + */ + boolean containsPipeline(PipelineLocation pipelineLocation); + + /** + * Returns the number of active backend partition buckets currently holding task snapshots. + * + *

This value is observability-oriented and may differ from {@link #size()}, which reports + * logical task snapshot count. + * + * @return active backend partition bucket count + */ + int activePartitionKeyCount(); +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/metrics/hazelcast/HazelcastMetricsSnapshotStateStore.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/metrics/hazelcast/HazelcastMetricsSnapshotStateStore.java new file mode 100644 index 000000000000..be4e3f83ebcb --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/common/statestore/metrics/hazelcast/HazelcastMetricsSnapshotStateStore.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore.metrics.hazelcast; + +import org.apache.seatunnel.engine.server.common.statestore.metrics.MetricsSnapshotStateStore; +import org.apache.seatunnel.engine.server.dag.physical.PipelineLocation; +import org.apache.seatunnel.engine.server.execution.TaskLocation; +import org.apache.seatunnel.engine.server.metrics.SeaTunnelMetricsContext; + +import com.hazelcast.core.EntryEvent; +import com.hazelcast.map.IMap; +import com.hazelcast.map.listener.EntryAddedListener; +import com.hazelcast.map.listener.EntryRemovedListener; +import com.hazelcast.map.listener.EntryUpdatedListener; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +/** Implementation backed by a partitioned Hazelcast metrics {@link IMap}. */ +public class HazelcastMetricsSnapshotStateStore + implements MetricsSnapshotStateStore, AutoCloseable { + + private final IMap> metricsImap; + private final int partitionCount; + private final AtomicLong activePartitionKeyCount = new AtomicLong(); + private final AtomicLong taskSnapshotCount = new AtomicLong(); + private final Object metricsStatsLock = new Object(); + private final Set dirtyPartitionKeys = ConcurrentHashMap.newKeySet(); + private final Map statsByPartitionKey = new HashMap<>(); + private final AtomicBoolean statsInitializing = new AtomicBoolean(false); + private volatile UUID metricsListenerId; + + public HazelcastMetricsSnapshotStateStore( + IMap> metricsImap, + int partitionCount) { + this.metricsImap = metricsImap; + this.partitionCount = partitionCount; + initStats(); + } + + @Override + public void merge(Map snapshot) { + if (snapshot == null || snapshot.isEmpty()) { + return; + } + + Map> partitioned = new HashMap<>(); + snapshot.forEach( + (key, value) -> { + long partition = partition(key); + partitioned.computeIfAbsent(partition, k -> new HashMap<>()).put(key, value); + }); + + partitioned + .entrySet() + .parallelStream() + .forEach( + entry -> { + metricsImap.compute( + entry.getKey(), + (k, oldVal) -> { + if (oldVal == null) oldVal = new HashMap<>(); + oldVal.putAll(entry.getValue()); + return oldVal; + }); + }); + } + + @Override + public SeaTunnelMetricsContext get(TaskLocation taskLocation) { + Map partitionMap = + metricsImap.get(partition(taskLocation)); + if (partitionMap == null) { + return null; + } + return partitionMap.get(taskLocation); + } + + @Override + public void remove(final TaskLocation taskLocation) { + metricsImap.compute( + partition(taskLocation), + (ignored, current) -> { + if (current == null) { + return null; + } + Map updated = new HashMap<>(current); + updated.remove(taskLocation); + return updated.isEmpty() ? null : updated; + }); + } + + @Override + public void removePipeline(final PipelineLocation pipelineLocation) { + Map> partitionedTasks = new HashMap<>(); + for (Map.Entry> entry : + metricsImap.entrySet()) { + long partition = entry.getKey(); + List tasksToRemove = + entry.getValue().keySet().stream() + .filter( + t -> + t.getTaskGroupLocation() + .getPipelineLocation() + .equals(pipelineLocation)) + .collect(Collectors.toList()); + if (!tasksToRemove.isEmpty()) { + partitionedTasks.put(partition, tasksToRemove); + } + } + + partitionedTasks + .entrySet() + .parallelStream() + .forEach( + entry -> { + long partition = entry.getKey(); + List tasks = entry.getValue(); + metricsImap.compute( + partition, + (k, oldVal) -> { + if (oldVal != null) { + tasks.forEach(oldVal::remove); + if (oldVal.isEmpty()) return null; + } + return oldVal; + }); + }); + } + + @Override + public boolean containsPipeline(PipelineLocation pipelineLocation) { + for (Map partitionMap : metricsImap.values()) { + boolean found = + partitionMap.keySet().stream() + .anyMatch( + taskLocation -> + pipelineLocation.equals( + taskLocation + .getTaskGroupLocation() + .getPipelineLocation())); + if (found) { + return true; + } + } + return false; + } + + @Override + public int size() { + return Math.toIntExact(taskSnapshotCount.get()); + } + + @Override + public boolean isEmpty() { + return taskSnapshotCount.get() == 0L; + } + + @Override + public int activePartitionKeyCount() { + return Math.toIntExact(activePartitionKeyCount.get()); + } + + private long partition(TaskLocation taskLocation) { + return (taskLocation.hashCode() & Integer.MAX_VALUE) % partitionCount; + } + + @Override + public void close() { + removeMetricsListener(); + } + + private void initStats() { + removeMetricsListener(); + metricsListenerId = metricsImap.addEntryListener(new MetricsEntryListener(), true); + statsInitializing.set(true); + dirtyPartitionKeys.clear(); + + Map snapshotStats = new HashMap<>(); + metricsImap.forEach( + (partitionKey, metrics) -> + snapshotStats.put(partitionKey, toRunningJobMetricsStats(metrics))); + + Set dirtyKeysDuringSnapshot = new HashSet<>(dirtyPartitionKeys); + for (Long partitionKey : dirtyKeysDuringSnapshot) { + snapshotStats.put( + partitionKey, toRunningJobMetricsStats(metricsImap.get(partitionKey))); + } + + synchronized (metricsStatsLock) { + statsByPartitionKey.clear(); + activePartitionKeyCount.set(0L); + taskSnapshotCount.set(0L); + snapshotStats.forEach(this::replaceRunningJobMetricsStatsLocked); + statsInitializing.set(false); + + Set postSnapshotDirtyKeys = new HashSet<>(dirtyPartitionKeys); + dirtyPartitionKeys.clear(); + for (Long partitionKey : postSnapshotDirtyKeys) { + replaceRunningJobMetricsStatsLocked( + partitionKey, toRunningJobMetricsStats(metricsImap.get(partitionKey))); + } + } + } + + private RunningJobMetricsStats toRunningJobMetricsStats( + Map metrics) { + if (metrics == null || metrics.isEmpty()) { + return RunningJobMetricsStats.EMPTY; + } + return new RunningJobMetricsStats(1L, metrics.size()); + } + + private void replaceRunningJobMetricsStatsLocked( + Long partitionKey, RunningJobMetricsStats stats) { + RunningJobMetricsStats currentStats = statsByPartitionKey.get(partitionKey); + if (currentStats != null) { + activePartitionKeyCount.addAndGet(-currentStats.partitionKeyCount); + taskSnapshotCount.addAndGet(-currentStats.taskSnapshotCount); + } + + if (stats.isEmpty()) { + statsByPartitionKey.remove(partitionKey); + return; + } + + statsByPartitionKey.put(partitionKey, stats); + activePartitionKeyCount.addAndGet(stats.partitionKeyCount); + taskSnapshotCount.addAndGet(stats.taskSnapshotCount); + } + + private void replaceRunningJobMetricsStats( + Long partitionKey, Map metrics) { + if (statsInitializing.get()) { + dirtyPartitionKeys.add(partitionKey); + return; + } + synchronized (metricsStatsLock) { + if (statsInitializing.get()) { + dirtyPartitionKeys.add(partitionKey); + return; + } + replaceRunningJobMetricsStatsLocked(partitionKey, toRunningJobMetricsStats(metrics)); + } + } + + private void removeMetricsListener() { + if (metricsListenerId != null) { + metricsImap.removeEntryListener(metricsListenerId); + metricsListenerId = null; + } + statsInitializing.set(false); + dirtyPartitionKeys.clear(); + synchronized (metricsStatsLock) { + statsByPartitionKey.clear(); + } + activePartitionKeyCount.set(0L); + taskSnapshotCount.set(0L); + } + + private final class MetricsEntryListener + implements EntryAddedListener>, + EntryUpdatedListener>, + EntryRemovedListener> { + + @Override + public void entryAdded(EntryEvent> event) { + replaceRunningJobMetricsStats(event.getKey(), event.getValue()); + } + + @Override + public void entryUpdated( + EntryEvent> event) { + replaceRunningJobMetricsStats(event.getKey(), event.getValue()); + } + + @Override + public void entryRemoved( + EntryEvent> event) { + replaceRunningJobMetricsStats(event.getKey(), null); + } + } + + private static final class RunningJobMetricsStats { + private static final RunningJobMetricsStats EMPTY = new RunningJobMetricsStats(0L, 0L); + + private final long partitionKeyCount; + private final long taskSnapshotCount; + + private RunningJobMetricsStats(long partitionKeyCount, long taskSnapshotCount) { + this.partitionKeyCount = partitionKeyCount; + this.taskSnapshotCount = taskSnapshotCount; + } + + private boolean isEmpty() { + return partitionKeyCount == 0L && taskSnapshotCount == 0L; + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/execution/TaskExecutionContext.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/execution/TaskExecutionContext.java index 74ffb7aad9ec..ad79765930cc 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/execution/TaskExecutionContext.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/execution/TaskExecutionContext.java @@ -17,32 +17,33 @@ package org.apache.seatunnel.engine.server.execution; -import org.apache.seatunnel.engine.common.Constant; -import org.apache.seatunnel.engine.server.SeaTunnelServer; import org.apache.seatunnel.engine.server.TaskExecutionService; +import org.apache.seatunnel.engine.server.common.SeaTunnelEngineContext; import org.apache.seatunnel.engine.server.metrics.SeaTunnelMetricsContext; import org.apache.seatunnel.engine.server.utils.NodeEngineUtil; import com.hazelcast.cluster.Address; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.logging.ILogger; -import com.hazelcast.map.IMap; import com.hazelcast.spi.impl.NodeEngineImpl; import com.hazelcast.spi.impl.operationservice.Operation; import com.hazelcast.spi.impl.operationservice.impl.InvocationFuture; -import java.util.HashMap; - public class TaskExecutionContext { private final Task task; private final NodeEngineImpl nodeEngine; + private final SeaTunnelEngineContext engineContext; private final TaskExecutionService taskExecutionService; public TaskExecutionContext( - Task task, NodeEngineImpl nodeEngine, TaskExecutionService taskExecutionService) { + Task task, + NodeEngineImpl nodeEngine, + SeaTunnelEngineContext engineContext, + TaskExecutionService taskExecutionService) { this.task = task; this.nodeEngine = nodeEngine; + this.engineContext = engineContext; this.taskExecutionService = taskExecutionService; } @@ -59,18 +60,10 @@ public ILogger getLogger() { } public SeaTunnelMetricsContext getOrCreateMetricsContext(TaskLocation taskLocation) { - IMap> map = - nodeEngine.getHazelcastInstance().getMap(Constant.IMAP_RUNNING_JOB_METRICS); - int partitionCount = - taskExecutionService - .getSeaTunnelConfig() - .getEngineConfig() - .getJobMetricsPartitionCount(); - long partition = SeaTunnelServer.getMetricsImapPartition(taskLocation, partitionCount); - HashMap centralMap = map.get(partition); - return centralMap == null || centralMap.get(taskLocation) == null - ? new SeaTunnelMetricsContext() - : centralMap.get(taskLocation); + return engineContext + .getStateStores() + .metricsSnapshotStore() + .getOrDefault(taskLocation, new SeaTunnelMetricsContext()); } public T getTask() { diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExports.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExports.java index c8a86970398d..0369818ea98f 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExports.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExports.java @@ -18,6 +18,7 @@ package org.apache.seatunnel.engine.server.telemetry.metrics.exports; import org.apache.seatunnel.engine.common.Constant; +import org.apache.seatunnel.engine.server.common.statestore.EngineStateStoreNames; import org.apache.seatunnel.engine.server.telemetry.metrics.AbstractCollector; import com.hazelcast.core.HazelcastInstanceNotActiveException; @@ -39,11 +40,11 @@ public class EngineStateStoreMetricExports extends AbstractCollector { Constant.IMAP_RUNNING_JOB_STATE, Constant.IMAP_STATE_TIMESTAMPS, Constant.IMAP_OWNED_SLOT_PROFILES, - Constant.IMAP_RUNNING_JOB_METRICS, + EngineStateStoreNames.RUNNING_JOB_METRICS, Constant.IMAP_FINISHED_JOB_STATE, Constant.IMAP_FINISHED_JOB_METRICS, Constant.IMAP_FINISHED_JOB_VERTEX_INFO, - Constant.IMAP_CHECKPOINT_MONITOR, + EngineStateStoreNames.CHECKPOINT_MONITOR, Constant.IMAP_CONNECTOR_JAR_REF_COUNTERS, Constant.IMAP_CHECKPOINT_ID, Constant.IMAP_PENDING_PIPELINE_CLEANUP); diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServicePipelineCleanupTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServicePipelineCleanupTest.java index 07f32459d1a5..e8218e46ba0c 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServicePipelineCleanupTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServicePipelineCleanupTest.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.engine.common.Constant; import org.apache.seatunnel.engine.core.job.PipelineStatus; +import org.apache.seatunnel.engine.server.common.statestore.metrics.MetricsSnapshotStateStore; import org.apache.seatunnel.engine.server.dag.physical.PipelineLocation; import org.apache.seatunnel.engine.server.execution.TaskGroupLocation; import org.apache.seatunnel.engine.server.execution.TaskLocation; @@ -252,14 +253,9 @@ private void upsertMetricsForPipeline(PipelineLocation pipelineLocation) { } private boolean hasMetricsForPipeline(PipelineLocation pipelineLocation) { - IMap> metricsIMap = - nodeEngine.getHazelcastInstance().getMap(Constant.IMAP_RUNNING_JOB_METRICS); - return metricsIMap.entrySet().stream() - .flatMap(entry -> entry.getValue().keySet().stream()) - .anyMatch( - taskLocation -> - pipelineLocation.equals( - taskLocation.getTaskGroupLocation().getPipelineLocation())); + MetricsSnapshotStateStore metricsStore = + server.getEngineContext().getStateStores().metricsSnapshotStore(); + return metricsStore.containsPipeline(pipelineLocation); } private void awaitCoordinatorActive(CoordinatorService coordinatorService) { diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceTest.java index a2ab94924875..59d86896a7c6 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CoordinatorServiceTest.java @@ -35,6 +35,8 @@ import org.apache.seatunnel.engine.core.job.JobImmutableInformation; import org.apache.seatunnel.engine.core.job.JobInfo; import org.apache.seatunnel.engine.core.job.PipelineStatus; +import org.apache.seatunnel.engine.server.common.SeaTunnelEngineContext; +import org.apache.seatunnel.engine.server.common.statestore.metrics.MetricsSnapshotStateStore; import org.apache.seatunnel.engine.server.dag.physical.PhysicalPlan; import org.apache.seatunnel.engine.server.dag.physical.PipelineLocation; import org.apache.seatunnel.engine.server.dag.physical.SubPlan; @@ -525,9 +527,11 @@ private CoordinatorService newMockCoordinatorService( Mockito.when(nodeEngine.getLogger(Mockito.any(Class.class))).thenReturn(logger); Mockito.when(nodeEngine.getHazelcastInstance()).thenReturn(hazelcastInstance); Mockito.when(hazelcastInstance.getMap(Mockito.anyString())).thenReturn(map); + SeaTunnelEngineContext engineContext = Mockito.mock(SeaTunnelEngineContext.class); + Mockito.when(server.getEngineContext()).thenReturn(engineContext); CoordinatorService coordinatorService = - new CoordinatorService(nodeEngine, server, engineConfig); + new CoordinatorService(nodeEngine, server, server.getEngineContext(), engineConfig); stopCoordinatorSchedulers(coordinatorService); return coordinatorService; } @@ -920,12 +924,12 @@ void testCleanupMetricsImap() { "batch_fake_to_console.conf", "test_cleanup_metrics_imap"); CoordinatorService coordinatorService = jobInformation.coordinatorService; - IMap> metricsImap = - coordinatorService.getMetricsImap(); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted(() -> Assertions.assertFalse(metricsImap.isEmpty())); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted(() -> Assertions.assertTrue(metricsImap.isEmpty())); + MetricsSnapshotStateStore metricsSnapshotStateStore = + coordinatorService.getMetricsSnapshotStateStore(); + await().atMost(10000, TimeUnit.MILLISECONDS) + .untilAsserted(() -> Assertions.assertFalse(metricsSnapshotStateStore.isEmpty())); + await().atMost(10000, TimeUnit.MILLISECONDS) + .untilAsserted(() -> Assertions.assertTrue(metricsSnapshotStateStore.isEmpty())); jobInformation.coordinatorService.clearCoordinatorService(); jobInformation.coordinatorServiceTest.shutdown(); @@ -941,12 +945,12 @@ void testCleanupMetricsImapWithPartitionConfig() { "batch_fake_to_console.conf", "test_cleanup_metrics_imap_with_partition_config"); CoordinatorService coordinatorService = jobInformation.coordinatorService; - IMap> metricsImap = - coordinatorService.getMetricsImap(); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted(() -> Assertions.assertFalse(metricsImap.isEmpty())); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted(() -> Assertions.assertTrue(metricsImap.isEmpty())); + MetricsSnapshotStateStore metricsSnapshotStateStore = + coordinatorService.getMetricsSnapshotStateStore(); + await().atMost(10000, TimeUnit.MILLISECONDS) + .untilAsserted(() -> Assertions.assertFalse(metricsSnapshotStateStore.isEmpty())); + await().atMost(10000, TimeUnit.MILLISECONDS) + .untilAsserted(() -> Assertions.assertTrue(metricsSnapshotStateStore.isEmpty())); jobInformation.coordinatorService.clearCoordinatorService(); jobInformation.coordinatorServiceTest.shutdown(); @@ -971,8 +975,8 @@ void testMetricsImapSizeWithPartitionConfig() { taskLocation.setTaskID(i); localMap.put(taskLocation, new SeaTunnelMetricsContext()); } - IMap> metricsImap = - server1.getCoordinatorService().getMetricsImap(); + MetricsSnapshotStateStore metricsSnapshotStateStore = + server1.getCoordinatorService().getMetricsSnapshotStateStore(); CompletableFuture.runAsync( () -> { try { @@ -988,8 +992,15 @@ void testMetricsImapSizeWithPartitionConfig() { throw new CompletionException(e); } }); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted(() -> Assertions.assertEquals(10, metricsImap.size())); + await().atMost(10000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> + Assertions.assertEquals( + 10, + metricsSnapshotStateStore.activePartitionKeyCount())); + await().atMost(10000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> Assertions.assertEquals(100, metricsSnapshotStateStore.size())); } finally { instance1.shutdown(); setDefaultConfigFile(); diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/hazelcast/HazelcastCheckpointOverviewStateStoreTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/hazelcast/HazelcastCheckpointOverviewStateStoreTest.java new file mode 100644 index 000000000000..17254bc969c3 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/common/statestore/checkpoint/hazelcast/HazelcastCheckpointOverviewStateStoreTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore.checkpoint.hazelcast; + +import org.apache.seatunnel.engine.core.checkpoint.CheckpointHistoryEntry; +import org.apache.seatunnel.engine.core.checkpoint.CheckpointInfo; +import org.apache.seatunnel.engine.core.checkpoint.CheckpointOverview; +import org.apache.seatunnel.engine.core.checkpoint.CheckpointStatus; +import org.apache.seatunnel.engine.core.checkpoint.CheckpointType; +import org.apache.seatunnel.engine.core.checkpoint.InProgressCheckpoint; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import com.hazelcast.config.Config; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.map.IMap; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class HazelcastCheckpointOverviewStateStoreTest { + + private static HazelcastInstance hazelcastInstance; + + @BeforeAll + static void beforeAll() { + Config config = new Config(); + config.setClusterName("HazelcastCheckpointOverviewStateStoreTest-" + System.nanoTime()); + hazelcastInstance = Hazelcast.newHazelcastInstance(config); + } + + @AfterAll + static void afterAll() { + if (hazelcastInstance != null) { + hazelcastInstance.shutdown(); + } + } + + @Test + void countAccessorsShouldUseLogicalOverviewCounts() { + IMap iMap = + hazelcastInstance.getMap("checkpoint-overview-counts"); + iMap.clear(); + HazelcastCheckpointOverviewStateStore store = + new HazelcastCheckpointOverviewStateStore(iMap); + + store.updateOverview( + 1L, + 1, + pipeline -> { + pipeline.getInProgress() + .add( + new InProgressCheckpoint( + 101L, CheckpointType.CHECKPOINT_TYPE, 10L, 1, 2)); + pipeline.getInProgress() + .add( + new InProgressCheckpoint( + 102L, CheckpointType.CHECKPOINT_TYPE, 20L, 1, 2)); + pipeline.addHistory(historyEntry(1L, 1, 201L), 8); + }); + store.updateOverview(1L, 2, pipeline -> pipeline.addHistory(historyEntry(1L, 2, 202L), 8)); + store.updateOverview( + 2L, + 1, + pipeline -> { + pipeline.getInProgress() + .add( + new InProgressCheckpoint( + 103L, CheckpointType.CHECKPOINT_TYPE, 30L, 1, 2)); + pipeline.addHistory(historyEntry(2L, 1, 203L), 8); + }); + + awaitOverviewStats(store, 2L, 3L, 3L, 2); + + store.remove(1L); + + awaitOverviewStats(store, 1L, 1L, 1L, 1); + } + + private static void awaitOverviewStats( + HazelcastCheckpointOverviewStateStore store, + long expectedOverviewJobCount, + long expectedInProgressCheckpointCount, + long expectedRetainedHistoryCount, + int expectedSize) { + await().atMost(10, TimeUnit.SECONDS) + .untilAsserted( + () -> { + assertEquals(expectedOverviewJobCount, store.getOverviewJobCount()); + assertEquals( + expectedInProgressCheckpointCount, + store.getInProgressCheckpointCount()); + assertEquals( + expectedRetainedHistoryCount, store.getRetainedHistoryCount()); + assertEquals(expectedSize, store.size()); + }); + } + + private static CheckpointHistoryEntry historyEntry( + long jobId, int pipelineId, long checkpointId) { + return CheckpointHistoryEntry.builder() + .jobId(jobId) + .pipelineId(pipelineId) + .checkpointInfo( + CheckpointInfo.builder() + .checkpointId(checkpointId) + .checkpointType(CheckpointType.CHECKPOINT_TYPE) + .status(CheckpointStatus.COMPLETED) + .triggerTimestamp(checkpointId) + .build()) + .build(); + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/common/statestore/metrics/hazelcast/HazelcastMetricsSnapshotStateStoreTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/common/statestore/metrics/hazelcast/HazelcastMetricsSnapshotStateStoreTest.java new file mode 100644 index 000000000000..7ef1ada64ef9 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/common/statestore/metrics/hazelcast/HazelcastMetricsSnapshotStateStoreTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.common.statestore.metrics.hazelcast; + +import org.apache.seatunnel.engine.server.dag.physical.PipelineLocation; +import org.apache.seatunnel.engine.server.execution.TaskGroupLocation; +import org.apache.seatunnel.engine.server.execution.TaskLocation; +import org.apache.seatunnel.engine.server.metrics.SeaTunnelMetricsContext; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import com.hazelcast.config.Config; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.map.IMap; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class HazelcastMetricsSnapshotStateStoreTest { + + private static HazelcastInstance hazelcastInstance; + private static final String METRIC_NAME = "test.metric"; + + @BeforeAll + static void beforeAll() { + Config config = new Config(); + config.setClusterName("HazelcastMetricsSnapshotStateStoreTest-" + System.nanoTime()); + hazelcastInstance = Hazelcast.newHazelcastInstance(config); + } + + @AfterAll + static void afterAll() { + if (hazelcastInstance != null) { + hazelcastInstance.shutdown(); + } + } + + @Test + void mergeShouldStoreAndOverwriteSnapshots() { + IMap> iMap = + hazelcastInstance.getMap("metrics-snapshot-merge"); + iMap.clear(); + HazelcastMetricsSnapshotStateStore store = new HazelcastMetricsSnapshotStateStore(iMap, 8); + + TaskLocation taskOne = taskLocation(1L, 10, 100L, 0L, 0); + TaskLocation taskTwo = taskLocation(1L, 11, 101L, 0L, 0); + SeaTunnelMetricsContext metricsOne = metricsContextWithCounterValue(1); + SeaTunnelMetricsContext metricsTwo = metricsContextWithCounterValue(2); + SeaTunnelMetricsContext updatedMetricsOne = metricsContextWithCounterValue(3); + + Map initialSnapshot = new LinkedHashMap<>(); + initialSnapshot.put(taskOne, metricsOne); + initialSnapshot.put(taskTwo, metricsTwo); + store.merge(initialSnapshot); + + assertCounterValue(1, store.get(taskOne)); + assertCounterValue(2, store.get(taskTwo)); + awaitSize(store, 2); + + store.merge(singletonSnapshot(taskOne, updatedMetricsOne)); + + assertCounterValue(3, store.get(taskOne)); + assertCounterValue(2, store.get(taskTwo)); + awaitSize(store, 2); + } + + @Test + void removeShouldDeleteSingleTaskSnapshot() { + IMap> iMap = + hazelcastInstance.getMap("metrics-snapshot-remove"); + iMap.clear(); + HazelcastMetricsSnapshotStateStore store = new HazelcastMetricsSnapshotStateStore(iMap, 8); + + TaskLocation taskOne = taskLocation(2L, 20, 200L, 0L, 0); + TaskLocation taskTwo = taskLocation(2L, 21, 201L, 0L, 0); + SeaTunnelMetricsContext metricsOne = metricsContextWithCounterValue(10); + SeaTunnelMetricsContext metricsTwo = metricsContextWithCounterValue(20); + + Map snapshot = new LinkedHashMap<>(); + snapshot.put(taskOne, metricsOne); + snapshot.put(taskTwo, metricsTwo); + store.merge(snapshot); + + store.remove(taskOne); + + assertNull(store.get(taskOne)); + assertCounterValue(20, store.get(taskTwo)); + awaitSize(store, 1); + + store.remove(taskTwo); + + assertNull(store.get(taskTwo)); + awaitSize(store, 0); + } + + @Test + void removePipelineShouldDeleteOnlyMatchingPipelineSnapshots() { + IMap> iMap = + hazelcastInstance.getMap("metrics-snapshot-remove-pipeline"); + iMap.clear(); + HazelcastMetricsSnapshotStateStore store = new HazelcastMetricsSnapshotStateStore(iMap, 8); + + PipelineLocation pipelineToRemove = new PipelineLocation(3L, 30); + PipelineLocation pipelineToKeep = new PipelineLocation(3L, 31); + TaskLocation removedOne = taskLocation(3L, 30, 300L, 0L, 0); + TaskLocation removedTwo = taskLocation(3L, 30, 301L, 0L, 0); + TaskLocation kept = taskLocation(3L, 31, 302L, 0L, 0); + SeaTunnelMetricsContext removedOneMetrics = metricsContextWithCounterValue(100); + SeaTunnelMetricsContext removedTwoMetrics = metricsContextWithCounterValue(200); + SeaTunnelMetricsContext keptMetrics = metricsContextWithCounterValue(300); + + Map snapshot = new LinkedHashMap<>(); + snapshot.put(removedOne, removedOneMetrics); + snapshot.put(removedTwo, removedTwoMetrics); + snapshot.put(kept, keptMetrics); + store.merge(snapshot); + + store.removePipeline(pipelineToRemove); + + assertNull(store.get(removedOne)); + assertNull(store.get(removedTwo)); + assertCounterValue(300, store.get(kept)); + awaitSize(store, 1); + assertEquals(pipelineToKeep, kept.getTaskGroupLocation().getPipelineLocation()); + } + + @Test + void sizeShouldCountTaskSnapshotsInsteadOfPartitionBuckets() { + IMap> iMap = + hazelcastInstance.getMap("metrics-snapshot-size"); + iMap.clear(); + HazelcastMetricsSnapshotStateStore store = new HazelcastMetricsSnapshotStateStore(iMap, 1); + + TaskLocation taskOne = taskLocation(4L, 40, 400L, 0L, 0); + TaskLocation taskTwo = taskLocation(4L, 40, 401L, 0L, 1); + + Map snapshot = new LinkedHashMap<>(); + snapshot.put(taskOne, metricsContextWithCounterValue(1)); + snapshot.put(taskTwo, metricsContextWithCounterValue(2)); + store.merge(snapshot); + + assertEquals(1, iMap.size()); + awaitSize(store, 2); + } + + private static void awaitSize(HazelcastMetricsSnapshotStateStore store, int expectedSize) { + await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(expectedSize, store.size())); + } + + private static Map singletonSnapshot( + TaskLocation taskLocation, SeaTunnelMetricsContext metricsContext) { + Map snapshot = new HashMap<>(); + snapshot.put(taskLocation, metricsContext); + return snapshot; + } + + private static SeaTunnelMetricsContext metricsContextWithCounterValue(long value) { + SeaTunnelMetricsContext metricsContext = new SeaTunnelMetricsContext(); + metricsContext.counter(METRIC_NAME).inc(value); + return metricsContext; + } + + private static void assertCounterValue(long expected, SeaTunnelMetricsContext metricsContext) { + assertEquals(expected, metricsContext.counter(METRIC_NAME).getCount()); + } + + private static TaskLocation taskLocation( + long jobId, + int pipelineId, + long taskGroupId, + long taskInGroupIndex, + int parallelismIndex) { + return new TaskLocation( + new TaskGroupLocation(jobId, pipelineId, taskGroupId), + taskInGroupIndex, + parallelismIndex); + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreLogicalMetricExportsTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreLogicalMetricExportsTest.java index dbbfa1dcd5a6..7cb201cdacd6 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreLogicalMetricExportsTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreLogicalMetricExportsTest.java @@ -32,6 +32,7 @@ import org.apache.seatunnel.engine.server.TestUtils; import org.apache.seatunnel.engine.server.checkpoint.CheckpointCloseReason; import org.apache.seatunnel.engine.server.checkpoint.CompletedCheckpoint; +import org.apache.seatunnel.engine.server.common.statestore.EngineStateStoreNames; import org.apache.seatunnel.engine.server.execution.TaskGroupLocation; import org.apache.seatunnel.engine.server.execution.TaskLocation; import org.apache.seatunnel.engine.server.master.JobHistoryService.JobState; @@ -187,7 +188,7 @@ void collectShouldTrackFinishedJobCleanupTotals() { private void seedRunningJobMetrics() { IMap> metricsMap = - instance.getMap(Constant.IMAP_RUNNING_JOB_METRICS); + instance.getMap(EngineStateStoreNames.RUNNING_JOB_METRICS); HashMap partitionZero = new HashMap<>(); partitionZero.put(new TaskLocation(new TaskGroupLocation(1L, 1, 1L), 0, 0), metricCtx()); partitionZero.put(new TaskLocation(new TaskGroupLocation(1L, 1, 2L), 0, 0), metricCtx()); diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExportsTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExportsTest.java index f9390dd99371..611be9dac882 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExportsTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/EngineStateStoreMetricExportsTest.java @@ -20,6 +20,7 @@ import org.apache.seatunnel.engine.common.Constant; import org.apache.seatunnel.engine.server.SeaTunnelServerStarter; import org.apache.seatunnel.engine.server.TestUtils; +import org.apache.seatunnel.engine.server.common.statestore.EngineStateStoreNames; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -110,11 +111,11 @@ void collectShouldCoverAllEngineStateStores() { Constant.IMAP_RUNNING_JOB_STATE, Constant.IMAP_STATE_TIMESTAMPS, Constant.IMAP_OWNED_SLOT_PROFILES, - Constant.IMAP_RUNNING_JOB_METRICS, + EngineStateStoreNames.RUNNING_JOB_METRICS, Constant.IMAP_FINISHED_JOB_STATE, Constant.IMAP_FINISHED_JOB_METRICS, Constant.IMAP_FINISHED_JOB_VERTEX_INFO, - Constant.IMAP_CHECKPOINT_MONITOR, + EngineStateStoreNames.CHECKPOINT_MONITOR, Constant.IMAP_CONNECTOR_JAR_REF_COUNTERS, Constant.IMAP_CHECKPOINT_ID, Constant.IMAP_PENDING_PIPELINE_CLEANUP)), From 0e2f4ef8d57000df40bfc2a3f32c635fe632b6c0 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Mon, 22 Jun 2026 10:25:40 +0800 Subject: [PATCH 050/375] [Fix][API] Add missing OptionRule validation for CatalogFactory creation path (#11127) --- .../api/table/factory/FactoryUtil.java | 8 ++- .../table/catalog/CatalogTableUtilTest.java | 66 +++++++++++++++++++ .../catalog/duckdb/DuckDBCatalogFactory.java | 12 +++- .../lance/catalog/LanceCatalogFactory.java | 2 +- 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryUtil.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryUtil.java index 8412bc60ce32..a4f078db0c30 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryUtil.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryUtil.java @@ -286,13 +286,17 @@ SeaTunnelSink createMultiTableSi public static Optional createOptionalCatalog( String catalogName, - ReadonlyConfig options, + ReadonlyConfig readonlyConfig, ClassLoader classLoader, String factoryIdentifier) { Optional optionalFactory = discoverOptionalFactory(classLoader, CatalogFactory.class, factoryIdentifier); + return optionalFactory.map( - catalogFactory -> catalogFactory.createCatalog(catalogName, options)); + catalogFactory -> { + ConfigValidator.of(readonlyConfig).validate(catalogFactory.optionRule()); + return catalogFactory.createCatalog(catalogName, readonlyConfig); + }); } public static URL getFactoryUrl(T factory) { diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtilTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtilTest.java index 73a6b3d47d42..e9cd815f277b 100644 --- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtilTest.java +++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtilTest.java @@ -22,7 +22,9 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.options.ConnectorCommonOptions; +import org.apache.seatunnel.api.table.factory.FactoryUtil; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; @@ -43,7 +45,10 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import static org.apache.seatunnel.common.constants.CollectionConstants.PLUGIN_NAME; @@ -208,4 +213,65 @@ public static String getTestConfigFile(String configFile) } return Paths.get(resource.toURI()).toString(); } + + @Test + void createOptionalCatalogWithValidConfig() { + Map configMap = new HashMap<>(); + configMap.put("username", "admin"); + configMap.put("password", "secret"); + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + Optional catalog = + FactoryUtil.createOptionalCatalog( + "test", config, Thread.currentThread().getContextClassLoader(), "InMemory"); + + Assertions.assertTrue(catalog.isPresent()); + } + + @Test + void createOptionalCatalogWithMissingRequiredOptionsThrows() { + Map configMap = new HashMap<>(); + configMap.put("host", "localhost"); + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + Assertions.assertThrows( + OptionValidationException.class, + () -> + FactoryUtil.createOptionalCatalog( + "test", + config, + Thread.currentThread().getContextClassLoader(), + "InMemory")); + } + + @Test + void createOptionalCatalogWithPartialRequiredOptionsThrows() { + Map configMap = new HashMap<>(); + configMap.put("username", "admin"); + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + Assertions.assertThrows( + OptionValidationException.class, + () -> + FactoryUtil.createOptionalCatalog( + "test", + config, + Thread.currentThread().getContextClassLoader(), + "InMemory")); + } + + @Test + void createOptionalCatalogWithUnknownFactoryReturnsEmpty() { + Map configMap = new HashMap<>(); + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + Optional catalog = + FactoryUtil.createOptionalCatalog( + "test", + config, + Thread.currentThread().getContextClassLoader(), + "NonExistentFactory"); + + Assertions.assertFalse(catalog.isPresent()); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/duckdb/DuckDBCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/duckdb/DuckDBCatalogFactory.java index 1e966111e7c7..53999ace8fea 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/duckdb/DuckDBCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/duckdb/DuckDBCatalogFactory.java @@ -28,6 +28,13 @@ import com.google.auto.service.AutoService; +import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.DECIMAL_TYPE_NARROWING; +import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.HANDLE_BLOB_AS_STRING; +import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.PASSWORD; +import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.SCHEMA; +import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.URL; +import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.USERNAME; + /** Factory for {@link DuckDBCatalog} */ @AutoService(Factory.class) public class DuckDBCatalogFactory implements CatalogFactory { @@ -50,6 +57,9 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig config) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return OptionRule.builder() + .required(URL) + .optional(USERNAME, PASSWORD, SCHEMA, DECIMAL_TYPE_NARROWING, HANDLE_BLOB_AS_STRING) + .build(); } } diff --git a/seatunnel-connectors-v2/connector-lance/src/main/java/org/apache/seatunnel/connectors/seatunnel/lance/catalog/LanceCatalogFactory.java b/seatunnel-connectors-v2/connector-lance/src/main/java/org/apache/seatunnel/connectors/seatunnel/lance/catalog/LanceCatalogFactory.java index 4235a141a4f3..7d26b4e8ef13 100644 --- a/seatunnel-connectors-v2/connector-lance/src/main/java/org/apache/seatunnel/connectors/seatunnel/lance/catalog/LanceCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-lance/src/main/java/org/apache/seatunnel/connectors/seatunnel/lance/catalog/LanceCatalogFactory.java @@ -35,6 +35,6 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { - return null; + return OptionRule.builder().build(); } } From e83aeb41af14b4c4afd2b1a879a824c627fd9fdd Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 22 Jun 2026 17:18:43 +0800 Subject: [PATCH 051/375] [Test][E2E] Stabilize schema evolution add-column assertions (#11154) --- .../jdbc/AbstractSchemaChangeBaseIT.java | 78 +++++++------------ .../src/test/resources/ddl/add_columns.sql | 7 +- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java index f95554f79d90..4448e6e90670 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java @@ -309,48 +309,8 @@ private void assertSchemaEvolution(String sourceTable, String sinkTable) { // case1 add columns with cdc data at same time sourceDatabase.setTemplateName("add_columns").createAndInitialize(); - await().atMost(120, TimeUnit.SECONDS) - .untilAsserted( - () -> - Assertions.assertIterableEquals( - querySource( - String.format( - SOURCE_QUERY_COLUMNS, - SOURCE_DATABASE, - sourceTable)), - querySink( - String.format( - schemaChangeCase.getSinkQueryColumns(), - schemaChangeCase.getSchemaName(), - sinkTable)))); - await().atMost(120, TimeUnit.SECONDS) - .untilAsserted( - () -> { - Assertions.assertIterableEquals( - querySource( - String.format(QUERY, SOURCE_DATABASE, sourceTable) - + " where id >= 128"), - querySink( - String.format( - QUERY, - schemaChangeCase.getSchemaName(), - sinkTable) - + " where id >= 128" - + ORDER_BY)); - - Assertions.assertIterableEquals( - querySource( - String.format( - PROJECTION_QUERY, - SOURCE_DATABASE, - sourceTable)), - querySink( - String.format( - PROJECTION_QUERY, - schemaChangeCase.getSchemaName(), - sinkTable) - + ORDER_BY)); - }); + waitForSinkColumnsCatchUp(sourceTable, sinkTable); + assertAddColumnsDataSynced(sourceTable, sinkTable); // case2 drop columns with cdc data at same time assertCaseByDdlName("drop_columns"); @@ -383,21 +343,37 @@ private void assertSchemaEvolutionForAddColumns(String sourceTable, String sinkT // case1 add columns with cdc data at same time sourceDatabase.setTemplateName("add_columns").createAndInitialize(); - given().pollDelay(Duration.ofSeconds(5)) - .await() - .atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) + waitForSinkColumnsCatchUp(sourceTable, sinkTable); + assertAddColumnsDataSynced(sourceTable, sinkTable); + } + + /** + * Schema-change sinks can publish the new rows before the sink table metadata is fully updated. + * Waiting for the column list first avoids racing the add-columns data assertions. + */ + private void waitForSinkColumnsCatchUp(String sourceTable, String sinkTable) { + await().atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( querySource( - String.format(QUERY, SOURCE_DATABASE, sourceTable)), + String.format( + SOURCE_QUERY_COLUMNS, + SOURCE_DATABASE, + sourceTable)), querySink( String.format( - QUERY, - schemaChangeCase.getSchemaName(), - sinkTable) - + ORDER_BY))); - await().atMost(120, TimeUnit.SECONDS) + schemaChangeCase.getSinkQueryColumns(), + schemaChangeCase.getSchemaName(), + sinkTable)))); + } + + /** + * Validates both the new add-columns rows and the projected full-table view once schema + * evolution has settled on the sink side. + */ + private void assertAddColumnsDataSynced(String sourceTable, String sinkTable) { + await().atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> { Assertions.assertIterableEquals( diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/ddl/add_columns.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/ddl/add_columns.sql index a56744de00ac..dc334b7b73d4 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/ddl/add_columns.sql +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/ddl/add_columns.sql @@ -34,6 +34,8 @@ update products set name = 'hawk9821' where id = 101; delete from products where id = 102; alter table products ADD COLUMN add_column1 varchar(64) not null default 'yy',ADD COLUMN add_column2 int not null default 1; +-- Let the source observe the DDL event before the first row uses the new columns. +DO SLEEP(5); update products set name = 'hawk9821' where id = 110; insert into products @@ -51,6 +53,8 @@ delete from products where id = 118; alter table products ADD COLUMN add_column3 float not null default 1.1; ## timestamp is not supported as a cross-database default values for DDL statements alter table products ADD COLUMN add_column4 timestamp; +-- The second add-columns batch also needs a short gap before the new-column DML arrives. +DO SLEEP(5); delete from products where id = 113; insert into products @@ -66,6 +70,8 @@ values (128,"scooter","Small 2-wheel scooter",3.14,'xx',1,1.1,'2023-02-02 09:09: update products set name = 'hawk9821' where id = 135; alter table products ADD COLUMN add_column6 varchar(64) not null default 'ff'; +-- Keep the final add-column DDL and the follow-up DML in separate CDC batches. +DO SLEEP(5); delete from products where id = 115; insert into products values (173,"scooter","Small 2-wheel scooter",3.14,'xx',1,1.1,'2023-02-02 09:09:09','tt'), @@ -80,4 +86,3 @@ values (173,"scooter","Small 2-wheel scooter",3.14,'xx',1,1.1,'2023-02-02 09:09: -- add column for irrelevant table ALTER TABLE products_on_hand ADD COLUMN add_column5 varchar(64) not null default 'yy'; - From 00c7871c1c06d50aa1c465543cd87edcf99f8d76 Mon Sep 17 00:00:00 2001 From: Jast Date: Mon, 22 Jun 2026 18:27:03 +0800 Subject: [PATCH 052/375] [Docs] Fix Chinese engine architecture page rendering (#11164) --- docs/zh/architecture/engine/engine-architecture.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zh/architecture/engine/engine-architecture.md b/docs/zh/architecture/engine/engine-architecture.md index 7122b9efaf54..383434f5ca49 100644 --- a/docs/zh/architecture/engine/engine-architecture.md +++ b/docs/zh/architecture/engine/engine-architecture.md @@ -77,7 +77,6 @@ flowchart TB style master fill:#081425,stroke:#5db8e2,stroke-width:1.5px,color:#f8fbff; style worker fill:#081425,stroke:#8d7cf6,stroke-width:1.5px,color:#f8fbff; ``` -``` ### 2.2 核心组件 From 152c4108825a9bf614db912033c305360fb1dd03 Mon Sep 17 00:00:00 2001 From: JeremyXin <739772893@qq.com> Date: Mon, 22 Jun 2026 18:36:13 +0800 Subject: [PATCH 053/375] [Fix][seatunnel-translation-base] Remove connector-specific test dependencies from translation base (#11075) --- .../seatunnel-translation-base/pom.xml | 13 - .../source/ParallelSourceTest.java | 343 +++++++++++------- 2 files changed, 221 insertions(+), 135 deletions(-) diff --git a/seatunnel-translation/seatunnel-translation-base/pom.xml b/seatunnel-translation/seatunnel-translation-base/pom.xml index 6c0cf4359e00..87636af0bb12 100644 --- a/seatunnel-translation/seatunnel-translation-base/pom.xml +++ b/seatunnel-translation/seatunnel-translation-base/pom.xml @@ -31,18 +31,5 @@ seatunnel-api ${project.version} - - org.apache.seatunnel - connector-file-base - ${project.version} - test - - - - org.apache.seatunnel - connector-doris - ${project.version} - test - diff --git a/seatunnel-translation/seatunnel-translation-base/src/test/java/org/apache/seatunnel/translation/source/ParallelSourceTest.java b/seatunnel-translation/seatunnel-translation-base/src/test/java/org/apache/seatunnel/translation/source/ParallelSourceTest.java index 1a2a3b98a7fd..71e28ad9a6ef 100644 --- a/seatunnel-translation/seatunnel-translation-base/src/test/java/org/apache/seatunnel/translation/source/ParallelSourceTest.java +++ b/seatunnel-translation/seatunnel-translation-base/src/test/java/org/apache/seatunnel/translation/source/ParallelSourceTest.java @@ -17,178 +17,277 @@ package org.apache.seatunnel.translation.source; -import org.apache.seatunnel.shade.com.google.common.collect.Maps; - -import org.apache.seatunnel.api.table.catalog.TablePath; -import org.apache.seatunnel.connectors.doris.config.DorisSourceConfig; -import org.apache.seatunnel.connectors.doris.rest.PartitionDefinition; -import org.apache.seatunnel.connectors.doris.rest.RestService; -import org.apache.seatunnel.connectors.doris.source.DorisSource; -import org.apache.seatunnel.connectors.doris.source.DorisSourceTable; -import org.apache.seatunnel.connectors.doris.source.reader.DorisSourceReader; -import org.apache.seatunnel.connectors.doris.source.split.DorisSourceSplit; -import org.apache.seatunnel.connectors.seatunnel.file.source.BaseFileSource; -import org.apache.seatunnel.connectors.seatunnel.file.source.split.FileSourceSplit; -import org.apache.seatunnel.connectors.seatunnel.file.source.split.FileSourceSplitEnumerator; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.MockedStatic; -import org.mockito.Mockito; - -import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import java.io.Serializable; import java.util.ArrayList; -import java.util.HashSet; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.mockito.ArgumentMatchers.any; - -@Slf4j -public class ParallelSourceTest { +class ParallelSourceTest { @Test - void fileParallelSourceSplitEnumeratorTest() throws Exception { - int fileSize = 15; + void shouldDistributeSplitsAcrossParallelReaders() throws Exception { + int splitCount = 15; int parallelism = 4; + List splits = + IntStream.range(0, splitCount) + .mapToObj(i -> new MockSplit("split-" + i)) + .collect(Collectors.toList()); + + MockSource source = new MockSource(splits); + Map> assignedSplits = new LinkedHashMap<>(); + + for (int subtaskId = 0; subtaskId < parallelism; subtaskId++) { + ParallelSource parallelSource = + new ParallelSource<>( + source, null, parallelism, "parallel-source-test", subtaskId); + parallelSource.open(); + parallelSource.splitEnumerator.run(); - List filePaths = new ArrayList<>(); - for (int i = 0; i < fileSize; i++) { - filePaths.add("file" + i + ".txt"); + assignedSplits.put(subtaskId, parallelSource.reader.snapshotState(0L)); + parallelSource.close(); } - BaseFileSource baseFileSource = Mockito.spy(BaseFileSource.class); - Set splitSet = new HashSet<>(); - for (int i = 0; i < parallelism; i++) { + int totalAssigned = assignedSplits.values().stream().mapToInt(List::size).sum(); + Assertions.assertEquals(splitCount, totalAssigned); - ParallelEnumeratorContext context = - Mockito.mock(ParallelEnumeratorContext.class); + List splitIds = + assignedSplits.values().stream() + .flatMap(List::stream) + .map(MockSplit::splitId) + .sorted() + .collect(Collectors.toList()); + Assertions.assertEquals( + splits.stream().map(MockSplit::splitId).sorted().collect(Collectors.toList()), + splitIds); - Mockito.when(context.currentParallelism()).thenReturn(parallelism); + for (int subtaskId = 0; subtaskId < parallelism; subtaskId++) { + Assertions.assertEquals( + expectedSplitCount(subtaskId, parallelism, splitCount), + assignedSplits.get(subtaskId).size()); + } + } - FileSourceSplitEnumerator fileSourceSplitEnumerator = - new FileSourceSplitEnumerator(context, filePaths); + @Test + void shouldRestoreReaderSplitsFromCheckpointState() throws Exception { + MockSource source = new MockSource(Collections.emptyList()); + Map> restoredState = new HashMap<>(); + restoredState.put(-1, Collections.singletonList(toBytes(Integer.valueOf(7)))); + restoredState.put(0, Collections.singletonList(toBytes(new MockSplit("restored-split-0")))); + + ParallelSource parallelSource = + new ParallelSource<>(source, restoredState, 4, "parallel-source-restore", 0); + parallelSource.open(); + parallelSource.splitEnumerator.run(); + + Assertions.assertEquals( + Collections.singletonList("restored-split-0"), + parallelSource.reader.snapshotState(0L).stream() + .map(MockSplit::splitId) + .sorted() + .collect(Collectors.toList())); + + MockSplitEnumerator restoredEnumerator = source.lastRestoredEnumerator; + Assertions.assertNotNull(restoredEnumerator); + Assertions.assertEquals(Integer.valueOf(7), restoredEnumerator.restoredState); + + parallelSource.close(); + } - Mockito.when(baseFileSource.createEnumerator(any())) - .thenReturn(fileSourceSplitEnumerator); + private static int expectedSplitCount(int subtaskId, int parallelism, int splitCount) { + int splitsPerReader = splitCount / parallelism; + int remainder = splitCount % parallelism; + return subtaskId < remainder ? splitsPerReader + 1 : splitsPerReader; + } - ParallelSource parallelSource = - new ParallelSource( - baseFileSource, null, parallelism, "parallel-source-test" + i, i); + private static byte[] toBytes(Serializable value) throws IOException { + return new org.apache.seatunnel.api.serialization.DefaultSerializer() + .serialize(value); + } - parallelSource.open(); - parallelSource.splitEnumerator.run(); + private static final class MockSource implements SeaTunnelSource { - ArgumentCaptor subtaskId = ArgumentCaptor.forClass(Integer.class); - ArgumentCaptor split = ArgumentCaptor.forClass(List.class); + private final List splits; + private MockSplitEnumerator lastRestoredEnumerator; - Mockito.verify(context, Mockito.times(parallelism)) - .assignSplit(subtaskId.capture(), split.capture()); + private MockSource(List splits) { + this.splits = splits; + } - List subTaskAllValues = subtaskId.getAllValues(); - List splitAllValues = split.getAllValues(); + @Override + public Boundedness getBoundedness() { + return Boundedness.BOUNDED; + } - Assertions.assertEquals(i, subTaskAllValues.get(i)); - Assertions.assertEquals( - allocateFiles(i, parallelism, fileSize), splitAllValues.get(i).size()); + @Override + public SourceReader createReader(SourceReader.Context readerContext) { + return new MockReader(); + } - splitSet.addAll(splitAllValues.get(i)); + @Override + public SourceSplitEnumerator createEnumerator( + SourceSplitEnumerator.Context enumeratorContext) { + return new MockSplitEnumerator(enumeratorContext, splits, null); } - // Check that there are no duplicate file assign - Assertions.assertEquals(splitSet.size(), fileSize); + @Override + public SourceSplitEnumerator restoreEnumerator( + SourceSplitEnumerator.Context enumeratorContext, + Integer checkpointState) { + lastRestoredEnumerator = + new MockSplitEnumerator( + enumeratorContext, Collections.emptyList(), checkpointState); + return lastRestoredEnumerator; + } + + @Override + public String getPluginName() { + return "mock-source"; + } } - @Test - public void dorisParallelSourceSplitEnumeratorTest() throws Exception { - int parallelism = 4; - int partitionNums = 30; + private static final class MockSplitEnumerator + implements SourceSplitEnumerator { + + private final SourceSplitEnumerator.Context context; + private final Map> pendingSplits = new HashMap<>(); + private final Integer restoredState; + private final AtomicInteger assignCount = new AtomicInteger(0); + + private MockSplitEnumerator( + SourceSplitEnumerator.Context context, + List splits, + Integer restoredState) { + this.context = context; + this.restoredState = restoredState; + addPendingSplits(splits); + } - DorisSourceConfig dorisSourceConfig = Mockito.mock(DorisSourceConfig.class); - DorisSourceTable dorisSourceTable = Mockito.mock(DorisSourceTable.class); + @Override + public void open() {} - Map dorisSourceTableMap = Maps.newHashMap(); - dorisSourceTableMap.put(new TablePath("default", null, "default_table"), dorisSourceTable); + @Override + public void run() { + assignSplits(context.registeredReaders()); + context.registeredReaders().forEach(context::signalNoMoreSplits); + } - DorisSource dorisSource = new DorisSource(dorisSourceConfig, dorisSourceTableMap); + private static int getSplitOwner(int assignCount, int parallelism) { + return assignCount % parallelism; + } - MockedStatic restServiceMockedStatic = Mockito.mockStatic(RestService.class); - restServiceMockedStatic - .when(() -> RestService.findPartitions(any(), any(), any())) - .thenReturn(buildPartitionDefinitions(partitionNums)); + private void assignSplits(Iterable readers) { + for (Integer reader : readers) { + List assigned = pendingSplits.remove(reader); + if (assigned != null && !assigned.isEmpty()) { + context.assignSplit(reader, assigned); + } + } + } - Set splitSet = new HashSet<>(); - for (int i = 0; i < parallelism; i++) { - ParallelSource parallelSource = - new ParallelSource( - dorisSource, null, parallelism, "parallel-doris-source" + i, i); - parallelSource.open(); + private void addPendingSplits(List splits) { + int parallelism = context.currentParallelism(); + List orderedSplits = + splits.stream() + .sorted((left, right) -> left.splitId().compareTo(right.splitId())) + .collect(Collectors.toList()); + for (MockSplit split : orderedSplits) { + int ownerReader = getSplitOwner(assignCount.getAndIncrement(), parallelism); + pendingSplits.computeIfAbsent(ownerReader, key -> new ArrayList<>()).add(split); + } + } - // execute file allocation process - parallelSource.splitEnumerator.run(); - List sourceSplits = - ((DorisSourceReader) parallelSource.reader).snapshotState(0); - log.info( - "parallel source{} splits => {}", - i + 1, - sourceSplits.stream() - .map(DorisSourceSplit::splitId) - .collect(Collectors.toList())); + @Override + public void close() {} - Assertions.assertEquals( - allocateFiles(i, parallelism, partitionNums), sourceSplits.size()); + @Override + public void addSplitsBack(List splits, int subtaskId) { + pendingSplits.computeIfAbsent(subtaskId, key -> new ArrayList<>()).addAll(splits); + } + + @Override + public int currentUnassignedSplitSize() { + return pendingSplits.values().stream().mapToInt(List::size).sum(); + } + + @Override + public void handleSplitRequest(int subtaskId) {} + + @Override + public void registerReader(int subtaskId) {} - // collect all splits - splitSet.addAll(sourceSplits); + @Override + public Integer snapshotState(long checkpointId) { + return restoredState; } - Assertions.assertEquals(splitSet.size(), partitionNums); + @Override + public void notifyCheckpointComplete(long checkpointId) {} + + @Override + public void notifyCheckpointAborted(long checkpointId) {} } - private List buildPartitionDefinitions(int partitionNUms) { + private static final class MockReader implements SourceReader { + + private final List splits = new ArrayList<>(); + + @Override + public void open() {} - List partitions = new ArrayList<>(); + @Override + public void close() {} - String beAddressPrefix = "doris-be-"; + @Override + public void pollNext(Collector output) {} - IntStream.range(0, partitionNUms) - .forEach( - i -> { - PartitionDefinition partitionDefinition = - new PartitionDefinition( - "default", - "default_table", - beAddressPrefix + i, - new HashSet<>(i), - "QUERY_PLAN"); + @Override + public List snapshotState(long checkpointId) { + return new ArrayList<>(splits); + } + + @Override + public void addSplits(List splits) { + this.splits.addAll(splits); + } - partitions.add(partitionDefinition); - }); + @Override + public void handleNoMoreSplits() {} - return partitions; + @Override + public void notifyCheckpointComplete(long checkpointId) {} + + @Override + public void notifyCheckpointAborted(long checkpointId) {} } - /** - * calculate the number of files assigned each time - * - * @param id id - * @param parallelism parallelism - * @param fileSize file size - * @return - */ - public int allocateFiles(int id, int parallelism, int fileSize) { - int filesPerIteration = fileSize / parallelism; - int remainder = fileSize % parallelism; - - if (id < remainder) { - return filesPerIteration + 1; - } else { - return filesPerIteration; + private static final class MockSplit implements SourceSplit { + + private final String splitId; + + private MockSplit(String splitId) { + this.splitId = splitId; + } + + @Override + public String splitId() { + return splitId; } } } From 60e026bd0a4669f7578d343e5ec6b35524095ac6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 22 Jun 2026 18:59:44 +0800 Subject: [PATCH 054/375] [Docs] Improve new user onboarding flow (#11152) Co-authored-by: David Zollo --- docs/en/connectors/formats/overview.md | 19 ++ docs/en/connectors/overview.md | 39 +++ docs/en/connectors/sink-overview.md | 21 ++ docs/en/connectors/source-overview.md | 21 ++ .../job-configuration-guide.md | 1 + docs/en/getting-started/locally/deployment.md | 7 + docs/en/getting-started/locally/overview.md | 29 +++ docs/en/getting-started/overview.md | 3 + .../getting-started/recipes/http-to-jdbc.md | 4 +- docs/en/getting-started/recipes/overview.md | 25 ++ docs/en/introduction/about.md | 101 ++++---- docs/en/introduction/concepts/config.md | 68 ++--- docs/en/introduction/how-it-works.md | 34 +++ docs/en/transforms/overview.md | 23 ++ docs/sidebars.js | 244 ++++++++---------- docs/zh/connectors/connector-faq.md | 65 ----- .../connector-isolated-dependency.md | 25 +- docs/zh/connectors/formats/overview.md | 19 ++ docs/zh/connectors/overview.md | 39 +++ docs/zh/connectors/sink-overview.md | 21 ++ docs/zh/connectors/source-overview.md | 21 ++ .../job-configuration-guide.md | 5 +- docs/zh/getting-started/locally/deployment.md | 7 + docs/zh/getting-started/locally/overview.md | 29 +++ docs/zh/getting-started/overview.md | 23 +- .../recipes/file-to-starrocks.md | 2 +- .../getting-started/recipes/http-to-jdbc.md | 4 +- docs/zh/getting-started/recipes/overview.md | 25 ++ docs/zh/introduction/about.md | 109 ++++---- docs/zh/introduction/concepts/config.md | 64 +++-- docs/zh/introduction/how-it-works.md | 52 +++- docs/zh/transforms/overview.md | 23 ++ 32 files changed, 743 insertions(+), 429 deletions(-) create mode 100644 docs/en/connectors/formats/overview.md create mode 100644 docs/en/connectors/overview.md create mode 100644 docs/en/connectors/sink-overview.md create mode 100644 docs/en/connectors/source-overview.md create mode 100644 docs/en/getting-started/locally/overview.md create mode 100644 docs/en/getting-started/recipes/overview.md create mode 100644 docs/en/transforms/overview.md delete mode 100644 docs/zh/connectors/connector-faq.md create mode 100644 docs/zh/connectors/formats/overview.md create mode 100644 docs/zh/connectors/overview.md create mode 100644 docs/zh/connectors/sink-overview.md create mode 100644 docs/zh/connectors/source-overview.md create mode 100644 docs/zh/getting-started/locally/overview.md create mode 100644 docs/zh/getting-started/recipes/overview.md create mode 100644 docs/zh/transforms/overview.md diff --git a/docs/en/connectors/formats/overview.md b/docs/en/connectors/formats/overview.md new file mode 100644 index 000000000000..3153b525ec31 --- /dev/null +++ b/docs/en/connectors/formats/overview.md @@ -0,0 +1,19 @@ +--- +slug: /connectors/formats +sidebar_position: 1 +--- + +# Formats + +Format docs explain how SeaTunnel maps between its internal row model and external data encodings such as Avro, Debezium JSON, or Protobuf. Read this section when the connector itself is not enough and you also need to control the payload shape. + +## When You Should Read Format Docs + +- your source or sink exchanges schema-based payloads +- your CDC pipeline depends on a specific envelope format +- you need to align external serialization with downstream consumers + +## Useful Next Pages + +- [Data Format Handling](../../architecture/data-format-handling.md) +- [Connector FAQ](../connector-faq.md) diff --git a/docs/en/connectors/overview.md b/docs/en/connectors/overview.md new file mode 100644 index 000000000000..c5cb7707a958 --- /dev/null +++ b/docs/en/connectors/overview.md @@ -0,0 +1,39 @@ +--- +slug: /connectors +--- + +# Connectors Overview + +This page is the shortest path for choosing the right SeaTunnel connector entry point. Start by answering three questions: where does data come from, where does it go, and do you need CDC or special formats. + +## Choose An Entry Point + +| What you need right now | Start here | +| --- | --- | +| Read data from an external system | [Source Connectors](./source-overview.md) | +| Write data into a target system | [Sink Connectors](./sink-overview.md) | +| Follow a real source-to-sink example | [Scenario Recipes](../getting-started/recipes/overview.md) | +| Understand shared connector parameters | [Source Common Options](./common-options/source-common-options.md) and [Sink Common Options](./common-options/sink-common-options.md) | +| Build a CDC pipeline | [CDC Production Cookbook](./cdc-production-cookbook.md) | +| Troubleshoot plugin installation or dependency conflicts | [Connector FAQ](./connector-faq.md) and [Connector Isolated Dependency Loading](./connector-isolated-dependency.md) | + +## Recommended Reading Order For New Users + +1. Run one local job first, then come back to choose real connectors. +2. Pick the source and sink before comparing transform or format details. +3. Check plugin installation and third-party driver requirements before copying connector examples. +4. Read CDC and recovery details only when your pipeline actually needs them. + +## What To Verify Before You Commit To A Connector + +- whether the connector supports your runtime engine +- whether extra drivers or plugin jars are required +- whether the connector supports batch, streaming, CDC, or exactly-once semantics +- whether the parameter names and examples match your SeaTunnel version + +## Useful Next Pages + +- [Job Configuration Guide](../getting-started/job-configuration-guide.md) +- [Scenario Recipes](../getting-started/recipes/overview.md) +- [Transforms Overview](../transforms) +- [Quick Start With SeaTunnel Engine](../getting-started/locally/quick-start-seatunnel-engine.md) diff --git a/docs/en/connectors/sink-overview.md b/docs/en/connectors/sink-overview.md new file mode 100644 index 000000000000..c3e28f45936a --- /dev/null +++ b/docs/en/connectors/sink-overview.md @@ -0,0 +1,21 @@ +--- +slug: /connectors/sink +sidebar_position: 1 +--- + +# Sink Connectors + +Use this page when your first question is "where should SeaTunnel write data?" Start by matching the target system, then verify write guarantees, table behavior, driver requirements, and any connector-specific delivery constraints. + +## Before You Pick A Sink + +- confirm the target system and table or object layout +- confirm whether you need at-least-once, exactly-once, or idempotent writes +- confirm whether extra drivers, SDKs, or cloud credentials are required +- confirm whether the connector supports the write mode your job expects + +## Useful Next Pages + +- [Sink Common Options](../common-options/sink-common-options.md) +- [Connector FAQ](../connector-faq.md) +- [Connector Isolated Dependency Loading](../connector-isolated-dependency.md) diff --git a/docs/en/connectors/source-overview.md b/docs/en/connectors/source-overview.md new file mode 100644 index 000000000000..7d4cb295d51d --- /dev/null +++ b/docs/en/connectors/source-overview.md @@ -0,0 +1,21 @@ +--- +slug: /connectors/source +sidebar_position: 1 +--- + +# Source Connectors + +Use this page when your first question is "where should SeaTunnel read data from?" Start by matching your external system, then verify plugin installation, driver dependencies, and whether the connector supports batch, streaming, or CDC behavior that your job needs. + +## Before You Pick A Source + +- confirm the external system you want to read from +- confirm whether you need snapshot, incremental, or CDC semantics +- confirm whether extra drivers, SDKs, or authentication setup are required +- confirm the connector examples match your current SeaTunnel version + +## Useful Next Pages + +- [Source Common Options](../common-options/source-common-options.md) +- [CDC Production Cookbook](../cdc-production-cookbook.md) +- [Connector FAQ](../connector-faq.md) diff --git a/docs/en/getting-started/job-configuration-guide.md b/docs/en/getting-started/job-configuration-guide.md index 6ed9f01e7492..a24f65a6e3e3 100644 --- a/docs/en/getting-started/job-configuration-guide.md +++ b/docs/en/getting-started/job-configuration-guide.md @@ -172,6 +172,7 @@ Before running a job, verify these points: ## Next Steps - Need a runnable first example: [Quick Start With SeaTunnel Engine](./locally/quick-start-seatunnel-engine.md) +- Need end-to-end examples built from real connectors: [Scenario Recipes](./recipes/overview.md) - Need connector parameters: [Source Connectors](../connectors/source) and [Sink Connectors](../connectors/sink) - Need transform capabilities: [Transforms](../transforms) - Need engine-level details: [Engine Overview](../engines/overview.md) diff --git a/docs/en/getting-started/locally/deployment.md b/docs/en/getting-started/locally/deployment.md index b0a91931b749..a0fd3ca14b35 100644 --- a/docs/en/getting-started/locally/deployment.md +++ b/docs/en/getting-started/locally/deployment.md @@ -83,6 +83,13 @@ When built from the source code, all the connector plugins and some necessary de Now you have downloaded the SeaTunnel binary package and the connector plugins. Next, you can choose different engine option to run synchronization tasks. +:::tip + +If you are new to SeaTunnel, start with [Quick Start With SeaTunnel Engine](quick-start-seatunnel-engine.md). +It is the default engine and usually the shortest path to a successful first run. + +::: + If you use Flink to run the synchronization task, there is no need to deploy the SeaTunnel Engine service cluster. You can refer to [Quick Start With Flink](quick-start-flink.md) to run your synchronization task. If you use Spark to run the synchronization task, there is no need to deploy the SeaTunnel Engine service cluster. You can refer to [Quick Start With Spark](quick-start-spark.md) to run your synchronization task. diff --git a/docs/en/getting-started/locally/overview.md b/docs/en/getting-started/locally/overview.md new file mode 100644 index 000000000000..668d78001d50 --- /dev/null +++ b/docs/en/getting-started/locally/overview.md @@ -0,0 +1,29 @@ +--- +slug: /getting-started/locally +--- + +# Local Quick Start + +Use this page when you want the shortest path to run SeaTunnel on your machine. For most new users, the default recommendation is still **SeaTunnel Engine (Zeta)** because it has the fastest feedback loop and the least setup friction. + +## Pick A Local Starting Path + +| Your situation | Start here | +| --- | --- | +| I need the default first-run path | [Quick Start With SeaTunnel Engine](./quick-start-seatunnel-engine.md) | +| I need installation and plugin setup first | [Deployment](./deployment.md) | +| My team already runs Flink | [Quick Start With Flink](./quick-start-flink.md) | +| My team already runs Spark | [Quick Start With Spark](./quick-start-spark.md) | + +## Recommended First-Run Order + +1. Read [Deployment](./deployment.md). +2. Install the sample plugins you need. +3. Run your first local job with [Run Your First Job](./run-your-first-job.md) or the [SeaTunnel Engine quick start](./quick-start-seatunnel-engine.md). +4. Move to the [Job Configuration Guide](../job-configuration-guide.md) after the sample succeeds. + +## When To Read The Other Paths + +- Use the Flink path only when you already operate Flink and want SeaTunnel to fit that environment. +- Use the Spark path only when your workload is already centered on Spark. +- Use this local section first, then move to cluster deployment after you have one working pipeline. diff --git a/docs/en/getting-started/overview.md b/docs/en/getting-started/overview.md index 487e1d4f4b23..ba242ea2aff1 100644 --- a/docs/en/getting-started/overview.md +++ b/docs/en/getting-started/overview.md @@ -50,6 +50,7 @@ If you want to validate your installation in the shortest path: - [Source Connectors](../connectors/source) - [Sink Connectors](../connectors/sink) - [Transforms](../transforms) +- [Scenario Recipes](./recipes/overview.md) ### Path C: I need to understand architecture first @@ -77,5 +78,7 @@ Once the sample job is working, the next step is usually one of these: Use these pages next: - [Job Configuration Guide](./job-configuration-guide.md) +- [Scenario Recipes](./recipes/overview.md) - [SeaTunnel Engine(Zeta) Deployment](../engines/zeta/deployment.md) - [REST API And Web UI](../engines/zeta/rest-api-and-web-ui.md) +- [Submitting Jobs to a Remote Zeta Cluster](../getting-started/submit-job-to-remote-zeta-cluster.md) diff --git a/docs/en/getting-started/recipes/http-to-jdbc.md b/docs/en/getting-started/recipes/http-to-jdbc.md index 45bfb660be07..9d239fa05391 100644 --- a/docs/en/getting-started/recipes/http-to-jdbc.md +++ b/docs/en/getting-started/recipes/http-to-jdbc.md @@ -1,9 +1,9 @@ --- sidebar_position: 4 -title: Http to JDBC +title: HTTP to JDBC --- -# Http to JDBC +# HTTP to JDBC Use this recipe when you want to pull structured data from an HTTP API and store the result in a relational database. diff --git a/docs/en/getting-started/recipes/overview.md b/docs/en/getting-started/recipes/overview.md new file mode 100644 index 000000000000..9bab5d5a7398 --- /dev/null +++ b/docs/en/getting-started/recipes/overview.md @@ -0,0 +1,25 @@ +--- +slug: /getting-started/recipes +--- + +# Scenario Recipes + +These recipes are best read after your first local job succeeds. Instead of reading every example in order, start with the pipeline shape that is closest to your real source and sink. + +## Choose A Recipe By Pipeline Goal + +| Goal | Start here | +| --- | --- | +| CDC from MySQL into an analytics database | [MySQL CDC to Doris](./mysql-cdc-to-doris.md) | +| JDBC extraction into object storage | [JDBC to S3](./jdbc-to-s3.md) | +| Streaming from Kafka into a table format | [Kafka to Iceberg](./kafka-to-iceberg.md) | +| HTTP ingestion into a relational target | [HTTP to JDBC](./http-to-jdbc.md) | +| File-based loading into an analytical system | [File to StarRocks](./file-to-starrocks.md) | +| Multi-table CDC orchestration | [Multi-Table CDC](./multi-table-cdc.md) | + +## How To Read A Recipe + +1. Confirm the source and sink combination matches your target pipeline. +2. Compare the `env`, `source`, `transform`, and `sink` sections with your own job. +3. Replace only one system at a time when adapting the sample. +4. If the sample depends on CDC, drivers, or extra plugins, verify those prerequisites before running it. diff --git a/docs/en/introduction/about.md b/docs/en/introduction/about.md index 99d51a5cf456..0a8b0f956469 100644 --- a/docs/en/introduction/about.md +++ b/docs/en/introduction/about.md @@ -5,11 +5,12 @@ [![Slack](../../images/seatunnel-slack.svg)](https://s.apache.org/seatunnel-slack) [![Twitter Follow](../../images/ASFSeaTunnel.svg)](https://x.com/ASFSeaTunnel) -SeaTunnel is a multimodal, ultra-high-performance, distributed data integration tool, capable of synchronizing vast amounts of data daily. It's trusted by numerous companies for its efficiency and stability. +SeaTunnel is a multimodal, high-performance, distributed data integration platform. +It helps teams move and synchronize data across databases, files, data lakes, and streaming systems with one unified job model. ## Start Here -If you are new to SeaTunnel, use this short reading path: +If this is your first time using SeaTunnel, follow this reading path:

-

-   -

-SeaTunnel enriches the CNCF CLOUD NATIVE Landscape. -

+- [Developer Setup](../developer/setup.md) if you want to build or debug SeaTunnel locally +- [Contribution Path](../developer/contribution-path.md) if you want to start contributing with the smallest reasonable scope +- [Contribute Plugin](../developer/contribute-plugin.md) if you want to contribute a connector or transform +- [GitHub Issues](https://github.com/apache/seatunnel/issues), [Slack](https://s.apache.org/seatunnel-slack), and the [dev mailing list](https://lists.apache.org/list.html?dev@seatunnel.apache.org) if you need community help -## Learn more +## Who Uses SeaTunnel -You can see [Run your first job](../getting-started/locally/run-your-first-job.md) for the next steps. +SeaTunnel has lots of users. You can find more information about them in [Users](https://seatunnel.apache.org/user). diff --git a/docs/en/introduction/concepts/config.md b/docs/en/introduction/concepts/config.md index 19855ffc9cc0..9a7e0891e4fe 100644 --- a/docs/en/introduction/concepts/config.md +++ b/docs/en/introduction/concepts/config.md @@ -1,19 +1,14 @@ # Intro To Config File -In SeaTunnel, the most important thing is the config file, through which users can customize their own data -synchronization requirements to maximize the potential of SeaTunnel. So next, I will introduce you how to -configure the config file. +If you are writing your first real SeaTunnel job, this page is the fastest way to understand the four blocks that appear in almost every config: `env`, `source`, `transform`, and `sink`. -The main format of the config file is `hocon`, for more details you can refer to [HOCON-GUIDE](https://github.com/lightbend/config/blob/main/HOCON.md), -BTW, we also support the `json` format, but you should keep in mind that the name of the config file should end with `.json`. +SeaTunnel supports `hocon`, `json`, and `SQL` config formats. HOCON is the most common format in quick starts and production examples. For SQL format, see [SQL configuration](../configuration/sql-config.md). -We also support the `SQL` format, please refer to [SQL configuration](../configuration/sql-config.md) for more details. +If you want the shortest first-run path before reading this page, start with [Getting Started Overview](../../getting-started/overview.md) and [Quick Start With SeaTunnel Engine](../../getting-started/locally/quick-start-seatunnel-engine.md). ## Example -Before you read on, you can find config file -examples [Here](https://github.com/apache/seatunnel/tree/dev/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources) from the binary package's -config directory. +Before you read on, you can find example configs [here](https://github.com/apache/seatunnel/tree/dev/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources) and in the binary package's `config` directory. ## Config File Structure @@ -67,33 +62,25 @@ sink { } ``` -As you can see, the config file contains several sections: env, source, transform, sink. Different modules -have different functions. After you understand these modules, you will see how SeaTunnel works. +Most SeaTunnel jobs follow this structure: `env`, `source`, `transform`, and `sink`. Once you understand these four sections, it becomes much easier to read quick starts and connector examples. ### env -Used to add some engine optional parameters, no matter which engine (Zeta, Spark or Flink), the corresponding -optional parameters should be filled in here. +Use `env` for job-level and engine-level settings such as `job.mode`, `parallelism`, checkpoint options, and engine-specific parameters. -Note that we have separated the parameters by engine, and for the common parameters, we can configure them as before. -For flink and spark engine, the specific configuration rules of their parameters can be referred to [JobEnvConfig](../configuration/JobEnvConfig.md). +Common parameters are shared across engines. Engine-specific parameters are separated by prefix. For Flink and Spark, see [JobEnvConfig](../configuration/JobEnvConfig.md). ### source -Source is used to define where SeaTunnel needs to fetch data, and use the fetched data for the next step. -Multiple sources can be defined at the same time. The supported source can be found -in [Source of SeaTunnel](../../connectors/source). Each source has its own specific parameters to define how to -fetch data, and SeaTunnel also extracts the parameters that each source will use, such as -the `plugin_output` parameter, which is used to specify the name of the data generated by the current -source, which is convenient for follow-up used by other modules. +`source` defines where SeaTunnel reads data from. You can declare multiple sources in one job. Each connector has its own parameters, plus common wiring fields such as `plugin_output`, which names the dataset produced by that source. + +See the full list in [Source Connectors](../../connectors/source). ### transform -When we have the data source, we may need to further process the data, so we have the transform module. Of -course, this uses the word 'may', which means that we can also directly treat the transform as non-existent, -directly from source to sink. Like below. +`transform` is optional. Use it when you need field mapping, filtering, type conversion, SQL processing, or other intermediate shaping between source and sink. If you do not need that layer, a job can go directly from source to sink, like this: ```hocon env { @@ -127,28 +114,22 @@ sink { } ``` -Like source, transform has specific parameters that belong to each module. The supported transform can be found -in [Transform V2 of SeaTunnel](../../transforms) +Like source connectors, each transform has its own parameters. See [Transforms](../../transforms). ### sink -Our purpose with SeaTunnel is to synchronize data from one place to another, so it is critical to define how -and where data is written. With the sink module provided by SeaTunnel, you can complete this operation quickly -and efficiently. Sink and source are very similar, but the difference is reading and writing. So please check out -[Supported Sinks](../../connectors/sink). +`sink` defines where the processed data is written. Sink connectors are similar to source connectors, but they focus on write behavior, destination schema, commit mode, and delivery guarantees. + +See [Supported Sinks](../../connectors/sink). + +### How `plugin_output` And `plugin_input` Work + +When a job contains multiple sources, transforms, or sinks, SeaTunnel needs a way to describe which dataset flows into which next step. That wiring is done by `plugin_output` and `plugin_input`. -### Other Information +- `plugin_output` names the dataset produced by the current source or transform +- `plugin_input` tells a transform or sink which upstream dataset to consume -You will find that when multiple sources and multiple sinks are defined, which data is read by each sink, and -which is the data read by each transform? We introduce two key configurations called `plugin_output` and -`plugin_input`. Each source module will be configured with a `plugin_output` to indicate the name of the -data source generated by the data source, and other transform and sink modules can use `plugin_input` to -refer to the corresponding data source name, indicating that I want to read the data for processing. Then -transform, as an intermediate processing module, can use both `plugin_output` and `plugin_input` -configurations at the same time. But you will find that in the above example config, not every module is -configured with these two parameters, because in SeaTunnel, there is a default convention, if these two -parameters are not configured, then the generated data from the last module of the previous node will be used. -This is much more convenient when there is only one source. +In simple one-source jobs, you can often omit them because SeaTunnel uses a default convention and passes the previous module's output forward automatically. ## Multi-line Support @@ -337,5 +318,6 @@ sink { ## What's More -- Start write your own config file now, choose the [connector](../../connectors/source) you want to use, and configure the parameters according to the connector's documentation. -- If you want to know the details of the format configuration, please see [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md). +- Start writing your own config file now, choose the [connector](../../connectors/source) you want to use, and configure it according to the connector documentation. +- See [JobEnvConfig](../configuration/JobEnvConfig.md) when you need engine-specific settings. +- See [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md) if you want the full syntax details. diff --git a/docs/en/introduction/how-it-works.md b/docs/en/introduction/how-it-works.md index e7d19575022f..a855a0a9c8cf 100644 --- a/docs/en/introduction/how-it-works.md +++ b/docs/en/introduction/how-it-works.md @@ -4,6 +4,18 @@ sidebar_position: 2 # How it works +## What New Users Should Know First + +You do not need to understand every internal module before running SeaTunnel. +For most first-time users, the practical order is: + +1. run one job locally +2. learn the config structure +3. choose the right connectors and engine +4. come back here when you want to understand the runtime model better + +SeaTunnel is easiest to understand as a config-driven pipeline that runs on a chosen execution engine. + ## Overview SeaTunnel is a distributed multimodal data integration tool with a pluggable architecture. It decouples the connector layer from the execution engine, allowing the same connectors to run on different engines. @@ -35,6 +47,28 @@ flowchart TD linkStyle default stroke:#5db8e2,stroke-width:2px; ``` +## The Four Building Blocks + +### 1. Job Configuration + +Your config file describes what to read, how to transform it, where to write it, and which engine settings should be used. + +### 2. SeaTunnel Core + +SeaTunnel parses the config, builds an execution plan, loads plugins, and coordinates submission to the selected engine. + +### 3. Source -> Transform -> Sink + +This is the data path most users should remember first: + +- **Source** reads from external systems +- **Transform** optionally reshapes or filters the data +- **Sink** writes the result to the target system + +### 4. Execution Engine + +The engine decides where the job runs. Most new users should start with [SeaTunnel Engine (Zeta)](../engines/zeta/about.md), then move to Flink or Spark only when their environment already depends on those platforms. + ## Recommended Reading Path If you are building your first system-level understanding, read in this order: diff --git a/docs/en/transforms/overview.md b/docs/en/transforms/overview.md new file mode 100644 index 000000000000..c4bcb6243016 --- /dev/null +++ b/docs/en/transforms/overview.md @@ -0,0 +1,23 @@ +--- +slug: /transforms +--- + +# Transforms Overview + +Transforms sit between source and sink. They are used for field mapping, filtering, SQL processing, table routing, and other mid-pipeline operations. New users do not need to read every transform first; come here after you already know what you want to read and where you want to write it. + +## Pick A Starting Point + +| Goal | Start here | +| --- | --- | +| Understand how transforms connect datasets | [Transform Common Options](./common-options/common-options.md) | +| Filter rows or trim fields | [Filter](./filter.md) and [Field Mapper](./field-mapper.md) | +| Use SQL-style expressions | [SQL](./sql.md) and [SQL Functions](./sql-functions.md) | +| Rename or reshape fields | [Field Rename](./field-rename.md) and [Split](./split.md) | +| Work with multiple tables | [Transform Multi Table](./transform-multi-table.md) and [Table Merge](./table-merge.md) | + +## Recommended Order For New Users + +1. Read the common options page first so `plugin_input` and `plugin_output` are clear. +2. Choose the simplest transform that matches your goal before moving to SQL or multi-table orchestration. +3. Add transforms one step at a time and keep the pipeline readable while you validate the job. diff --git a/docs/sidebars.js b/docs/sidebars.js index 863de991a0eb..eda13f8e2907 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -30,74 +30,9 @@ const sidebars = { "type": "category", "label": "Concepts", "items": [ - "introduction/concepts/config", "introduction/concepts/connector-v2-features", "introduction/concepts/schema-feature" ] - }, - { - "type": "category", - "label": "Configuration", - "items": [ - "introduction/configuration/JobEnvConfig", - "introduction/configuration/sql-config", - "introduction/configuration/config-encryption-decryption", - "introduction/configuration/metalake", - "introduction/configuration/sink-options-placeholders", - "introduction/configuration/schema-evolution", - "introduction/configuration/speed-limit" - ] - } - ] - }, - { - "type": "category", - "label": "Architecture", - "items": [ - "architecture/overview", - "architecture/design-philosophy", - "architecture/configuration-and-option-system", - "architecture/core-api-design", - "architecture/transform-plugin-system", - "architecture/cdc-pipeline-architecture", - "architecture/data-format-handling", - "architecture/table-schema-and-type-system", - "architecture/plugin-discovery-and-class-loading", - { - "type": "category", - "label": "API Design", - "items": [ - "architecture/api-design/source-architecture", - "architecture/api-design/sink-architecture", - "architecture/api-design/catalog-table", - "architecture/api-design/translation-layer", - "architecture/api-design/flink-translation-layer", - "architecture/api-design/spark-translation-layer" - ] - }, - { - "type": "category", - "label": "Engine", - "items": [ - "architecture/engine/engine-architecture", - "architecture/engine/dag-execution", - "architecture/engine/resource-management" - ] - }, - { - "type": "category", - "label": "Fault Tolerance", - "items": [ - "architecture/fault-tolerance/checkpoint-mechanism", - "architecture/fault-tolerance/exactly-once" - ] - }, - { - "type": "category", - "label": "Features", - "items": [ - "architecture/features/multi-table" - ] } ] }, @@ -110,12 +45,8 @@ const sidebars = { "type": "category", "label": "Locally", "link": { - "type": "generated-index", - "title": "Local Getting Started", - "description": "Use SeaTunnel locally, run your first job, and validate common pipelines before moving to larger environments.", - "slug": "/getting-started/locally", - "keywords": ["getting-started", "local"], - "image": "/img/favicon.ico" + "type": "doc", + "id": "getting-started/locally/overview" }, "items": [ "getting-started/locally/run-your-first-job", @@ -125,28 +56,6 @@ const sidebars = { "getting-started/locally/quick-start-spark" ] }, - { - "type": "category", - "label": "Recipes", - "link": { - "type": "generated-index", - "title": "Scenario Recipes", - "description": "Practical SeaTunnel recipes for common source-to-sink pipelines.", - "slug": "/getting-started/recipes", - "keywords": ["recipes", "examples"], - "image": "/img/favicon.ico" - }, - "items": [ - "getting-started/recipes/mysql-cdc-to-doris", - "getting-started/recipes/jdbc-to-s3", - "getting-started/recipes/kafka-to-iceberg", - "getting-started/recipes/http-to-jdbc", - "getting-started/recipes/file-to-starrocks", - "getting-started/recipes/multi-table-cdc" - ] - }, - "getting-started/job-configuration-guide", - "getting-started/submit-job-to-remote-zeta-cluster", { "type": "category", "label": "Docker", @@ -164,23 +73,35 @@ const sidebars = { } ] }, + { + "type": "category", + "label": "Configuration", + "items": [ + "introduction/concepts/config", + "getting-started/job-configuration-guide", + "introduction/configuration/JobEnvConfig", + "introduction/configuration/sql-config", + "introduction/configuration/config-encryption-decryption", + "introduction/configuration/metalake", + "introduction/configuration/sink-options-placeholders", + "introduction/configuration/schema-evolution", + "introduction/configuration/speed-limit" + ] + }, { "type": "category", "label": "Connectors", + "link": { + "type": "doc", + "id": "connectors/overview" + }, "items": [ - "connectors/connector-isolated-dependency", - "connectors/connector-faq", - "connectors/cdc-production-cookbook", { "type": "category", "label": "Source", "link": { - "type": "generated-index", - "title": "Source Connectors", - "description": "List all source connectors supported by Apache SeaTunnel.", - "slug": "/connectors/source", - "keywords": ["source"], - "image": "/img/favicon.ico" + "type": "doc", + "id": "connectors/source-overview" }, "items": [ { @@ -193,12 +114,8 @@ const sidebars = { "type": "category", "label": "Sink", "link": { - "type": "generated-index", - "title": "Sink Connectors", - "description": "List all sink connectors supported by Apache SeaTunnel.", - "slug": "/connectors/sink", - "keywords": ["sink"], - "image": "/img/favicon.ico" + "type": "doc", + "id": "connectors/sink-overview" }, "items": [ { @@ -207,16 +124,20 @@ const sidebars = { } ] }, + { + "type": "category", + "label": "Common Options", + "items": [ + "connectors/common-options/source-common-options", + "connectors/common-options/sink-common-options" + ] + }, { "type": "category", "label": "Formats", "link": { - "type": "generated-index", - "title": "Formats", - "description": "List some special formats supported by Apache SeaTunnel.", - "slug": "/connectors/formats", - "keywords": ["formats"], - "image": "/img/favicon.ico" + "type": "doc", + "id": "connectors/formats/overview" }, "items": [ { @@ -225,44 +146,33 @@ const sidebars = { } ] }, + "connectors/cdc-production-cookbook", { "type": "category", - "label": "Common Options", - "items": [ - "connectors/common-options/source-common-options", - "connectors/common-options/sink-common-options" - ] - }, - { - "type": "category", - "label": "Changelog", + "label": "Scenario Recipes", "link": { - "type": "generated-index", - "title": "Connector Changelog", - "description": "Changelog for all connectors supported by Apache SeaTunnel.", - "slug": "/connectors/changelog", - "keywords": ["changelog"], - "image": "/img/favicon.ico" + "type": "doc", + "id": "getting-started/recipes/overview" }, "items": [ - { - "type": "autogenerated", - "dirName": "connectors/changelog" - } + "getting-started/recipes/mysql-cdc-to-doris", + "getting-started/recipes/jdbc-to-s3", + "getting-started/recipes/kafka-to-iceberg", + "getting-started/recipes/http-to-jdbc", + "getting-started/recipes/file-to-starrocks", + "getting-started/recipes/multi-table-cdc" ] - } + }, + "connectors/connector-faq", + "connectors/connector-isolated-dependency" ] }, { "type": "category", "label": "Transforms", "link": { - "type": "generated-index", - "title": "Transforms", - "description": "List all transforms supported by Apache SeaTunnel.", - "slug": "/transforms", - "keywords": ["transforms"], - "image": "/img/favicon.ico" + "type": "doc", + "id": "transforms/overview" }, "items": [ { @@ -322,12 +232,13 @@ const sidebars = { "engines/zeta/separated-cluster-deployment" ] }, + "engines/zeta/rest-api-and-web-ui", + "getting-started/submit-job-to-remote-zeta-cluster", "engines/zeta/checkpoint-storage", "engines/zeta/state-storage-and-recovery", "engines/zeta/engine-jar-storage-mode", "engines/zeta/tcp", "engines/zeta/resource-isolation", - "engines/zeta/rest-api-and-web-ui", { "type": "category", "label": "REST API", @@ -359,6 +270,57 @@ const sidebars = { "engines/spark" ] }, + { + "type": "category", + "label": "Architecture", + "items": [ + "architecture/overview", + "architecture/design-philosophy", + "architecture/configuration-and-option-system", + "architecture/core-api-design", + "architecture/transform-plugin-system", + "architecture/cdc-pipeline-architecture", + "architecture/data-format-handling", + "architecture/table-schema-and-type-system", + "architecture/plugin-discovery-and-class-loading", + { + "type": "category", + "label": "API Design", + "items": [ + "architecture/api-design/source-architecture", + "architecture/api-design/sink-architecture", + "architecture/api-design/catalog-table", + "architecture/api-design/translation-layer", + "architecture/api-design/flink-translation-layer", + "architecture/api-design/spark-translation-layer" + ] + }, + { + "type": "category", + "label": "Engine", + "items": [ + "architecture/engine/engine-architecture", + "architecture/engine/dag-execution", + "architecture/engine/resource-management" + ] + }, + { + "type": "category", + "label": "Fault Tolerance", + "items": [ + "architecture/fault-tolerance/checkpoint-mechanism", + "architecture/fault-tolerance/exactly-once" + ] + }, + { + "type": "category", + "label": "Features", + "items": [ + "architecture/features/multi-table" + ] + } + ] + }, { "type": "category", "label": "Tools", diff --git a/docs/zh/connectors/connector-faq.md b/docs/zh/connectors/connector-faq.md deleted file mode 100644 index 53f90bf07c32..000000000000 --- a/docs/zh/connectors/connector-faq.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -sidebar_position: 10 ---- - -# Connector 常见问题 - -本页面是按 Connector 类别组织的常见问题索引，每个条目均链接到对应 Connector 文档页面的 FAQ 小节。 - -这些 FAQ 的定位是“快速导航”，不是另一套独立事实源。涉及精确配置项名称、默认值和完整示例时，应以 -各 Connector 页面中的 option 表和对应详细章节为准。 - -关于 SeaTunnel 通用问题（引擎部署、变量替换、调度等），请查阅[通用 FAQ](../faq.md)。 - ---- - -## CDC 类 Connector - -CDC（变更数据捕获）Connector 从数据库事务日志中实时读取 INSERT / UPDATE / DELETE 变更事件。 - -| Connector | 常见问题主题 | -|---|---| -| [MySQL CDC](./source/MySQL-CDC.md#常见问题) | 所需权限、binlog 配置、从库支持、无主键表、快照阶段、DDL 传播、server-id 冲突、快照性能、时区/字符集 | -| [PostgreSQL CDC](./source/PostgreSQL-CDC.md#常见问题) | 所需权限、逻辑解码插件、备库限制、无主键表、复制槽管理、复制滞后 | -| [Oracle CDC](./source/Oracle-CDC.md#常见问题) | LogMiner 权限、附加日志、CDB/PDB 多租户、无主键表、LogMiner 性能、支持的 Oracle 版本 | - ---- - -## 消息队列类 Connector - -| Connector | 常见问题主题 | -|---|---| -| [Kafka Source](./source/Kafka.md#常见问题) | `start_mode` 各取值对比、按消息 key 过滤、支持的格式、SASL/Kerberos 认证、消费组 offset 提交 | -| [Kafka Sink](./sink/Kafka.md#常见问题) | 自动创建 topic（Broker 端行为说明）、`partition_key_fields` 为空时的行为、精确一次、SASL/Kerberos 认证、支持的格式 | - ---- - -## Sink 类 Connector - -### OLAP / 分析型存储 - -| Connector | 常见问题主题 | -|---|---| -| [Doris Sink](./sink/Doris.md#常见问题) | 自动建表、2PC 精确一次、"Label already exists" 报错、DELETE 传播、列名大小写、Stream Load 格式 | -| [StarRocks Sink](./sink/StarRocks.md#常见问题) | 自动建表、Upsert/DELETE 支持、`labelPrefix` 用法、列名大小写、`nodeUrls` 与 `base-url` 的区别 | -| [ClickHouse Sink](./sink/Clickhouse.md#常见问题) | 自动建表、批量写入调优、支持的数据类型、"Table doesn't exist" 报错 | - -### 关系型数据库 - -| Connector | 常见问题主题 | -|---|---| -| [JDBC Sink](./sink/Jdbc.md#常见问题) | 自动建表、XA 精确一次、Upsert/主键配置、多表写入、JDBC 驱动未找到 | - -### 数据湖 / 文件系统 - -| Connector | 常见问题主题 | -|---|---| -| [Hive Sink](./sink/Hive.md#常见问题) | 支持的文件格式、分区表、Kerberos 认证、小文件问题、Schema 演变 | - ---- - -## 找答案的建议 - -1. **Connector 特定问题** → 直接进入对应 Connector 页面，滚动到 **FAQ** 小节。 -2. **跨 Connector 通用问题**（如"SeaTunnel 是否支持 CDC？""什么是 `schema_save_mode`？"）→ 查阅[通用 FAQ](../faq.md)。 -3. **仍未解决？** → 搜索 [GitHub Issues](https://github.com/apache/seatunnel/issues) 或通过[邮件列表](https://lists.apache.org/list.html?dev@seatunnel.apache.org)联系社区。 diff --git a/docs/zh/connectors/connector-isolated-dependency.md b/docs/zh/connectors/connector-isolated-dependency.md index 0ef00316158a..951fdd5a7734 100644 --- a/docs/zh/connectors/connector-isolated-dependency.md +++ b/docs/zh/connectors/connector-isolated-dependency.md @@ -1,25 +1,25 @@ -# Connector 依赖隔离加载机制 +# 连接器依赖隔离加载机制 -SeaTunnel 提供了针对每个 connector 的依赖隔离加载机制，方便用户管理不同连接器单独的依赖，同时避免依赖冲突并提升系统的可扩展性。 -当加载 connector 时，SeaTunnel 会从 `${SEATUNNEL_HOME}` 下的 `plugins/connector-xxx` 目录中，查找并加载该 connector 独立的依赖 jar。这种方式确保了不同 connector 所需的依赖不会相互影响，便于在复杂环境下管理大量 connector。 +SeaTunnel 提供了针对每个连接器的依赖隔离加载机制，方便用户管理不同连接器各自的依赖，同时避免依赖冲突并提升系统可扩展性。 +当加载连接器时，SeaTunnel 会从 `${SEATUNNEL_HOME}` 下的 `plugins/connector-xxx` 目录中，查找并加载该连接器独立的依赖 jar。这种方式确保不同连接器所需的依赖不会相互影响，便于在复杂环境下管理大量连接器。 ## 实现原理 -每个 connector 需要将自己的依赖 jar 放置在 `${SEATUNNEL_HOME}/plugins/connector-xxx` 目录下的独立子目录中（需要手动创建）。 -子目录名称由 `plugin-mapping` 文件中的 value 值指定。SeaTunnel 启动并加载 connector 时，只会加载对应目录下的 jar，从而实现依赖的隔离。 +每个连接器都需要将自己的依赖 jar 放置在 `${SEATUNNEL_HOME}/plugins/connector-xxx` 目录下的独立子目录中（需要手动创建）。 +子目录名称由 `plugin-mapping` 文件中的 value 值指定。SeaTunnel 启动并加载连接器时，只会加载对应目录下的 jar，从而实现依赖隔离。 -目前，Zeta 引擎会保证同一个任务不同connector的jar分开加载。其他两个引擎仍然会将所有 connector 的依赖 jar 一起加载，同一个任务放置了不同版本的jar在Spark/Flink环境可能导致依赖冲突。 +目前，Zeta 引擎会保证同一个任务中的不同连接器 jar 分开加载。其他两个引擎仍然会将所有连接器依赖 jar 一起加载，同一个任务如果放置了不同版本的 jar，在 Spark/Flink 环境中可能导致依赖冲突。 ## 目录结构示例 -- 通过`${SEATUNNEL_HOME}/connectors/plugin-mapping.properties` 获取每个connector对应的文件夹目录命名。 +- 通过 `${SEATUNNEL_HOME}/connectors/plugin-mapping.properties` 获取每个连接器对应的目录名称。 以AmazonDynamodb为例，假设在 `plugin-mapping` 文件中有以下配置： ``` seatunnel.source.AmazonDynamodb = connector-amazondynamodb ``` -则对应的connector依赖目录就是value值 `connector-amazondynamodb`。 +则对应的连接器依赖目录就是 value 值 `connector-amazondynamodb`。 最终的目录结构如下所示： @@ -36,15 +36,14 @@ SEATUNNEL_HOME/ ## 限制说明 -- 在Zeta引擎中，请确保所有节点的 `${SEATUNNEL_HOME}/plugins/` 目录结构一致。都需要包含相同的子目录和依赖 jar。 -- 任何没有以`connector-`开头的目录或者jar都将被当作通用依赖目录处理，所有引擎和connector都会加载此类jar。 -- 在Zeta引擎中，可以通过将通用的jar放到 `${SEATUNNEL_HOME}/lib/` 目录下来实现所有 connector 的共享依赖。 +- 在 Zeta 引擎中，请确保所有节点的 `${SEATUNNEL_HOME}/plugins/` 目录结构一致，都包含相同的子目录和依赖 jar。 +- 任何没有以 `connector-` 开头的目录或 jar 都会被当作通用依赖目录处理，所有引擎和连接器都会加载此类 jar。 +- 在 Zeta 引擎中，可以通过将通用 jar 放到 `${SEATUNNEL_HOME}/lib/` 目录下，实现所有连接器共享依赖。 ## 验证 -- 通过追踪任务日志，确认每个 connector 只加载了其独立的依赖 jar。 +- 通过追踪任务日志，确认每个连接器只加载了自己独立的依赖 jar。 ```log 2025-08-13T17:55:48.7732601Z [] 2025-08-13 17:55:47,270 INFO org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery - find connector jar and dependency for PluginIdentifier{engineType='seatunnel', pluginType='source', pluginName='Jdbc'}: [file:/tmp/seatunnel/plugins/Jdbc/lib/vertica-jdbc-12.0.3-0.jar, file:/tmp/seatunnel/connectors/connector-jdbc-3.0.0-SNAPSHOT-2.12.15.jar] ``` - diff --git a/docs/zh/connectors/formats/overview.md b/docs/zh/connectors/formats/overview.md new file mode 100644 index 000000000000..b7148995c636 --- /dev/null +++ b/docs/zh/connectors/formats/overview.md @@ -0,0 +1,19 @@ +--- +slug: /connectors/formats +sidebar_position: 1 +--- + +# 数据格式 + +格式文档关注的是 SeaTunnel 内部数据模型与外部编码之间如何对应，例如 Avro、Debezium JSON、Protobuf 等。当连接器本身还不够，任务还需要你明确控制消息体、Schema 或 CDC 包装格式时，就进入这一节。 + +## 什么时候需要看格式文档 + +- source 或 sink 读写的是带 Schema 的外部载荷 +- CDC 链路依赖特定的 envelope 格式 +- 需要让外部序列化格式与下游消费方保持一致 + +## 常用下一步 + +- [数据格式处理](../../architecture/data-format-handling.md) +- [连接器常见问题](../connector-faq.md) diff --git a/docs/zh/connectors/overview.md b/docs/zh/connectors/overview.md new file mode 100644 index 000000000000..956e6968cf68 --- /dev/null +++ b/docs/zh/connectors/overview.md @@ -0,0 +1,39 @@ +--- +slug: /connectors +--- + +# 数据连接器总览 + +这一页不是参数大全，而是帮助第一次选择 SeaTunnel 连接器的用户先找到正确入口。建议先确认“从哪里读”“写到哪里”“是否需要 CDC 或特殊格式”，再进入具体连接器参数页。 + +## 先按任务目标选入口 + +| 你现在要做什么 | 先看这里 | +| --- | --- | +| 从外部系统读取数据 | [数据来源连接器](./source-overview.md) | +| 把数据写入目标系统 | [数据写入连接器](./sink-overview.md) | +| 先找一条接近真实业务的链路示例 | [场景示例](../getting-started/recipes/overview.md) | +| 先理解连接器共有参数 | [来源端通用参数](./common-options/source-common-options.md) 和 [写入端通用参数](./common-options/sink-common-options.md) | +| 构建 CDC 链路 | [CDC 生产实战手册](./cdc-production-cookbook.md) | +| 排查插件安装或依赖冲突 | [连接器常见问题](./connector-faq.md) 和 [连接器依赖隔离加载机制](./connector-isolated-dependency.md) | + +## 新用户推荐顺序 + +1. 先跑通一个本地任务，再回来选择真实连接器。 +2. 先确定来源端和写入端，再判断是否需要额外的数据转换或格式处理。 +3. 在复制示例参数前，先确认插件安装和第三方驱动是否齐全。 +4. 只有在任务真的涉及多表、增量或恢复语义时，再深入 CDC 和容错细节。 + +## 选择连接器时重点关注什么 + +- 是否支持你正在使用的执行引擎 +- 是否需要额外驱动或插件安装 +- 是否支持批处理、流处理、CDC、精确一次等能力 +- 参数名、默认值和示例是否与你的 SeaTunnel 版本一致 + +## 常用下一步 + +- [作业配置指南](../getting-started/job-configuration-guide.md) +- [场景示例](../getting-started/recipes/overview.md) +- [数据转换总览](../transforms) +- [SeaTunnel 引擎快速开始](../getting-started/locally/quick-start-seatunnel-engine.md) diff --git a/docs/zh/connectors/sink-overview.md b/docs/zh/connectors/sink-overview.md new file mode 100644 index 000000000000..c7dc48c2e1a3 --- /dev/null +++ b/docs/zh/connectors/sink-overview.md @@ -0,0 +1,21 @@ +--- +slug: /connectors/sink +sidebar_position: 1 +--- + +# 数据写入连接器 + +当你的第一个问题是“SeaTunnel 最终要把数据写到哪里”时，就先看这一页。更稳妥的顺序是先匹配目标系统，再确认写入保证、表结构行为、驱动依赖，以及连接器本身的投递约束。 + +## 选择 Sink 前先确认什么 + +- 最终目标系统以及表、对象或主题的落点形态 +- 任务需要至少一次、精确一次还是幂等写入 +- 是否依赖额外驱动、SDK 或云鉴权配置 +- 该连接器是否支持你要使用的写入模式 + +## 常用下一步 + +- [Sink 常用选项](../common-options/sink-common-options.md) +- [连接器常见问题](../connector-faq.md) +- [连接器依赖隔离加载机制](../connector-isolated-dependency.md) diff --git a/docs/zh/connectors/source-overview.md b/docs/zh/connectors/source-overview.md new file mode 100644 index 000000000000..fdfa8df16c79 --- /dev/null +++ b/docs/zh/connectors/source-overview.md @@ -0,0 +1,21 @@ +--- +slug: /connectors/source +sidebar_position: 1 +--- + +# 数据来源连接器 + +当你的第一个问题是“SeaTunnel 要从哪里读取数据”时，就先看这一页。更好的顺序是先按外部系统找到合适的 Source，再确认插件安装、驱动依赖，以及当前任务是否需要批处理、流处理或 CDC 语义。 + +## 选择 Source 前先确认什么 + +- 你实际要读取的是哪一种外部系统 +- 任务需要快照、增量还是 CDC 语义 +- 是否依赖额外驱动、SDK 或鉴权配置 +- 示例参数是否与你当前使用的 SeaTunnel 版本一致 + +## 常用下一步 + +- [Source 常用选项](../common-options/source-common-options.md) +- [CDC 生产实战手册](../cdc-production-cookbook.md) +- [连接器常见问题](../connector-faq.md) diff --git a/docs/zh/getting-started/job-configuration-guide.md b/docs/zh/getting-started/job-configuration-guide.md index a749979e8fb5..b5cdf3793b23 100644 --- a/docs/zh/getting-started/job-configuration-guide.md +++ b/docs/zh/getting-started/job-configuration-guide.md @@ -172,6 +172,7 @@ SeaTunnel 支持多种配置方式： ## 下一步 - 需要先跑通一个可执行示例：查看 [SeaTunnel 引擎快速开始](./locally/quick-start-seatunnel-engine.md) -- 需要具体参数说明：查看 [Source 连接器](../connectors/source) 和 [Sink 连接器](../connectors/sink) -- 需要了解转换能力：查看 [Transforms](../transforms) +- 需要看一条更接近真实业务的完整链路：查看 [场景示例](./recipes/overview.md) +- 需要具体参数说明：查看 [数据来源连接器总览](../connectors/source-overview.md) 和 [数据写入连接器总览](../connectors/sink-overview.md) +- 需要了解转换能力：查看 [数据转换总览](../transforms) - 需要理解引擎差异：查看 [执行引擎概览](../engines/overview.md) diff --git a/docs/zh/getting-started/locally/deployment.md b/docs/zh/getting-started/locally/deployment.md index 0c5510b782da..1ecad853981f 100644 --- a/docs/zh/getting-started/locally/deployment.md +++ b/docs/zh/getting-started/locally/deployment.md @@ -83,6 +83,13 @@ tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" 现在您已经下载了SeaTunnel二进制包和连接器插件。接下来，您可以选择不同的引擎选项来运行同步任务。 +:::tip 提示 + +如果您是第一次使用 SeaTunnel，建议优先从 [SeaTunnel 引擎快速开始](quick-start-seatunnel-engine.md) 入手。 +这是默认引擎，通常也是第一次跑通任务的最短路径。 + +::: + 如果您使用Flink来运行同步任务，则无需部署SeaTunnel引擎服务集群。您可以参考[Flink 引擎快速开始](quick-start-flink.md)来运行您的同步任务。 如果您使用Spark来运行同步任务，则无需部署SeaTunnel引擎服务集群。您可以参考[Spark 引擎快速开始](quick-start-spark.md)来运行您的同步任务。 diff --git a/docs/zh/getting-started/locally/overview.md b/docs/zh/getting-started/locally/overview.md new file mode 100644 index 000000000000..0d108397c222 --- /dev/null +++ b/docs/zh/getting-started/locally/overview.md @@ -0,0 +1,29 @@ +--- +slug: /getting-started/locally +--- + +# 本地快速开始 + +如果你的目标是用最短路径把 SeaTunnel 在本机跑起来，就先看这一页。对大多数第一次接触 SeaTunnel 的用户来说，默认最推荐的仍然是 **SeaTunnel 引擎（Zeta）**，因为它部署最短、反馈最快、最适合作为首跑路径。 + +## 先选一条本地起步路径 + +| 你的情况 | 推荐入口 | +| --- | --- | +| 我想走默认的首跑路径 | [SeaTunnel 引擎快速开始](./quick-start-seatunnel-engine.md) | +| 我需要先完成安装和插件准备 | [部署](./deployment.md) | +| 团队已经有 Flink 环境 | [Flink 引擎快速开始](./quick-start-flink.md) | +| 团队已经有 Spark 环境 | [Spark 引擎快速开始](./quick-start-spark.md) | + +## 推荐首跑顺序 + +1. 先看 [部署](./deployment.md)。 +2. 安装示例任务所需插件。 +3. 通过 [跑第一个任务](./run-your-first-job.md) 或 [SeaTunnel 引擎快速开始](./quick-start-seatunnel-engine.md) 跑通首个本地作业。 +4. 示例成功后，再进入 [作业配置指南](../job-configuration-guide.md) 编写真实作业。 + +## 什么时候再看其他路径 + +- 只有在你已经维护 Flink 集群时，才优先走 Flink 路径。 +- 只有在你的现有作业体系本来就围绕 Spark 时，才优先走 Spark 路径。 +- 更稳妥的顺序仍然是先把本地链路跑通，再进入集群部署或远程提交。 diff --git a/docs/zh/getting-started/overview.md b/docs/zh/getting-started/overview.md index bcb6d4ebfd47..c78bd054f972 100644 --- a/docs/zh/getting-started/overview.md +++ b/docs/zh/getting-started/overview.md @@ -8,22 +8,22 @@ sidebar_position: 1 ## SeaTunnel 能帮你做什么 -SeaTunnel 是一个分布式数据集成平台，用统一的 Connector 模型处理异构系统之间的数据流转。常见场景包括： +SeaTunnel 是一个分布式数据集成平台，用统一的连接器模型处理异构系统之间的数据流转。常见场景包括： - 数据库、文件、数据仓库之间的批量同步 - CDC 与实时同步 - 多表同步或全库迁移 - 结构化、非结构化和二进制数据的多模态集成 -如果你是第一次评估 SeaTunnel，建议优先从内置的 **SeaTunnel Engine (Zeta)** 开始。它的部署路径最短，也是新项目的默认推荐执行引擎。 +如果你是第一次评估 SeaTunnel，建议优先从内置的 **SeaTunnel 引擎（Zeta）** 开始。它的部署路径最短，也是新项目的默认推荐执行引擎。 ## 如何选择执行引擎 | 引擎 | 适用场景 | 推荐入口 | | --- | --- | --- | -| SeaTunnel Engine (Zeta) | 新项目、CDC、低资源环境、本地快速验证 | [SeaTunnel 引擎快速开始](./locally/quick-start-seatunnel-engine.md) | -| Flink | 已有 Flink 集群和运维体系 | [Flink 快速开始](./locally/quick-start-flink.md) | -| Spark | 已有 Spark 集群和运维体系 | [Spark 快速开始](./locally/quick-start-spark.md) | +| SeaTunnel 引擎（Zeta） | 新项目、CDC、低资源环境、本地快速验证 | [SeaTunnel 引擎快速开始](./locally/quick-start-seatunnel-engine.md) | +| Flink 引擎 | 已有 Flink 集群和运维体系 | [Flink 快速开始](./locally/quick-start-flink.md) | +| Spark 引擎 | 已有 Spark 集群和运维体系 | [Spark 快速开始](./locally/quick-start-spark.md) | 如需更完整的引擎比较，请查看 [执行引擎概览](../engines/overview.md)。 @@ -34,7 +34,7 @@ SeaTunnel 是一个分布式数据集成平台，用统一的 Connector 模型 1. 阅读 [安装部署](./locally/deployment.md)，完成二进制包安装。 2. 安装示例任务所需的插件。 3. 使用 `FakeSource -> FieldMapper -> Console` 跑通本地 SeaTunnel Engine 快速开始。 -4. 示例成功后，再替换成真实的 Source 和 Sink。 +4. 示例成功后，再替换成真实连接器，开始编写真正的同步链路。 ## 推荐阅读路径 @@ -47,9 +47,10 @@ SeaTunnel 是一个分布式数据集成平台，用统一的 Connector 模型 ### 路径 B：我已经知道要接什么数据源 - [作业配置指南](./job-configuration-guide.md) -- [Source 连接器列表](../connectors/source) -- [Sink 连接器列表](../connectors/sink) -- [Transform 列表](../transforms) +- [数据来源连接器总览](../connectors/source-overview.md) +- [数据写入连接器总览](../connectors/sink-overview.md) +- [数据转换总览](../transforms) +- [场景示例](./recipes/overview.md) ### 路径 C：我想先理解整体架构 @@ -77,5 +78,7 @@ SeaTunnel 是一个分布式数据集成平台，用统一的 Connector 模型 推荐继续阅读： - [作业配置指南](./job-configuration-guide.md) -- [SeaTunnel Engine(Zeta) 安装部署](../engines/zeta/deployment.md) +- [场景示例](./recipes/overview.md) +- [SeaTunnel 引擎（Zeta）安装部署](../engines/zeta/deployment.md) - [REST API 与 Web UI](../engines/zeta/rest-api-and-web-ui.md) +- [向远程 Zeta 集群提交作业](./submit-job-to-remote-zeta-cluster.md) diff --git a/docs/zh/getting-started/recipes/file-to-starrocks.md b/docs/zh/getting-started/recipes/file-to-starrocks.md index 167356cf9920..9662c9530824 100644 --- a/docs/zh/getting-started/recipes/file-to-starrocks.md +++ b/docs/zh/getting-started/recipes/file-to-starrocks.md @@ -3,7 +3,7 @@ sidebar_position: 5 title: File 到 StarRocks --- -# File 到 StarRocks +# 文件到 StarRocks 当你想把本地 CSV 或文本文件导入 StarRocks，供后续高性能分析查询使用时，可以使用这条链路。 diff --git a/docs/zh/getting-started/recipes/http-to-jdbc.md b/docs/zh/getting-started/recipes/http-to-jdbc.md index 420d02ee18c0..5aa6873ae8a3 100644 --- a/docs/zh/getting-started/recipes/http-to-jdbc.md +++ b/docs/zh/getting-started/recipes/http-to-jdbc.md @@ -1,9 +1,9 @@ --- sidebar_position: 4 -title: Http 到 JDBC +title: HTTP 到 JDBC --- -# Http 到 JDBC +# HTTP 到 JDBC 当你想从 HTTP API 拉取结构化数据，并把结果落到关系型数据库中时，可以使用这条链路。 diff --git a/docs/zh/getting-started/recipes/overview.md b/docs/zh/getting-started/recipes/overview.md new file mode 100644 index 000000000000..f4fa7c3dfd71 --- /dev/null +++ b/docs/zh/getting-started/recipes/overview.md @@ -0,0 +1,25 @@ +--- +slug: /getting-started/recipes +--- + +# 场景示例 + +这些示例更适合在你已经跑通第一个本地任务之后再阅读。不要按顺序把所有示例都看一遍，而是优先找到最接近你真实 source 和 sink 组合的那条链路。 + +## 按业务目标选择示例 + +| 目标 | 推荐入口 | +| --- | --- | +| 从 MySQL CDC 同步到分析型数据库 | [MySQL CDC 到 Doris](./mysql-cdc-to-doris.md) | +| 把 JDBC 数据抽取到对象存储 | [JDBC 到 S3](./jdbc-to-s3.md) | +| 从 Kafka 流式写入表格式存储 | [Kafka 到 Iceberg](./kafka-to-iceberg.md) | +| 把 HTTP 数据写入关系型数据库 | [HTTP 到 JDBC](./http-to-jdbc.md) | +| 把文件数据加载到分析型系统 | [文件到 StarRocks](./file-to-starrocks.md) | +| 多表 CDC 编排 | [多表 CDC](./multi-table-cdc.md) | + +## 阅读示例时建议这样看 + +1. 先确认 source 和 sink 组合与你的目标链路是否一致。 +2. 再对照 `env`、`source`、`transform`、`sink` 四段结构理解参数。 +3. 改造示例时，一次只替换一个系统，避免同时改太多变量。 +4. 如果示例依赖 CDC、额外驱动或插件安装，先确认前置条件再运行。 diff --git a/docs/zh/introduction/about.md b/docs/zh/introduction/about.md index 89c2d6073cb9..1e8cd4b2fdfe 100644 --- a/docs/zh/introduction/about.md +++ b/docs/zh/introduction/about.md @@ -5,11 +5,12 @@ [![Slack](../../images/seatunnel-slack.svg)](https://s.apache.org/seatunnel-slack) [![Twitter Follow](../../images/ASFSeaTunnel.svg)](https://x.com/ASFSeaTunnel) -SeaTunnel是一个多模态、超高性能、分布式的海量数据集成工具，每天可稳定高效同步数百亿数据，已被数千家企业应用于生产，以其高效和稳定性深受众多企业信赖。 +SeaTunnel 是一个多模态、高性能、分布式的数据集成平台。 +它用统一的作业模型帮助团队在数据库、文件系统、数据湖、消息系统之间完成数据读取、转换与同步。 ## 从这里开始 -如果你是第一次接触 SeaTunnel，建议按下面路径进入文档： +如果您是第一次接触 SeaTunnel，建议按下面的顺序阅读：
-- [快速入门总览](../getting-started/overview.md)，先建立整体路径 -- [SeaTunnel 引擎快速开始](../getting-started/locally/quick-start-seatunnel-engine.md)，先跑通第一个本地任务 -- [作业配置指南](../getting-started/job-configuration-guide.md)，开始编写真实作业 -- [架构概览](../architecture/overview.md)，从系统层面理解 SeaTunnel +- [快速入门总览](../getting-started/overview.md)：先建立整体路径 +- [SeaTunnel 引擎快速开始](../getting-started/locally/quick-start-seatunnel-engine.md)：先跑通第一个任务 +- [作业配置指南](../getting-started/job-configuration-guide.md)：开始编写真实作业 +- [工作原理](how-it-works.md)：先理解运行模型，再进入更深层架构 -## 获取帮助与加入社区 - -如果你正在评估 SeaTunnel，或者在使用过程中遇到问题，建议优先从这些入口进入： - -- [常见问题解答](../faq.md)，快速解决常见使用、CDC 与配置问题 -- [开发环境搭建](../developer/setup.md)，如果你要本地构建或调试 SeaTunnel -- [贡献路径](../developer/contribution-path.md)，如果你想从最小、最稳妥的范围开始参与贡献 -- [贡献插件](../developer/contribute-plugin.md)，如果你准备贡献 connector 或 transform -- [GitHub Issues](https://github.com/apache/seatunnel/issues) 和 [dev 邮件列表](https://lists.apache.org/list.html?dev@seatunnel.apache.org)，如果你需要社区帮助 +如果您已经有 Flink 或 Spark 运行环境，也可以直接跳到 +[Flink 引擎快速开始](../getting-started/locally/quick-start-flink.md) 或 +[Spark 引擎快速开始](../getting-started/locally/quick-start-spark.md)。 -## 为什么需要 SeaTunnel +## SeaTunnel 能帮您做什么 -SeaTunnel专注于数据集成和数据同步，主要旨在解决数据集成领域的常见问题： +SeaTunnel 面向的是数据团队最常见、也最先要交付的几类任务： -* **数据源多样**：常用数据源有数百种，版本不兼容。 随着新技术的出现，更多的数据源不断出现。 用户很难找到一个能够全面、快速支持这些数据源的工具。 -* **多模态数据集成**：除了结构化数据外，用户还需要集成视频、图像、二进制文件、结构化和非结构化文本数据。 但是，现有的数据集成工具主要集中在结构化数据上。 -* **同步场景复杂**：数据同步需要支持离线全量同步、离线增量同步、CDC、实时同步、全库同步等多种同步场景。 -* **资源需求高**：现有的数据集成和数据同步工具往往需要大量的计算资源或JDBC连接资源来完成海量小表的实时同步。 这增加了企业的负担。 -* **缺乏质量和监控**：数据集成和同步过程经常会出现数据丢失或重复的情况。 同步过程缺乏监控，无法直观了解任务过程中数据的真实情况。 -* **技术栈复杂**：企业使用的技术组件不同，用户需要针对不同组件开发相应的同步程序来完成数据集成。 -* **管理和维护困难**：受限于底层技术组件（Flink/Spark）不同，离线同步和实时同步往往需要分开开发和管理，增加了管理和维护的难度。 +- **在多种系统之间搬运数据**：包括数据库、消息队列、文件系统、对象存储、数据湖和 SaaS 系统 +- **同时支持批处理和流处理**：同一套连接器模型可以覆盖全量、增量、CDC 和实时同步 +- **让作业定义保持清晰**：一个 SeaTunnel 作业仍然主要由 `env`、`source`、`transform`、`sink` 四部分组成 +- **降低运行复杂度**：SeaTunnel 关注高吞吐、较低依赖成本和实用的运行可观测性 -## SeaTunnel 相关特性 +## 团队为什么会选择 SeaTunnel -* **丰富且可扩展的Connector**：SeaTunnel提供了不依赖于特定执行引擎的Connector API。 基于该API开发的Connector（Source、Transform、Sink）可以运行在很多不同的引擎上，例如目前支持的SeaTunnel引擎（Zeta）、Flink、Spark等。 -* **Connector插件**：插件式设计让用户可以轻松开发自己的Connector并将其集成到SeaTunnel项目中。 目前，SeaTunnel 已支持超过 160 个连接器，并且生态仍在持续扩展。 -* **批流集成**：基于SeaTunnel Connector API开发的Connector完美兼容离线同步、实时同步、全量同步、增量同步等场景。 它们大大降低了管理数据集成任务的难度。 -* **分布式快照**：支持分布式快照算法，保证数据一致性。 -* **多引擎支持**：SeaTunnel默认使用SeaTunnel引擎（Zeta）进行数据同步。 SeaTunnel还支持使用Flink或Spark作为Connector的执行引擎，以适应企业现有的技术组件。 SeaTunnel 支持 Spark 和 Flink 的多个版本。 -* **JDBC复用、数据库日志多表解析**：SeaTunnel支持多表或全库同步，解决了过度JDBC连接的问题； 支持多表或全库日志读取解析，解决了CDC多表同步场景下需要处理日志重复读取解析的问题。 -* **高吞吐量、低延迟**：SeaTunnel支持并行读写，提供稳定可靠、高吞吐量、低延迟的数据同步能力。 -* **完善的实时监控**：SeaTunnel支持数据同步过程中每一步的详细监控信息，让用户轻松了解同步任务读写的数据数量、数据大小、QPS等信息。 -* **支持两种作业开发方法**：编码和画布设计。 SeaTunnel Web 项目 https://github.com/apache/seatunnel-web 提供作业、调度、运行和监控功能的可视化管理。 +- **连接器优先的设计**：SeaTunnel 提供统一的连接器接口（Connector API），Source、Transform、Sink 可以跨引擎复用 +- **引擎选择灵活**：可以直接从 SeaTunnel 引擎（Zeta）起步，也可以运行在 Flink 或 Spark 上 +- **面向真实同步场景**：多表同步、CDC、大规模作业执行都是第一类使用场景 +- **运行态可观察**：作业能够暴露运行指标和任务信息，方便理解吞吐、延迟和稳定性 +- **适合从小到大演进**：既可以本地先跑一个简单任务，也可以逐步扩展到更复杂的集群部署 -## SeaTunnel 工作流图 +## 用一张图理解 SeaTunnel -![SeaTunnel Work Flowchart](../../images/architecture_diagram.png) +![SeaTunnel 工作流程图](../../images/architecture_diagram.png) -SeaTunnel的运行流程如上图所示。 +您可以先抓住三个最重要的理解点： -用户配置作业信息并选择提交作业的执行引擎。 +### 1. SeaTunnel 作业本质上是一条数据管道 -Source Connector负责并行读取数据并将数据发送到下游Transform或直接发送到Sink，Sink将数据写入目的地。 值得注意的是，Source、Transform 和 Sink 可以很容易地自行开发和扩展。 +您用配置文件描述作业，SeaTunnel 再把它执行成一条从 **Source（读取）** 到 **Transform（转换）** 再到 **Sink（写入）** 的数据处理链路。 -SeaTunnel 是一个 EtL(T) 数据集成工具。 因此，在SeaTunnel中，transform(t)只能用于对数据进行一些简单的转换，例如将一列的数据转换为大写或小写，更改列名，或者将一列拆分为多列。 +### 2. 连接器决定读什么、写到哪里 -SeaTunnel 使用的默认引擎是 [SeaTunnel Zeta Engine](../engines/zeta/about.md)。 如果您选择使用Flink或Spark引擎，SeaTunnel会将Connector打包成Flink或Spark程序并提交给Flink或Spark运行。 +SeaTunnel 提供了丰富的 [源连接器](../connectors/source-overview.md)、 +[目标连接器](../connectors/sink-overview.md) 和 [数据转换](../transforms)。 +如果有特殊需求，您也可以自行扩展这些插件类型。 -## 连接器 +### 3. 引擎决定这条作业跑在哪儿 -- **源连接器** SeaTunnel 支持从各种关系、图形、NoSQL、文档和内存数据库读取数据； 分布式文件系统，例如HDFS； 以及各种云存储解决方案，例如S3和OSS。 我们还支持很多常见SaaS服务的数据读取。 您可以在[此处](../connectors/source) 访问详细列表。 如果您愿意，您可以开发自己的源连接器并将其轻松集成到 SeaTunnel 中。 +[SeaTunnel 引擎（Zeta）](../engines/zeta/about.md) 是默认选择，也是大多数新用户最推荐的起点。 +如果您已经在使用 Flink 或 Spark，SeaTunnel 也可以把同一套连接器作业模型运行在这些平台上。 -- **转换连接器** 如果源和接收器之间的架构不同，您可以使用转换连接器更改从源读取的架构，使其与接收器架构相同。 +## 如何选择运行引擎 -- **Sink Connector** SeaTunnel 支持将数据写入各种关系型、图形、NoSQL、文档和内存数据库； 分布式文件系统，例如HDFS； 以及各种云存储解决方案，例如S3和OSS。 我们还支持将数据写入许多常见的 SaaS 服务。 您可以在[此处](../connectors/sink)访问详细列表。 如果您愿意，您可以开发自己的 Sink 连接器并轻松将其集成到 SeaTunnel 中。 +| 引擎 | 推荐起点 | 适用场景 | +| --- | --- | --- | +| [SeaTunnel 引擎（Zeta）](../engines/zeta/about.md) | 推荐大多数新用户先从这里开始 | 希望以最短路径跑通 SeaTunnel 作业 | +| [Apache Flink](../engines/flink.md) | 适合已有 Flink 环境的团队 | 已经维护 Flink 集群，希望让 SeaTunnel 接入现有平台 | +| [Apache Spark](../engines/spark.md) | 适合已有 Spark 环境的团队 | 主要是批处理任务，希望复用现有 Spark 技术栈 | -## 谁在使用 SeaTunnel +## 继续阅读 -SeaTunnel 拥有大量用户。 您可以在[用户](https://seatunnel.apache.org/zh-CN/user)中找到有关他们的更多信息. +- [工作原理](how-it-works.md)：以新手能接受的层次理解运行模型 +- [配置文件简介](concepts/config.md)：开始写真实作业 +- [数据连接器总览](../connectors)：先确认读写方向，再进入具体连接器参数页 +- [系统架构概览](../architecture/overview.md)：当您需要深入内部设计时再继续往下读 +- [常见问题](../faq.md)：快速处理常见使用、CDC 与配置问题 -## 展望 +## 获取帮助与加入社区 -

-

-   -

-SeaTunnel 丰富了CNCF 云原生景观。 -

+- [开发环境搭建](../developer/setup.md)：如果您要本地构建或调试 SeaTunnel +- [贡献路径](../developer/contribution-path.md)：如果您想从最小、最稳妥的范围开始参与贡献 +- [贡献插件](../developer/contribute-plugin.md)：如果您准备贡献连接器或 transform 插件 +- [GitHub Issues](https://github.com/apache/seatunnel/issues)、[Slack](https://s.apache.org/seatunnel-slack) 和 [dev 邮件列表](https://lists.apache.org/list.html?dev@seatunnel.apache.org)：如果您需要社区帮助 -## 了解更多 +## 谁在使用 SeaTunnel -您可以参阅[跑第一个任务](../getting-started/locally/run-your-first-job.md) 了解后续相关步骤。 +SeaTunnel 拥有大量用户。您可以在[用户](https://seatunnel.apache.org/zh-CN/user)中找到有关他们的更多信息。 diff --git a/docs/zh/introduction/concepts/config.md b/docs/zh/introduction/concepts/config.md index ab482f66c7cf..980629988b75 100644 --- a/docs/zh/introduction/concepts/config.md +++ b/docs/zh/introduction/concepts/config.md @@ -1,15 +1,14 @@ # 配置文件简介 -在SeaTunnel中，最重要的事情就是配置文件，尽管用户可以自定义他们自己的数据同步需求以发挥SeaTunnel最大的潜力。那么接下来我将会向你介绍如何设置配置文件。 +如果您正在编写第一个真正可用的 SeaTunnel 作业，这一页最重要的目标，就是帮您先理解几乎所有配置里都会出现的四个部分：`env`、`source`、`transform`、`sink`。 -配置文件的主要格式是 `hocon`, 有关该格式类型的更多信息你可以参考[HOCON-GUIDE](https://github.com/lightbend/config/blob/main/HOCON.md), -顺便提一下，我们也支持 `json`格式，但你应该知道配置文件的名称应该是以 `.json`结尾。 +SeaTunnel 支持 `hocon`、`json` 和 `SQL` 三种配置格式。其中 **HOCON** 是快速开始和生产示例中最常见的格式。SQL 格式请参考 [SQL 配置文件](../configuration/sql-config.md)。 -我们同时提供了以 `SQL` 格式，详细可以参考[SQL配置文件](../configuration/sql-config.md)。 +如果您还没有跑通过第一个任务，建议先阅读 [快速入门总览](../../getting-started/overview.md) 和 [SeaTunnel 引擎快速开始](../../getting-started/locally/quick-start-seatunnel-engine.md)，再回到这一页。 ## 例子 -在你阅读之前，你可以在发布包中的config目录[这里](https://github.com/apache/seatunnel/tree/dev/config)找到配置文件的例子。 +继续往下看之前，您可以先在发布包的 `config` 目录，或者 [这里](https://github.com/apache/seatunnel/tree/dev/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources) 查看示例配置。 ## 配置文件结构 @@ -21,7 +20,7 @@ ::: -### hocon +### HOCON 示例 ```hocon env { @@ -63,28 +62,25 @@ sink { } ``` -正如你看到的，配置文件包括几个部分：env, source, transform, sink。不同的模块具有不同的功能。 -当你了解了这些模块后，你就会懂得SeaTunnel到底是如何工作的。 +大多数 SeaTunnel 作业都会遵循 `env`、`source`、`transform`、`sink` 这四段结构。只要先理解这四段，后面读快速开始、连接器示例和真实任务配置都会容易很多。 -### env +### `env`：作业与引擎参数 -用于添加引擎可选的参数，不管是什么引擎（Zeta、Spark 或者 Flink），对应的可选参数应该在这里填写。 +`env` 用来放作业级和引擎级参数，比如 `job.mode`、`parallelism`、checkpoint 相关配置，以及引擎特有参数。 -注意，我们按照引擎分离了参数，对于公共参数我们可以像以前一样配置。对于Flink和Spark引擎，其参数的具体配置规则可以参考[JobEnvConfig](../configuration/JobEnvConfig.md)。 +公共参数可以直接配置；引擎专属参数则按前缀区分。Flink 和 Spark 的具体写法请参考 [JobEnvConfig](../configuration/JobEnvConfig.md)。 -### source +### `source`：数据读取入口 -source用于定义SeaTunnel在哪儿检索数据，并将检索的数据用于下一步。 -可以同时定义多个source。目前支持的source请看[Source of SeaTunnel](../../connectors/source)。每种source都有自己特定的参数用来 -定义如何检索数据，SeaTunnel也抽象了每种source所使用的参数，例如 `plugin_output` 参数，用于指定当前source生成的数据的名称， -方便后续其他模块使用。 +`source` 用来定义 SeaTunnel 从哪里读取数据。一个作业中可以同时声明多个 source。每个连接器都有自己的参数，同时也有一些通用的链路字段，例如 `plugin_output`，它用于给当前 source 产出的数据集命名，方便后续模块引用。 -### transform +完整列表请查看 [数据来源连接器](../../connectors/source-overview.md)。 -当我们有了数据源之后，我们可能需要对数据进行进一步的处理，所以我们就有了transform模块。当然，这里使用了“可能”这个词， -这意味着我们也可以直接将transform视为不存在，直接从source到sink，像下面这样： +### `transform`：中间处理步骤 + +`transform` 是可选的。当您需要做字段映射、过滤、类型转换、SQL 处理，或者其它中间加工时，就使用这一层；如果不需要，也可以直接从 source 到 sink，例如下面这样： ```hocon env { @@ -118,22 +114,22 @@ sink { } ``` -与source类似, transform也有属于每个模块的特定参数。目前支持的source请看。目前支持的transform请看 [Transform V2 of SeaTunnel](../../transforms) +和 source 一样，每个 transform 也有自己的专属参数。完整列表请查看 [数据转换目录](../../transforms)。 + +### `sink`：数据写入目标 + +`sink` 用来定义处理后的数据写到哪里去。它和 source 很相似，但更关注写入行为、目标表结构、提交方式以及投递保证。 - +完整列表请查看 [数据写入连接器](../../connectors/sink-overview.md)。 -### sink +### `plugin_output` 和 `plugin_input` 是怎么工作的 -我们使用SeaTunnel的作用是将数据从一个地方同步到其它地方，所以定义数据如何写入，写入到哪里是至关重要的。通过SeaTunnel提供的 -sink模块，你可以快速高效地完成这个操作。Sink和source非常相似，区别在于读取和写入。所以去看看我们[Sink of SeaTunnel](../../connectors/sink)吧。 +当一个作业里同时存在多个 source、transform 或 sink 时，SeaTunnel 需要知道“哪一份数据流向下一步的哪个模块”。这就是 `plugin_output` 和 `plugin_input` 的作用。 -### 其它 +- `plugin_output` 给当前 source 或 transform 产出的数据集命名 +- `plugin_input` 告诉下游 transform 或 sink 应该消费哪一个上游数据集 -你会疑惑当定义了多个source和多个sink时，每个sink读取哪些数据，每个transform读取哪些数据？我们使用`plugin_output` 和 -`plugin_input` 两个配置。每个source模块都会配置一个`plugin_output`来指示数据源生成的数据源名称，其它transform和sink -模块可以使用`plugin_input` 引用相应的数据源名称，表示要读取数据进行处理。然后transform，作为一个中间的处理模块，可以同时使用 -`plugin_output` 和 `plugin_input` 配置。但你会发现在上面的配置例子中，不是每个模块都配置了这些参数，因为在SeaTunnel中， -有一个默认的约定，如果这两个参数没有配置，则使用上一个节点的最后一个模块生成的数据。当只有一个source时这是非常方便的。 +如果只是单一 source 的简单链路，很多时候可以省略它们，因为 SeaTunnel 会按默认约定自动把上一个模块的输出继续往下传递。 ## 多行文本支持 @@ -148,7 +144,7 @@ distributed, massive data integration tool. sql = """ select * from "table" """ ``` -## Json格式支持 +## JSON 格式支持 在编写配置文件之前，请确保配置文件的名称应以 `.json` 结尾。 @@ -198,7 +194,7 @@ sql = """ select * from "table" """ ## 配置变量替换 -在配置文件中,我们可以定义一些变量并在运行时替换它们。但是注意仅支持 hocon 格式的文件。 +在配置文件中，我们可以定义一些变量并在运行时替换它们。但请注意，目前仅支持 HOCON 格式的文件。 变量使用方法： - `${varName}`，如果变量未传值，则抛出异常。 @@ -324,4 +320,6 @@ sink { - 不能使用指定系统保留字符，它将不会被`-i`替换，如:`${database_name}`、`${schema_name}`、`${table_name}`、`${schema_full_name}`、`${table_full_name}`、`${primary_key}`、`${unique_key}`、`${field_names}`、`${partition_keys}`。具体可参考[Sink参数占位符](../configuration/sink-options-placeholders.md) ## 此外 -如果你想了解更多关于格式配置的详细信息，请查看 [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md)。 +- 现在就可以开始写自己的配置文件，选择要使用的 [连接器](../../connectors/source-overview.md)，再按对应文档填写参数。 +- 如果您需要按引擎配置参数，请继续阅读 [JobEnvConfig](../configuration/JobEnvConfig.md)。 +- 如果您想了解更完整的语法细节，请查看 [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md)。 diff --git a/docs/zh/introduction/how-it-works.md b/docs/zh/introduction/how-it-works.md index 5ea92db16f6d..d68633cb000c 100644 --- a/docs/zh/introduction/how-it-works.md +++ b/docs/zh/introduction/how-it-works.md @@ -4,6 +4,18 @@ sidebar_position: 2 # 工作原理 +## 新用户先抓住这几点 + +在第一次使用 SeaTunnel 时，您不需要先理解所有内部模块。 +对大多数新用户来说，更实用的顺序是： + +1. 先在本地跑通一个任务 +2. 再理解配置文件结构 +3. 然后选择合适的连接器和执行引擎 +4. 当您需要理解运行模型时，再回到这一页 + +把 SeaTunnel 先理解成“一条由配置驱动、运行在某个执行引擎上的数据管道”，通常最容易入门。 + ## 概述 SeaTunnel 是一个分布式多模态数据集成工具，采用插件化架构。连接器层与执行引擎解耦，同一套连接器可在不同引擎上运行。 @@ -35,6 +47,28 @@ flowchart TD linkStyle default stroke:#5db8e2,stroke-width:2px; ``` +## 四个核心构件 + +### 1. 作业配置 + +配置文件描述了读什么、怎么转换、写到哪里，以及需要使用哪些引擎参数。 + +### 2. SeaTunnel 核心层 + +SeaTunnel 会解析配置、生成执行计划、加载插件，并把作业提交到选定的执行引擎。 + +### 3. 数据链路：Source -> Transform -> Sink + +这是大多数新用户最应该先记住的数据路径： + +- **Source（读取）** 负责从外部系统读取数据 +- **Transform（转换）** 负责按需做字段映射、过滤或简单转换 +- **Sink（写入）** 负责把结果写入目标系统 + +### 4. 执行引擎 + +引擎决定作业最终跑在哪儿。对大多数新用户来说，建议先从 [SeaTunnel 引擎（Zeta）](../engines/zeta/about.md) 开始；只有在现有环境已经依赖 Flink 或 Spark 时，再切换到对应引擎。 + ## 推荐阅读路径 如果你希望先建立一套系统级理解，建议按下面顺序阅读： @@ -43,26 +77,26 @@ flowchart TD - 本页，先建立执行模型的整体图景 - [引擎概览](../engines/overview.md)，理解执行引擎如何选择 - [架构概览](../architecture/overview.md)，再进入更完整的分层视图 -- [核心 API 设计](../architecture/core-api-design.md)，理解 connector 与元数据契约 +- [核心 API 设计](../architecture/core-api-design.md)，理解连接器与元数据契约 - 如果你还需要理解数据集编排与 transform 行为，再看 [Transform 插件体系](../architecture/transform-plugin-system.md) ## 核心组件 -### 1. Connector API +### 1. 连接器接口（Connector API） -与引擎无关的统一 API，用于开发 Source、Transform、Sink 连接器。 +与引擎无关的统一接口，用于开发 Source、Transform、Sink 连接器。 | 组件 | 说明 | |------|------| -| **Source** | 从外部系统读取数据（数据库、文件、消息队列） | -| **Transform** | 数据转换（字段映射、过滤、类型转换） | -| **Sink** | 将数据写入目标系统 | +| **Source（读取）** | 从外部系统读取数据（数据库、文件、消息队列） | +| **Transform（转换）** | 数据转换（字段映射、过滤、类型转换） | +| **Sink（写入）** | 将数据写入目标系统 | ### 2. 执行引擎 | 引擎 | 适用场景 | |------|---------| -| **SeaTunnel Engine (Zeta)** | 数据同步、CDC、低资源消耗 | +| **SeaTunnel 引擎（Zeta）** | 数据同步、CDC、低资源消耗 | | **Apache Flink** | 复杂流处理、已有 Flink 基础设施 | | **Apache Spark** | 大规模批处理、已有 Spark 基础设施 | @@ -99,7 +133,7 @@ flowchart LR | `seatunnel-api` | 核心 API 定义 | | `seatunnel-connectors-v2` | Source 和 Sink 连接器 | | `seatunnel-transforms-v2` | Transform 插件 | -| `seatunnel-engine` | SeaTunnel Engine (Zeta) | +| `seatunnel-engine` | SeaTunnel 引擎（Zeta） | | `seatunnel-translation` | Flink 和 Spark 的引擎适配器 | | `seatunnel-core` | 作业提交与 CLI | | `seatunnel-formats` | 数据格式处理 | @@ -119,4 +153,4 @@ flowchart LR - [快速入门总览](../getting-started/overview.md) - [SeaTunnel 引擎快速开始](../getting-started/locally/quick-start-seatunnel-engine.md) - [架构概览](../architecture/overview.md) -- [连接器列表](../connectors) +- [数据连接器总览](../connectors) diff --git a/docs/zh/transforms/overview.md b/docs/zh/transforms/overview.md new file mode 100644 index 000000000000..ae14283ba4ac --- /dev/null +++ b/docs/zh/transforms/overview.md @@ -0,0 +1,23 @@ +--- +slug: /transforms +--- + +# 数据转换总览 + +Transform 位于 source 和 sink 之间，用来做字段映射、过滤、SQL 处理、表级编排等中间加工。第一次上手时，不需要先把所有 transform 都看完；更合适的顺序是先确定数据从哪来、写到哪去，再回来选择需要的转换能力。 + +## 先按目标找入口 + +| 目标 | 推荐入口 | +| --- | --- | +| 先理解 transform 如何连接数据集 | [转换通用参数](./common-options/common-options.md) | +| 做行过滤或字段裁剪 | [数据过滤（Filter）](./filter.md) 和 [字段映射（Field Mapper）](./field-mapper.md) | +| 用 SQL 表达式处理数据 | [SQL 转换](./sql.md) 和 [SQL 函数](./sql-functions.md) | +| 重命名或重组字段 | [字段重命名（Field Rename）](./field-rename.md) 和 [字段拆分（Split）](./split.md) | +| 处理多表链路 | [多表转换（Transform Multi Table）](./transform-multi-table.md) 和 [表合并（Table Merge）](./table-merge.md) | + +## 新用户推荐顺序 + +1. 先看通用参数页，把 `plugin_input` 和 `plugin_output` 的作用理解清楚。 +2. 先选择最简单、最贴近目标的 transform，再进入 SQL 或多表编排。 +3. 每次只新增一个 transform 步骤，让整条 pipeline 在验证时保持可读、可排障。 From bccea91c923c6fb4bcf4f52a3a71384f3b813f6a Mon Sep 17 00:00:00 2001 From: Jast Date: Mon, 22 Jun 2026 20:55:26 +0800 Subject: [PATCH 055/375] [Fix][Connector-V2] Support MongoDB CDC SRV URI (#11129) --- docs/en/connectors/source/MongoDB-CDC.md | 4 +- docs/zh/connectors/source/MongoDB-CDC.md | 4 +- .../MongodbIncrementalSourceOptions.java | 4 +- .../cdc/mongodb/utils/MongodbRecordUtils.java | 7 +- .../cdc/mongodb/utils/MongodbUtils.java | 100 +++++++++++++++++- .../java/mongodb/utils/MongodbUtilsTest.java | 82 ++++++++++++++ 6 files changed, 187 insertions(+), 14 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/test/java/mongodb/utils/MongodbUtilsTest.java diff --git a/docs/en/connectors/source/MongoDB-CDC.md b/docs/en/connectors/source/MongoDB-CDC.md index 3c153350fb0f..03d3bc221d86 100644 --- a/docs/en/connectors/source/MongoDB-CDC.md +++ b/docs/en/connectors/source/MongoDB-CDC.md @@ -114,7 +114,7 @@ For specific types in MongoDB, we use Extended JSON format to map them to Seatun | Name | Type | Required | Default | Description | |------------------------------------|--------|----------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| hosts | String | Yes | - | The comma-separated list of hostname and port pairs of the MongoDB servers. eg. `localhost:27017,localhost:27018` | +| hosts | String | Yes | - | The comma-separated list of hostname and port pairs of the MongoDB servers, or a standard MongoDB connection URI using `mongodb://` or `mongodb+srv://`. eg. `localhost:27017,localhost:27018` or `mongodb+srv://cluster.example.net` | | username | String | No | - | Name of the database user to be used when connecting to MongoDB. | | password | String | No | - | Password to be used when connecting to MongoDB. | | database | List | Yes | - | Name of the database to watch for changes. If not set then all databases will be captured. The database also supports regular expressions to monitor multiple databases matching the regular expression. eg. `db1,db2`. | @@ -373,4 +373,4 @@ sink { ## Changelog - \ No newline at end of file + diff --git a/docs/zh/connectors/source/MongoDB-CDC.md b/docs/zh/connectors/source/MongoDB-CDC.md index 83782ea38028..9f0362608e48 100644 --- a/docs/zh/connectors/source/MongoDB-CDC.md +++ b/docs/zh/connectors/source/MongoDB-CDC.md @@ -114,7 +114,7 @@ db.grantRolesToUser("", [""]) | Name | 类型 | 必须 | 默认值 | 描述 | |------------------------------------|--------|----------|-------|---------------------------------------------------------------------------------------| -| hosts | String | 是 | - | MongoDB服务器的主机名和端口对的逗号分隔列表。如 `localhost:27017,localhost:27018` | +| hosts | String | 是 | - | MongoDB服务器的主机名和端口对的逗号分隔列表，也可以是使用 `mongodb://` 或 `mongodb+srv://` 的标准 MongoDB 连接 URI。如 `localhost:27017,localhost:27018` 或 `mongodb+srv://cluster.example.net` | | username | String | 否 | - | 连接到MongoDB时要使用的数据库用户的名称。 | | password | String | 否 | - | 连接到MongoDB时使用的密码。 | | database | List | 是 | - | 要监视更改的数据库的名称。如果未设置，则将捕获所有数据库。该数据库还支持正则表达式，以监视与正则表达式匹配的多个数据库。例如db1、db2。 | @@ -373,4 +373,4 @@ sink { ## 修改日志 - \ No newline at end of file + diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/config/MongodbIncrementalSourceOptions.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/config/MongodbIncrementalSourceOptions.java index ff2449e15df1..2602449a7bda 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/config/MongodbIncrementalSourceOptions.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/config/MongodbIncrementalSourceOptions.java @@ -39,7 +39,9 @@ public class MongodbIncrementalSourceOptions extends SourceOptions implements Ta .noDefaultValue() .withDescription( "The comma-separated list of hostname and port pairs of the MongoDB servers. " - + "eg. localhost:27017,localhost:27018"); + + "A standard MongoDB connection URI with mongodb:// or mongodb+srv:// " + + "scheme is also supported. " + + "eg. localhost:27017,localhost:27018 or mongodb+srv://cluster.example.net"); public static final Option USERNAME = Options.key("username") diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbRecordUtils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbRecordUtils.java index 1487962f2069..9b378df42480 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbRecordUtils.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbRecordUtils.java @@ -53,6 +53,7 @@ import static org.apache.seatunnel.connectors.seatunnel.cdc.mongodb.config.MongodbSourceConstants.NS_FIELD; import static org.apache.seatunnel.connectors.seatunnel.cdc.mongodb.config.MongodbSourceConstants.OUTPUT_SCHEMA; import static org.apache.seatunnel.connectors.seatunnel.cdc.mongodb.config.MongodbSourceConstants.SOURCE_FIELD; +import static org.apache.seatunnel.connectors.seatunnel.cdc.mongodb.utils.MongodbUtils.buildConnectionNamespacePrefix; public class MongodbRecordUtils { @@ -194,9 +195,7 @@ public static String getOffsetValue(@Nonnull SourceRecord sourceRecord, String k public static @Nonnull Map createPartitionMap( String hosts, String database, String collection) { - StringBuilder builder = new StringBuilder(); - builder.append("mongodb://"); - builder.append(hosts); + StringBuilder builder = new StringBuilder(buildConnectionNamespacePrefix(hosts)); builder.append("/"); if (StringUtils.isNotEmpty(database)) { builder.append(database); @@ -209,7 +208,7 @@ public static String getOffsetValue(@Nonnull SourceRecord sourceRecord, String k } public static @Nonnull Map createHeartbeatPartitionMap(String hosts) { - String builder = "mongodb://" + hosts + "/" + "__mongodb_heartbeats"; + String builder = buildConnectionNamespacePrefix(hosts) + "/" + "__mongodb_heartbeats"; return Collections.singletonMap(NS_FIELD, builder); } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbUtils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbUtils.java index 8dba927f0b92..a446d748c4c0 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbUtils.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbUtils.java @@ -84,6 +84,9 @@ @Slf4j public class MongodbUtils { + private static final String MONGODB_SCHEME = "mongodb://"; + private static final String MONGODB_SRV_SCHEME = "mongodb+srv://"; + private static final String DEFAULT_DATABASE = "/"; public static ChangeStreamDescriptor getChangeStreamDescriptor( @Nonnull MongodbSourceConfig sourceConfig, @@ -371,19 +374,106 @@ public static MongoClient createMongoClient(MongodbSourceConfig sourceConfig) { public static @Nonnull ConnectionString buildConnectionString( String username, String password, String hosts, String connectionOptions) { - StringBuilder sb = new StringBuilder("mongodb://"); + String uri = + isConnectionUri(hosts) + ? buildFromUri(hosts, username, password, connectionOptions) + : buildFromHosts(hosts, username, password, connectionOptions); + return new ConnectionString(uri); + } + private static String buildFromHosts( + String hosts, String username, String password, String connectionOptions) { + StringBuilder sb = new StringBuilder(MONGODB_SCHEME); if (hasCredentials(username, password)) { appendCredentials(sb, username, password); } - sb.append(hosts); - if (StringUtils.isNotEmpty(connectionOptions)) { sb.append("/?").append(connectionOptions); } + return sb.toString(); + } + + private static String buildFromUri( + String uri, String username, String password, String connectionOptions) { + StringBuilder sb = new StringBuilder(uri); + if (hasCredentials(username, password) && !hasCredentialsInUri(uri)) { + sb.insert(getScheme(uri).length(), credentialString(username, password)); + } + appendConnectionOptions(sb, connectionOptions); + return sb.toString(); + } + + public static boolean isConnectionUri(String hosts) { + return StringUtils.startsWith(hosts, MONGODB_SCHEME) + || StringUtils.startsWith(hosts, MONGODB_SRV_SCHEME); + } + + public static String buildConnectionNamespacePrefix(String hosts) { + if (!isConnectionUri(hosts)) { + return MONGODB_SCHEME + hosts; + } + + String scheme = getScheme(hosts); + String remaining = hosts.substring(scheme.length()); + int boundary = findAuthorityBoundary(remaining); + String authority = boundary == -1 ? remaining : remaining.substring(0, boundary); + int credentialIndex = authority.lastIndexOf('@'); + if (credentialIndex >= 0) { + authority = authority.substring(credentialIndex + 1); + } + return scheme + authority; + } + + private static String getScheme(String uri) { + return StringUtils.startsWith(uri, MONGODB_SRV_SCHEME) + ? MONGODB_SRV_SCHEME + : MONGODB_SCHEME; + } + + private static boolean hasCredentialsInUri(String uri) { + String scheme = getScheme(uri); + String remaining = uri.substring(scheme.length()); + int boundary = findAuthorityBoundary(remaining); + String authority = boundary == -1 ? remaining : remaining.substring(0, boundary); + return authority.contains("@"); + } + + private static int findAuthorityBoundary(String uriWithoutScheme) { + int slashIndex = uriWithoutScheme.indexOf('/'); + int queryIndex = uriWithoutScheme.indexOf('?'); + if (slashIndex == -1) { + return queryIndex; + } + if (queryIndex == -1) { + return slashIndex; + } + return Math.min(slashIndex, queryIndex); + } + + private static String credentialString(String username, String password) { + return encodeValue(username) + ":" + encodeValue(password) + "@"; + } + + private static void appendConnectionOptions(StringBuilder sb, String connectionOptions) { + if (StringUtils.isEmpty(connectionOptions)) { + return; + } + if (StringUtils.contains(sb, "?")) { + if (!StringUtils.endsWith(sb, "?") && !StringUtils.endsWith(sb, "&")) { + sb.append("&"); + } + } else if (hasDatabasePath(sb)) { + sb.append("?"); + } else { + sb.append(DEFAULT_DATABASE).append("?"); + } + sb.append(connectionOptions); + } - return new ConnectionString(sb.toString()); + private static boolean hasDatabasePath(StringBuilder sb) { + int schemeEnd = sb.indexOf("://") + 3; + return sb.indexOf("/", schemeEnd) >= 0; } private static boolean hasCredentials(String username, String password) { @@ -392,7 +482,7 @@ private static boolean hasCredentials(String username, String password) { private static void appendCredentials( @Nonnull StringBuilder sb, String username, String password) { - sb.append(encodeValue(username)).append(":").append(encodeValue(password)).append("@"); + sb.append(credentialString(username, password)); } public static String encodeValue(String value) { diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/test/java/mongodb/utils/MongodbUtilsTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/test/java/mongodb/utils/MongodbUtilsTest.java new file mode 100644 index 000000000000..b441717da693 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/test/java/mongodb/utils/MongodbUtilsTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mongodb.utils; + +import org.apache.seatunnel.connectors.seatunnel.cdc.mongodb.utils.MongodbRecordUtils; +import org.apache.seatunnel.connectors.seatunnel.cdc.mongodb.utils.MongodbUtils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import static org.apache.seatunnel.connectors.seatunnel.cdc.mongodb.config.MongodbSourceConstants.NS_FIELD; + +public class MongodbUtilsTest { + + @Test + public void testBuildConnectionStringFromHostsKeepsLegacyFormat() { + String connectionString = + MongodbUtils.buildConnectionString( + "user", "password", "localhost:27017", "replicaSet=test") + .getConnectionString(); + + Assertions.assertEquals( + "mongodb://user:password@localhost:27017/?replicaSet=test", connectionString); + } + + @Test + public void testBuildConnectionStringKeepsSrvUri() { + String srvUri = + "mongodb+srv://user:password@cluster0.example.mongodb.net/test" + + "?retryWrites=true&w=majority"; + + String connectionString = + MongodbUtils.buildConnectionString(null, null, srvUri, null).getConnectionString(); + + Assertions.assertEquals(srvUri, connectionString); + } + + @Test + public void testBuildConnectionStringAddsCredentialsAndOptionsToSrvUri() { + String connectionString = + MongodbUtils.buildConnectionString( + "user", + "password", + "mongodb+srv://cluster0.example.mongodb.net", + "retryWrites=true") + .getConnectionString(); + + Assertions.assertEquals( + "mongodb+srv://user:password@cluster0.example.mongodb.net/?retryWrites=true", + connectionString); + } + + @Test + public void testPartitionMapUsesSrvNamespaceWithoutCredentialsOrOptions() { + String hosts = + "mongodb+srv://user:password@cluster0.example.mongodb.net/admin" + + "?retryWrites=true"; + + Assertions.assertEquals( + "mongodb+srv://cluster0.example.mongodb.net/inventory.products", + MongodbRecordUtils.createPartitionMap(hosts, "inventory", "products") + .get(NS_FIELD)); + Assertions.assertEquals( + "mongodb+srv://cluster0.example.mongodb.net/__mongodb_heartbeats", + MongodbRecordUtils.createHeartbeatPartitionMap(hosts).get(NS_FIELD)); + } +} From cda69534f0b35d36edc15af6e54c79a1a515955e Mon Sep 17 00:00:00 2001 From: zoo-code <75787789+zooo-code@users.noreply.github.com> Date: Mon, 22 Jun 2026 22:03:01 +0900 Subject: [PATCH 056/375] [Fix][Connector-V2][MySQL-CDC] Fix BinlogOffset.compareTo ignoring restartSkipRows when GTID sets are equal (#10811) --- .../cdc/mysql/source/offset/BinlogOffset.java | 11 ++- .../mysql/source/offset/BinlogOffsetTest.java | 88 +++++++++++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/offset/BinlogOffsetTest.java diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/offset/BinlogOffset.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/offset/BinlogOffset.java index 06e71dd4a6c3..fc555650785f 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/offset/BinlogOffset.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/offset/BinlogOffset.java @@ -141,9 +141,14 @@ public int compareTo(Offset offset) { GtidSet gtidSet = new GtidSet(gtidSetStr); GtidSet targetGtidSet = new GtidSet(targetGtidSetStr); if (gtidSet.equals(targetGtidSet)) { - long restartSkipEvents = this.getRestartSkipEvents(); - long targetRestartSkipEvents = that.getRestartSkipEvents(); - return Long.compare(restartSkipEvents, targetRestartSkipEvents); + // Same GTID set means both offsets are within the same transaction + // boundary, so ordering must also consider per-event and per-row progress. + int eventCompare = + Long.compare(this.getRestartSkipEvents(), that.getRestartSkipEvents()); + if (eventCompare != 0) { + return eventCompare; + } + return Long.compare(this.getRestartSkipRows(), that.getRestartSkipRows()); } // The GTIDs are not an exact match, so figure out if this is a subset of the target // offset diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/offset/BinlogOffsetTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/offset/BinlogOffsetTest.java new file mode 100644 index 000000000000..4889308644e6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/offset/BinlogOffsetTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.mysql.source.offset; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class BinlogOffsetTest { + + private static final String GTID_SET_A = "036d85a9-64e5-11e6-9b48-42010af0000c:1-100"; + private static final String GTID_SET_B = "036d85a9-64e5-11e6-9b48-42010af0000c:1-200"; + + @Test + public void testCompareToWithEqualGtidSetConsidersRestartSkipRows() { + BinlogOffset lower = new BinlogOffset("mysql-bin.000001", 4L, 1L, 5L, 0L, GTID_SET_A, 1); + BinlogOffset higher = new BinlogOffset("mysql-bin.000001", 4L, 1L, 9L, 0L, GTID_SET_A, 1); + + Assertions.assertTrue( + lower.compareTo(higher) < 0, + "offset with smaller restartSkipRows must be ordered before a larger one " + + "when the GTID set and restartSkipEvents are equal"); + Assertions.assertTrue(higher.compareTo(lower) > 0); + Assertions.assertEquals(0, lower.compareTo(lower)); + } + + @Test + public void testCompareToWithEqualGtidSetPrefersRestartSkipEvents() { + BinlogOffset earlierEvent = + new BinlogOffset("mysql-bin.000001", 4L, 1L, 9L, 0L, GTID_SET_A, 1); + BinlogOffset laterEvent = + new BinlogOffset("mysql-bin.000001", 4L, 2L, 0L, 0L, GTID_SET_A, 1); + + Assertions.assertTrue( + earlierEvent.compareTo(laterEvent) < 0, + "restartSkipEvents must take precedence over restartSkipRows"); + } + + @Test + public void testCompareToWithEqualGtidSetAndEqualProgress() { + BinlogOffset a = new BinlogOffset("mysql-bin.000001", 4L, 1L, 5L, 0L, GTID_SET_A, 1); + BinlogOffset b = new BinlogOffset("mysql-bin.000001", 4L, 1L, 5L, 0L, GTID_SET_A, 1); + + Assertions.assertEquals(0, a.compareTo(b)); + } + + @Test + public void testCompareToWithGtidSubsetAndSuperset() { + BinlogOffset subset = new BinlogOffset("mysql-bin.000001", 4L, 0L, 0L, 0L, GTID_SET_A, 1); + BinlogOffset superset = new BinlogOffset("mysql-bin.000001", 4L, 0L, 0L, 0L, GTID_SET_B, 1); + + Assertions.assertTrue(subset.compareTo(superset) < 0); + Assertions.assertTrue(superset.compareTo(subset) > 0); + } + + @Test + public void testCompareToWithoutGtidFallsBackToEventsAndRows() { + BinlogOffset lowerRow = new BinlogOffset("mysql-bin.000001", 4L, 1L, 3L, 0L, null, 1); + BinlogOffset higherRow = new BinlogOffset("mysql-bin.000001", 4L, 1L, 7L, 0L, null, 1); + + Assertions.assertTrue(lowerRow.compareTo(higherRow) < 0); + Assertions.assertTrue(higherRow.compareTo(lowerRow) > 0); + } + + @Test + public void testNoStoppingOffsetIsAlwaysMaximum() { + BinlogOffset regular = new BinlogOffset("mysql-bin.000001", 4L, 1L, 5L, 0L, GTID_SET_A, 1); + + Assertions.assertTrue(BinlogOffset.NO_STOPPING_OFFSET.compareTo(regular) > 0); + Assertions.assertTrue(regular.compareTo(BinlogOffset.NO_STOPPING_OFFSET) < 0); + Assertions.assertEquals( + 0, BinlogOffset.NO_STOPPING_OFFSET.compareTo(BinlogOffset.NO_STOPPING_OFFSET)); + } +} From ac89062f1f881975319e19fe0b69633da7c13b69 Mon Sep 17 00:00:00 2001 From: Jast Date: Mon, 22 Jun 2026 21:04:35 +0800 Subject: [PATCH 057/375] [Fix][Transform-V2] Align Scala compiler with Spark Scala version (#11117) --- .../api/ScalaCompilerVersionCheckTest.java | 97 +++++++++++++++++++ .../seatunnel-scala-compiler/pom.xml | 5 - .../parse/ScalaClassParser.java | 3 +- tools/dependencies/known-dependencies.txt | 8 +- 4 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/ScalaCompilerVersionCheckTest.java diff --git a/seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/ScalaCompilerVersionCheckTest.java b/seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/ScalaCompilerVersionCheckTest.java new file mode 100644 index 000000000000..ddf110a4cfe1 --- /dev/null +++ b/seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/ScalaCompilerVersionCheckTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.api; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilderFactory; + +import java.io.File; + +public class ScalaCompilerVersionCheckTest { + + @Test + public void testScalaCompilerModuleUsesRootScalaVersion() throws Exception { + File scalaCompilerPom = new File("../seatunnel-shade/seatunnel-scala-compiler/pom.xml"); + Document pom = parsePom(scalaCompilerPom); + + Assertions.assertFalse( + hasDirectProperty(pom, "scala.version"), + "seatunnel-scala-compiler must inherit scala.version from root pom.xml"); + Assertions.assertFalse( + hasDirectProperty(pom, "scala.binary.version"), + "seatunnel-scala-compiler must inherit scala.binary.version from root pom.xml"); + Assertions.assertEquals( + "${scala.version}", + findDependencyVersion(pom, "org.scala-lang", "scala-compiler"), + "scala-compiler dependency must track the root scala.version property"); + } + + private Document parsePom(File pomFile) throws Exception { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setNamespaceAware(false); + return factory.newDocumentBuilder().parse(pomFile); + } + + private boolean hasDirectProperty(Document pom, String propertyName) { + NodeList properties = pom.getDocumentElement().getChildNodes(); + for (int i = 0; i < properties.getLength(); i++) { + Node node = properties.item(i); + if ("properties".equals(node.getNodeName())) { + NodeList children = node.getChildNodes(); + for (int j = 0; j < children.getLength(); j++) { + if (propertyName.equals(children.item(j).getNodeName())) { + return true; + } + } + } + } + return false; + } + + private String findDependencyVersion(Document pom, String groupId, String artifactId) { + NodeList dependencies = pom.getElementsByTagName("dependency"); + for (int i = 0; i < dependencies.getLength(); i++) { + Node dependency = dependencies.item(i); + if (groupId.equals(childText(dependency, "groupId")) + && artifactId.equals(childText(dependency, "artifactId"))) { + return childText(dependency, "version"); + } + } + Assertions.fail("Dependency not found: " + groupId + ":" + artifactId); + return null; + } + + private String childText(Node node, String childName) { + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + if (childName.equals(child.getNodeName())) { + return child.getTextContent().trim(); + } + } + return null; + } +} diff --git a/seatunnel-shade/seatunnel-scala-compiler/pom.xml b/seatunnel-shade/seatunnel-scala-compiler/pom.xml index 58b8d9bfd268..c3d3878d5b42 100644 --- a/seatunnel-shade/seatunnel-scala-compiler/pom.xml +++ b/seatunnel-shade/seatunnel-scala-compiler/pom.xml @@ -26,11 +26,6 @@ seatunnel-scala-compiler SeaTunnel : Shade : Scala - - 2.13.11 - 2.13 - - org.scala-lang diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/dynamiccompile/parse/ScalaClassParser.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/dynamiccompile/parse/ScalaClassParser.java index e50289d903e0..042e5080ce9e 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/dynamiccompile/parse/ScalaClassParser.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/dynamiccompile/parse/ScalaClassParser.java @@ -18,7 +18,6 @@ import org.apache.seatunnel.shade.scala.tools.nsc.Settings; import org.apache.seatunnel.shade.scala.tools.nsc.interpreter.IMain; -import org.apache.seatunnel.shade.scala.tools.nsc.interpreter.shell.ReplReporterImpl; import org.apache.seatunnel.transform.exception.TransformException; @@ -38,7 +37,7 @@ public class ScalaClassParser extends AbstractParser { try { Settings settings = new Settings(); settings.usejavacp().v_$eq(true); - scalaInterpreter = new IMain(settings, new ReplReporterImpl(settings)); + scalaInterpreter = new IMain(settings); } catch (Exception e) { throw new TransformException(COMPILE_TRANSFORM_ERROR_CODE, e.getMessage()); } diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index efc70e55f593..e64fc4d6ed53 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -27,8 +27,9 @@ protostuff-collectionschema-1.8.0.jar protostuff-core-1.8.0.jar protostuff-runtime-1.8.0.jar scala-library-2.12.15.jar -scala-compiler-2.13.11.jar -scala-reflect-2.13.11.jar +scala-compiler-2.12.15.jar +scala-reflect-2.12.15.jar +scala-xml_2.12-1.0.6.jar seatunnel-scala-compiler-3.0.0-SNAPSHOT-optional.jar seatunnel-jackson-3.0.0-SNAPSHOT-optional.jar seatunnel-guava-3.0.0-SNAPSHOT-optional.jar @@ -76,7 +77,6 @@ jetty-util-9.4.56.v20240826.jar jetty-util-ajax-9.4.56.v20240826.jar javax.servlet-api-3.1.0.jar seatunnel-jetty9-9.4.56-3.0.0-SNAPSHOT-optional.jar -jna-5.13.0.jar jna-5.15.0.jar jna-platform-5.15.0.jar oshi-core-6.6.5.jar @@ -128,5 +128,3 @@ netty-common-4.1.118.Final.jar netty-handler-4.1.118.Final.jar netty-resolver-4.1.118.Final.jar eventstream-1.0.1.jar -java-diff-utils-4.12.jar -jline-3.22.0.jar From fd2a5963a42af7e9f7d4fd03b5d02889220c9999 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Tue, 23 Jun 2026 15:57:57 +0800 Subject: [PATCH 058/375] [Fix][Connector-V2] Add missing MULTI_TABLE_SINK_REPLICA option to PulsarSinkFactory (#11160) --- .../seatunnel/pulsar/sink/PulsarSink.java | 3 +-- .../pulsar/sink/PulsarSinkFactory.java | 4 +++- .../e2e/connector/pulsar/PulsarSinkIT.java | 21 +++++++++++-------- .../src/test/resources/fake_to_pulsar.conf | 3 ++- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSink.java b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSink.java index 0826feefcabf..33e84d5bd92e 100644 --- a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSink.java +++ b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSink.java @@ -70,8 +70,7 @@ public PulsarSink(ReadonlyConfig readonlyConfig, CatalogTable catalogTable) { } @Override - public SinkWriter createWriter( - SinkWriter.Context context) { + public PulsarSinkWriter createWriter(SinkWriter.Context context) { return new PulsarSinkWriter( context, clientConfig, diff --git a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactory.java b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactory.java index 5db5575c69f5..caf550218e90 100644 --- a/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactory.java +++ b/seatunnel-connectors-v2/connector-pulsar/src/main/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactory.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.options.SinkConnectorCommonOptions; import org.apache.seatunnel.api.table.connector.TableSink; import org.apache.seatunnel.api.table.factory.Factory; import org.apache.seatunnel.api.table.factory.TableSinkFactory; @@ -48,7 +49,8 @@ public OptionRule optionRule() { PulsarSinkOptions.SEMANTICS, PulsarSinkOptions.TRANSACTION_TIMEOUT, PulsarSinkOptions.PULSAR_CONFIG, - PulsarSinkOptions.PARTITION_KEY_FIELDS) + PulsarSinkOptions.PARTITION_KEY_FIELDS, + SinkConnectorCommonOptions.MULTI_TABLE_SINK_REPLICA) .conditional( PulsarSinkOptions.FORMAT, PulsarSinkOptions.TEXT_FORMAT, diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/java/org/apache/seatunnel/e2e/connector/pulsar/PulsarSinkIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/java/org/apache/seatunnel/e2e/connector/pulsar/PulsarSinkIT.java index 252fdcbd1244..9a9201f341dd 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/java/org/apache/seatunnel/e2e/connector/pulsar/PulsarSinkIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/java/org/apache/seatunnel/e2e/connector/pulsar/PulsarSinkIT.java @@ -53,7 +53,7 @@ public class PulsarSinkIT extends TestSuiteBase implements TestResource { private static final String PULSAR_IMAGE_NAME = "apachepulsar/pulsar:2.3.1"; public static final String PULSAR_HOST = "pulsar.e2e.sink"; - public static final String TOPIC = "topic-test02"; + public static final String TOPIC = "topic_test02"; private PulsarContainer pulsarContainer; @Override @@ -70,7 +70,12 @@ public void startUp() throws Exception { .ignoreExceptions() .atLeast(100, TimeUnit.MILLISECONDS) .pollInterval(500, TimeUnit.MILLISECONDS) - .atMost(180, TimeUnit.SECONDS); + .atMost(180, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertTrue( + pulsarContainer.isRunning(), + "Pulsar container should be running")); } @Override @@ -92,21 +97,19 @@ private List getPulsarConsumerData() { .subscriptionType(SubscriptionType.Exclusive) .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); - int i = 0; - while (true) { - i++; - Message msg = consumer.receive(); + for (int i = 0; i < 10; i++) { + Message msg = consumer.receive(30, TimeUnit.SECONDS); if (msg != null) { data.add(new String(msg.getData())); consumer.acknowledge(msg.getMessageId()); log.info("value:{}", new String(msg.getData())); - } - if (i == 10) { + } else { + log.warn("No message received within timeout, received {} so far", data.size()); break; } } } catch (Exception e) { - e.printStackTrace(); + throw new RuntimeException("Failed to get pulsar consumer data", e); } return data; } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/resources/fake_to_pulsar.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/resources/fake_to_pulsar.conf index f6590308aeff..5cf7e8ffb1e5 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/resources/fake_to_pulsar.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/src/test/resources/fake_to_pulsar.conf @@ -29,6 +29,7 @@ env { source { FakeSource { + plugin_output = "topic_test02" row.num = 10 map.size = 10 array.size = 10 @@ -58,7 +59,7 @@ source { sink { pulsar { - topic = "topic-test02" + topic = "topic_test02" client.service-url = "pulsar://pulsar.e2e.sink:6650" admin.service-url = "http://pulsar.e2e.sink:8080" format = json From c5460a442e8e1341b5da7d36305a905aa21fcd2e Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 23 Jun 2026 21:13:08 +0800 Subject: [PATCH 059/375] [Chore][Core] Inject GitHub token for website build workflow (#11177) --- .github/workflows/backend.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 20565bccea8f..dd7676043a6e 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -376,6 +376,9 @@ jobs: with: node-version: 18.20.7 - name: Run docusaurus build + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd seatunnel-website npm set strict-ssl false From 63a7b529c588de68d88cfdfd6ee148480f772d02 Mon Sep 17 00:00:00 2001 From: zhiliang-wu Date: Tue, 23 Jun 2026 15:27:24 +0200 Subject: [PATCH 060/375] [Feature][Connector-V2] Maxcompute Source Round-Robin Split Assignment (#11131) Co-authored-by: WU Zhiliang (External) --- .../MaxcomputeSourceSplitEnumerator.java | 61 ++++--- .../source/MaxcomputeSourceSplitTest.java | 151 ++++++++++++++---- 2 files changed, 162 insertions(+), 50 deletions(-) diff --git a/seatunnel-connectors-v2/connector-maxcompute/src/main/java/org/apache/seatunnel/connectors/seatunnel/maxcompute/source/MaxcomputeSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-maxcompute/src/main/java/org/apache/seatunnel/connectors/seatunnel/maxcompute/source/MaxcomputeSourceSplitEnumerator.java index 0f582bdd37f3..4dbd5abbd5d7 100644 --- a/seatunnel-connectors-v2/connector-maxcompute/src/main/java/org/apache/seatunnel/connectors/seatunnel/maxcompute/source/MaxcomputeSourceSplitEnumerator.java +++ b/seatunnel-connectors-v2/connector-maxcompute/src/main/java/org/apache/seatunnel/connectors/seatunnel/maxcompute/source/MaxcomputeSourceSplitEnumerator.java @@ -32,6 +32,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -108,37 +109,51 @@ public void notifyCheckpointComplete(long checkpointId) {} @Override public void handleSplitRequest(int subtaskId) {} + // visible for testing + static Set computeSplits( + int numReaders, + Collection sourceTableInfos, + Map tableRecordCounts) { + Set allSplit = new LinkedHashSet<>(); + int chunkIndex = 0; + for (SourceTableInfo sourceTableInfo : sourceTableInfos) { + TablePath tablePath = sourceTableInfo.getCatalogTable().getTablePath(); + long recordCount = tableRecordCounts.get(tablePath); + int splitRow = MaxcomputeSourceOptions.SPLIT_ROW.defaultValue(); + if (sourceTableInfo.getSplitRow() != null && sourceTableInfo.getSplitRow() > 0) { + splitRow = sourceTableInfo.getSplitRow(); + } + for (long num = 0; num < recordCount; num += splitRow) { + int ownerReader = chunkIndex % numReaders; + allSplit.add( + new MaxcomputeSourceSplit( + num, + Math.min((long) splitRow, recordCount - num), + tablePath, + ownerReader)); + chunkIndex++; + } + } + return allSplit; + } + private void discoverySplits() throws TunnelException { int numReaders = enumeratorContext.currentParallelism(); - Set allSplit = new HashSet<>(); + Map tableRecordCounts = new HashMap<>(); for (SourceTableInfo sourceTableInfo : sourceTableInfos.values()) { - Set splits = new HashSet<>(); TableTunnel.DownloadSession session = MaxcomputeUtil.getDownloadSession( readonlyConfig, sourceTableInfo.getCatalogTable().getTablePath(), sourceTableInfo.getPartitionSpec()); - long recordCount = session.getRecordCount(); - int splitRowNum = (int) Math.ceil((double) recordCount / numReaders); - int splitRow = MaxcomputeSourceOptions.SPLIT_ROW.defaultValue(); - if (sourceTableInfo.getSplitRow() != null && sourceTableInfo.getSplitRow() > 0) { - splitRow = sourceTableInfo.getSplitRow(); - } - for (int i = 0; i < numReaders; i++) { - int readerStart = i * splitRowNum; - int readerEnd = (int) Math.min((i + 1) * splitRowNum, recordCount); - for (int num = readerStart; num < readerEnd; num += splitRow) { - splits.add( - new MaxcomputeSourceSplit( - num, - Math.min(splitRow, readerEnd - num), - sourceTableInfo.getCatalogTable().getTablePath(), - i)); - } - } - assignedSplits.forEach(splits::remove); - allSplit.addAll(splits); + tableRecordCounts.put( + sourceTableInfo.getCatalogTable().getTablePath(), session.getRecordCount()); } + + Set allSplit = + computeSplits(numReaders, sourceTableInfos.values(), tableRecordCounts); + assignedSplits.forEach(allSplit::remove); + addSplitChangeToPendingAssignments(allSplit); log.debug("Assigned {} to {} readers.", allSplit, numReaders); log.info("Calculated splits successfully, the size of splits is {}.", allSplit.size()); @@ -147,7 +162,7 @@ private void discoverySplits() throws TunnelException { private void addSplitChangeToPendingAssignments(Collection newSplits) { for (MaxcomputeSourceSplit split : newSplits) { int ownerReader = split.getIndex() % enumeratorContext.currentParallelism(); - pendingSplits.computeIfAbsent(ownerReader, r -> new HashSet<>()).add(split); + pendingSplits.computeIfAbsent(ownerReader, r -> new LinkedHashSet<>()).add(split); } } diff --git a/seatunnel-connectors-v2/connector-maxcompute/src/test/java/org/apache/seatunnel/connectors/seatunnel/maxcompute/source/MaxcomputeSourceSplitTest.java b/seatunnel-connectors-v2/connector-maxcompute/src/test/java/org/apache/seatunnel/connectors/seatunnel/maxcompute/source/MaxcomputeSourceSplitTest.java index d6dfcf6eba37..83aae67404b2 100644 --- a/seatunnel-connectors-v2/connector-maxcompute/src/test/java/org/apache/seatunnel/connectors/seatunnel/maxcompute/source/MaxcomputeSourceSplitTest.java +++ b/seatunnel-connectors-v2/connector-maxcompute/src/test/java/org/apache/seatunnel/connectors/seatunnel/maxcompute/source/MaxcomputeSourceSplitTest.java @@ -17,14 +17,22 @@ package org.apache.seatunnel.connectors.seatunnel.maxcompute.source; +import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.TablePath; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class MaxcomputeSourceSplitTest { @Test @@ -33,42 +41,131 @@ public void testSplitIdUniquenessAndIndexDistribution() { long recordCount = 105000; int splitRow = 10000; - int splitRowNum = (int) Math.ceil((double) recordCount / numReaders); // 21000 TablePath tablePath = TablePath.of("test_db", "test_schema", "test_table"); + Map tableRecordCounts = new HashMap<>(); + tableRecordCounts.put(tablePath, recordCount); + + CatalogTable catalogTable = mock(CatalogTable.class); + when(catalogTable.getTablePath()).thenReturn(tablePath); + + SourceTableInfo tableInfo = new SourceTableInfo(catalogTable, null, splitRow); + List tableInfos = new ArrayList<>(); + tableInfos.add(tableInfo); + + Set splits = + MaxcomputeSourceSplitEnumerator.computeSplits( + numReaders, tableInfos, tableRecordCounts); + Set splitIds = new HashSet<>(); int[] indexCounts = new int[numReaders]; - int totalSplits = 0; - - // Simulate the inner logic of MaxcomputeSourceSplitEnumerator.discoverySplits - for (int i = 0; i < numReaders; i++) { - int readerStart = i * splitRowNum; - int readerEnd = (int) Math.min((i + 1) * splitRowNum, recordCount); - for (int num = readerStart; num < readerEnd; num += splitRow) { - MaxcomputeSourceSplit split = - new MaxcomputeSourceSplit( - num, Math.min(splitRow, readerEnd - num), tablePath, i); - - // Verify splitId uniqueness - Assertions.assertTrue( - splitIds.add(split.splitId()), - "Duplicate splitId found: " + split.splitId()); - // Verify index assigned to the split matches the reader index - Assertions.assertEquals(i, split.getIndex()); + for (MaxcomputeSourceSplit split : splits) { + Assertions.assertTrue( + splitIds.add(split.splitId()), "Duplicate splitId found: " + split.splitId()); + indexCounts[split.getIndex()]++; + } - indexCounts[i]++; - totalSplits++; - } + Assertions.assertEquals(11, splits.size()); + + // Verify index distribution (round-robin): + // 11 splits / 5 readers = 2 splits each, plus 1 remainder for reader 0. + Assertions.assertEquals(3, indexCounts[0], "Reader 0 split count mismatch!"); + Assertions.assertEquals(2, indexCounts[1], "Reader 1 split count mismatch!"); + Assertions.assertEquals(2, indexCounts[2], "Reader 2 split count mismatch!"); + Assertions.assertEquals(2, indexCounts[3], "Reader 3 split count mismatch!"); + Assertions.assertEquals(2, indexCounts[4], "Reader 4 split count mismatch!"); + } + + @Test + public void testMultiTableRoundRobinDistribution() { + int numReaders = 3; + int numTables = 5; + long recordCountPerTable = 15000; + int splitRow = 10000; + + Map tableRecordCounts = new HashMap<>(); + List tableInfos = new ArrayList<>(); + + for (int t = 0; t < numTables; t++) { + TablePath tablePath = TablePath.of("test_db", "test_schema", "test_table_" + t); + tableRecordCounts.put(tablePath, recordCountPerTable); + + CatalogTable catalogTable = mock(CatalogTable.class); + when(catalogTable.getTablePath()).thenReturn(tablePath); + + SourceTableInfo tableInfo = new SourceTableInfo(catalogTable, null, splitRow); + tableInfos.add(tableInfo); } - Assertions.assertTrue(totalSplits == 15); + Set splits = + MaxcomputeSourceSplitEnumerator.computeSplits( + numReaders, tableInfos, tableRecordCounts); + + int[] indexCounts = new int[numReaders]; - // Verify index distribution: - // Record count 105000 / 5 readers = 21000 splits per reader. - // Split row is 10000. Loop steps: 0, 10000, 20000. So 3 splits per reader. - for (int i = 0; i < numReaders; i++) { - Assertions.assertEquals(3, indexCounts[i], "Reader " + i + " split count mismatch!"); + for (MaxcomputeSourceSplit split : splits) { + indexCounts[split.getIndex()]++; } + + Assertions.assertEquals(10, splits.size()); + + // 10 splits / 3 readers = 3 splits each, plus 1 remainder for reader 0. + Assertions.assertEquals(4, indexCounts[0], "Reader 0 split count mismatch!"); + Assertions.assertEquals(3, indexCounts[1], "Reader 1 split count mismatch!"); + Assertions.assertEquals(3, indexCounts[2], "Reader 2 split count mismatch!"); + } + + @Test + public void testComputeSplitsAscendingRowStartOrder() { + int numReaders = 3; + int numTables = 2; + Map tableRecordCounts = new HashMap<>(); + List tableInfos = new ArrayList<>(); + + for (int i = 0; i < numTables; i++) { + TablePath path = TablePath.of("db", "schema", "table" + i); + // 105,000 rows / 10,000 splitRow = 11 splits per table. + // With 3 readers, each reader gets 3-4 splits PER TABLE. + tableRecordCounts.put(path, 105000L); + + CatalogTable catalogTable = mock(CatalogTable.class); + when(catalogTable.getTablePath()).thenReturn(path); + + SourceTableInfo tableInfo = new SourceTableInfo(catalogTable, null, 10000); + tableInfos.add(tableInfo); + } + + // Drive the actual production logic + Set splits = + MaxcomputeSourceSplitEnumerator.computeSplits( + numReaders, tableInfos, tableRecordCounts); + + // Guard against HashSet regression: splits iterated from the Set MUST be in insertion + // order, + // which means for any given reader and table, the rowStart must be strictly ascending. + Map> lastRowStarts = new HashMap<>(); + + for (MaxcomputeSourceSplit split : splits) { + int reader = split.getIndex(); + TablePath path = split.getTablePath(); + long rowStart = split.getRowStart(); + + lastRowStarts.computeIfAbsent(reader, r -> new HashMap<>()); + Long lastStart = lastRowStarts.get(reader).get(path); + + if (lastStart != null) { + Assertions.assertTrue( + rowStart > lastStart, + "HashSet Regression! Splits for reader " + + reader + + " and table " + + path + + " are not in ascending order!"); + } + lastRowStarts.get(reader).put(path, rowStart); + } + + Assertions.assertEquals(22, splits.size(), "Should have exactly 22 splits total"); } } From 4690b6bc1f6fb7453addcec398feffcf537f3ea1 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Tue, 23 Jun 2026 22:42:42 +0800 Subject: [PATCH 061/375] [Fix][Doc] Fix connector doc dead link (#11175) --- docs/en/connectors/overview.md | 2 +- docs/en/connectors/sink-overview.md | 6 +-- docs/en/connectors/source-overview.md | 6 +-- docs/zh/connectors/connector-faq.md | 64 +++++++++++++++++++++++++++ docs/zh/connectors/overview.md | 16 +++---- docs/zh/connectors/sink-overview.md | 6 +-- docs/zh/connectors/source-overview.md | 6 +-- 7 files changed, 85 insertions(+), 21 deletions(-) create mode 100644 docs/zh/connectors/connector-faq.md diff --git a/docs/en/connectors/overview.md b/docs/en/connectors/overview.md index c5cb7707a958..5ba4122f6a6c 100644 --- a/docs/en/connectors/overview.md +++ b/docs/en/connectors/overview.md @@ -35,5 +35,5 @@ This page is the shortest path for choosing the right SeaTunnel connector entry - [Job Configuration Guide](../getting-started/job-configuration-guide.md) - [Scenario Recipes](../getting-started/recipes/overview.md) -- [Transforms Overview](../transforms) +- [Transforms Overview](../transforms/overview.md) - [Quick Start With SeaTunnel Engine](../getting-started/locally/quick-start-seatunnel-engine.md) diff --git a/docs/en/connectors/sink-overview.md b/docs/en/connectors/sink-overview.md index c3e28f45936a..068dc8bdd88c 100644 --- a/docs/en/connectors/sink-overview.md +++ b/docs/en/connectors/sink-overview.md @@ -16,6 +16,6 @@ Use this page when your first question is "where should SeaTunnel write data?" S ## Useful Next Pages -- [Sink Common Options](../common-options/sink-common-options.md) -- [Connector FAQ](../connector-faq.md) -- [Connector Isolated Dependency Loading](../connector-isolated-dependency.md) +- [Sink Common Options](./common-options/sink-common-options.md) +- [Connector FAQ](./connector-faq.md) +- [Connector Isolated Dependency Loading](./connector-isolated-dependency.md) diff --git a/docs/en/connectors/source-overview.md b/docs/en/connectors/source-overview.md index 7d4cb295d51d..7bd9089728a5 100644 --- a/docs/en/connectors/source-overview.md +++ b/docs/en/connectors/source-overview.md @@ -16,6 +16,6 @@ Use this page when your first question is "where should SeaTunnel read data from ## Useful Next Pages -- [Source Common Options](../common-options/source-common-options.md) -- [CDC Production Cookbook](../cdc-production-cookbook.md) -- [Connector FAQ](../connector-faq.md) +- [Source Common Options](./common-options/source-common-options.md) +- [CDC Production Cookbook](./cdc-production-cookbook.md) +- [Connector FAQ](./connector-faq.md) diff --git a/docs/zh/connectors/connector-faq.md b/docs/zh/connectors/connector-faq.md new file mode 100644 index 000000000000..8f429c183d44 --- /dev/null +++ b/docs/zh/connectors/connector-faq.md @@ -0,0 +1,64 @@ +--- +sidebar_position: 10 +--- + +# 连接器常见问题 + +本页按连接器类别整理了常见问题的索引。每条目会链接到对应连接器文档页面内的 FAQ 章节。 + +这些 FAQ 章节主要用于快速导航，并非第二套权威说明。如需准确的参数名、默认值和完整示例，请以连接器页面中的参数表及链接的详细章节为准。 + +关于 SeaTunnel 的通用问题（引擎部署、变量替换、调度等），请参阅[通用常见问题](../faq.md)。 + +--- + +## CDC 连接器 + +CDC（Change Data Capture）连接器从数据库事务日志中读取实时变更事件（INSERT / UPDATE / DELETE）。 + +| 连接器 | 常见 FAQ 主题 | +|---|---| +| [MySQL CDC](./source/MySQL-CDC.md#faq) | 所需权限、binlog 配置、从库支持、无主键表、快照阶段、DDL 传播、`server-id` 冲突、快照性能、时区/字符集 | +| [PostgreSQL CDC](./source/PostgreSQL-CDC.md#faq) | 所需权限、逻辑解码插件、从库支持、无主键表、复制槽管理、复制延迟 | +| [Oracle CDC](./source/Oracle-CDC.md#faq) | LogMiner 权限、补充日志、CDB/PDB 多租户、无主键表、LogMiner 性能、支持的 Oracle 版本 | + +--- + +## 消息队列连接器 + +| 连接器 | 常见 FAQ 主题 | +|---|---| +| [Kafka Source](./source/Kafka.md#faq) | `start_mode` 选项、按消息 key 过滤、支持的格式、SASL/Kerberos 认证、消费组 offset 提交 | +| [Kafka Sink](./sink/Kafka.md#faq) | 自动创建 Topic、`partition_key_fields` 行为、精确一次投递、SASL/Kerberos 认证、支持的格式 | + +--- + +## Sink 连接器 + +### OLAP / 分析型存储 + +| 连接器 | 常见 FAQ 主题 | +|---|---| +| [Doris Sink](./sink/Doris.md#faq) | 自动建表、2PC 精确一次、"Label already exists" 错误、DELETE 传播、列名大小写、Stream Load 格式 | +| [StarRocks Sink](./sink/StarRocks.md#faq) | 自动建表、Upsert 与 DELETE 支持、`labelPrefix` 用法、列名大小写、`nodeUrls` 与 `base-url` | +| [ClickHouse Sink](./sink/Clickhouse.md#faq) | 自动建表、批量写入性能、支持的数据类型、"Table doesn't exist" 错误 | + +### 关系型数据库 + +| 连接器 | 常见 FAQ 主题 | +|---|---| +| [JDBC Sink](./sink/Jdbc.md#faq) | 自动建表、XA 事务精确一次、Upsert / 主键配置、多表写入、缺少 JDBC 驱动 | + +### 数据湖 / 文件系统 + +| 连接器 | 常见 FAQ 主题 | +|---|---| +| [Hive Sink](./sink/Hive.md#faq) | 支持的文件格式、分区表、Kerberos 认证、小文件问题、Schema 演进 | + +--- + +## 查找答案的技巧 + +1. **连接器相关问题** → 直接进入对应连接器页面，滚动到 **FAQ** 章节。 +2. **跨连接器主题**（例如「SeaTunnel 是否支持 CDC？」「`schema_save_mode` 是什么？」）→ 参阅[通用常见问题](../faq.md)。 +3. **仍未解决？** → 在 [GitHub Issues](https://github.com/apache/seatunnel/issues) 中搜索，或通过[邮件列表](https://lists.apache.org/list.html?dev@seatunnel.apache.org)联系社区。 diff --git a/docs/zh/connectors/overview.md b/docs/zh/connectors/overview.md index 956e6968cf68..4a43fadb2598 100644 --- a/docs/zh/connectors/overview.md +++ b/docs/zh/connectors/overview.md @@ -8,14 +8,14 @@ slug: /connectors ## 先按任务目标选入口 -| 你现在要做什么 | 先看这里 | -| --- | --- | -| 从外部系统读取数据 | [数据来源连接器](./source-overview.md) | -| 把数据写入目标系统 | [数据写入连接器](./sink-overview.md) | -| 先找一条接近真实业务的链路示例 | [场景示例](../getting-started/recipes/overview.md) | +| 你现在要做什么 | 先看这里 | +| --- |-----------------------------------------------------------------------------------------------------------| +| 从外部系统读取数据 | [数据来源连接器](./source-overview.md) | +| 把数据写入目标系统 | [数据写入连接器](./sink-overview.md) | +| 先找一条接近真实业务的链路示例 | [场景示例](../getting-started/recipes/overview.md) | | 先理解连接器共有参数 | [来源端通用参数](./common-options/source-common-options.md) 和 [写入端通用参数](./common-options/sink-common-options.md) | -| 构建 CDC 链路 | [CDC 生产实战手册](./cdc-production-cookbook.md) | -| 排查插件安装或依赖冲突 | [连接器常见问题](./connector-faq.md) 和 [连接器依赖隔离加载机制](./connector-isolated-dependency.md) | +| 构建 CDC 链路 | [CDC 生产实战手册](./cdc-production-cookbook.md) | +| 排查插件安装或依赖冲突 | [连接器常见问题](./connector-faq.md) 和 [连接器依赖隔离加载机制](./connector-isolated-dependency.md) | ## 新用户推荐顺序 @@ -35,5 +35,5 @@ slug: /connectors - [作业配置指南](../getting-started/job-configuration-guide.md) - [场景示例](../getting-started/recipes/overview.md) -- [数据转换总览](../transforms) +- [数据转换总览](../transforms/overview.md) - [SeaTunnel 引擎快速开始](../getting-started/locally/quick-start-seatunnel-engine.md) diff --git a/docs/zh/connectors/sink-overview.md b/docs/zh/connectors/sink-overview.md index c7dc48c2e1a3..a4c460811208 100644 --- a/docs/zh/connectors/sink-overview.md +++ b/docs/zh/connectors/sink-overview.md @@ -16,6 +16,6 @@ sidebar_position: 1 ## 常用下一步 -- [Sink 常用选项](../common-options/sink-common-options.md) -- [连接器常见问题](../connector-faq.md) -- [连接器依赖隔离加载机制](../connector-isolated-dependency.md) +- [Sink 常用选项](./common-options/sink-common-options.md) +- [连接器常见问题](./connector-faq.md) +- [连接器依赖隔离加载机制](./connector-isolated-dependency.md) diff --git a/docs/zh/connectors/source-overview.md b/docs/zh/connectors/source-overview.md index fdfa8df16c79..3759998c3820 100644 --- a/docs/zh/connectors/source-overview.md +++ b/docs/zh/connectors/source-overview.md @@ -16,6 +16,6 @@ sidebar_position: 1 ## 常用下一步 -- [Source 常用选项](../common-options/source-common-options.md) -- [CDC 生产实战手册](../cdc-production-cookbook.md) -- [连接器常见问题](../connector-faq.md) +- [Source 常用选项](./common-options/source-common-options.md) +- [CDC 生产实战手册](./cdc-production-cookbook.md) +- [连接器常见问题](./connector-faq.md) From e9c446638c55fa16fcc0bc03de8eb0eff0ae26c8 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Thu, 25 Jun 2026 20:44:26 +0800 Subject: [PATCH 062/375] [Fix][API] Fix backward compatibility issue in CatalogFactory optionRule validation (#11165) Co-authored-by: Daniel --- .../concepts/incompatible-changes.md | 7 + .../concepts/incompatible-changes.md | 7 + ...SqlServerIncrementalSourceFactoryTest.java | 33 +++++ .../jdbc/catalog/dm/DamengCatalogFactory.java | 2 +- .../catalog/duckdb/DuckDBCatalogFactory.java | 16 +-- .../catalog/highgo/HighGoCatalogFactory.java | 2 +- .../jdbc/catalog/iris/IrisCatalogFactory.java | 2 +- .../kingbase/KingbaseCatalogFactory.java | 2 +- .../catalog/mysql/MySqlCatalogFactory.java | 2 +- .../opengauss/OpenGaussCatalogFactory.java | 2 +- .../catalog/oracle/OracleCatalogFactory.java | 31 ++++- .../catalog/psql/PostgresCatalogFactory.java | 2 +- .../redshift/RedshiftCatalogFactory.java | 2 +- .../saphana/SapHanaCatalogFactory.java | 30 ++++- .../catalog/saphana/SapHanaURLParser.java | 3 +- .../sqlserver/SqlServerCatalogFactory.java | 31 ++++- .../jdbc/catalog/tidb/TiDBCatalogFactory.java | 2 +- .../jdbc/catalog/xugu/XuguCatalogFactory.java | 5 +- .../jdbc/config/JdbcCommonOptions.java | 50 ++++--- .../jdbc/catalog/JdbcCatalogFactoryTest.java | 122 ++++++++++++++++-- .../lance/catalog/LanceCatalogFactory.java | 4 + 21 files changed, 301 insertions(+), 56 deletions(-) diff --git a/docs/en/introduction/concepts/incompatible-changes.md b/docs/en/introduction/concepts/incompatible-changes.md index e5db071e740a..6ea2dc2ac1d3 100644 --- a/docs/en/introduction/concepts/incompatible-changes.md +++ b/docs/en/introduction/concepts/incompatible-changes.md @@ -61,6 +61,13 @@ You need to check this document before you upgrade to related version. ### Configuration Changes +- **Breaking Change: CatalogFactory creation path now validates `optionRule()`** + - **Affected component**: `seatunnel-api` — `FactoryUtil.createOptionalCatalog()` + - **Description**: The `FactoryUtil.createOptionalCatalog()` method now calls `ConfigValidator.validate(catalogFactory.optionRule())` before creating a catalog instance. Previously, no validation was performed on the catalog factory's option rules during catalog creation. + - **Impact**: Catalog factories whose `optionRule()` declares options as `required` that are not always present in the config passed to `createOptionalCatalog()` will now throw `OptionValidationException`. This primarily affects the JDBC connector path via `JdbcCatalogUtils.findCatalog()`. + - **Migration Guide**: If you have a custom `CatalogFactory` implementation, ensure that its `optionRule()` accurately reflects which options are truly mandatory vs optional in the config that reaches it at runtime. + + ### Connector Changes - **Breaking Change: Iceberg Connector — source table primary key is no longer silently inherited** diff --git a/docs/zh/introduction/concepts/incompatible-changes.md b/docs/zh/introduction/concepts/incompatible-changes.md index 97907ef653f4..8a6d7b0a5198 100644 --- a/docs/zh/introduction/concepts/incompatible-changes.md +++ b/docs/zh/introduction/concepts/incompatible-changes.md @@ -60,6 +60,13 @@ ### 配置变更 +- **破坏性变更：CatalogFactory 创建路径现在会校验 `optionRule()`** + - **影响范围**：`seatunnel-api` — `FactoryUtil.createOptionalCatalog()` + - **变更说明**：`FactoryUtil.createOptionalCatalog()` 方法现在在创建 catalog 实例之前会调用 `ConfigValidator.validate(catalogFactory.optionRule())` 进行校验。此前，catalog 创建路径不会对 catalog factory 的 option rules 执行任何校验。 + - **影响**：如果 catalog factory 的 `optionRule()` 将某些选项声明为 `required`，而传入 `createOptionalCatalog()` 的配置中这些选项并不总是存在，则会抛出 `OptionValidationException`。这主要影响通过 `JdbcCatalogUtils.findCatalog()` 触发的 JDBC 连接器路径。 + - **迁移指南**：如果您有自定义的 `CatalogFactory` 实现，请确保其 `optionRule()` 准确反映在运行时到达它的配置中，哪些选项是真正必填的，哪些是可选的。 + + ### 连接器变更 - **破坏性变更：Iceberg 连接器 — 不再自动继承源表主键** diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSourceFactoryTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSourceFactoryTest.java index af86c4fc1167..e814f76ee5b0 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSourceFactoryTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/SqlServerIncrementalSourceFactoryTest.java @@ -17,12 +17,45 @@ package org.apache.seatunnel.connectors.seatunnel.cdc.sqlserver.source; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.table.catalog.Catalog; +import org.apache.seatunnel.api.table.factory.FactoryUtil; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + class SqlServerIncrementalSourceFactoryTest { @Test public void testOptionRule() { Assertions.assertNotNull((new SqlServerIncrementalSourceFactory()).optionRule()); } + + /** + * SQLServer CDC source creation must accept the driver-specific databaseName URL syntax during + * the submission-time catalog validation step. + */ + @Test + public void testCreateOptionalCatalogWithSqlServerStyleUrl() { + Map config = new HashMap<>(); + config.put("url", "jdbc:sqlserver://localhost:1433;databaseName=seatunnel"); + config.put("username", "sa"); + config.put("password", "Password!"); + config.put("database-names", Arrays.asList("seatunnel")); + config.put("table-names", Arrays.asList("seatunnel.dbo.orders")); + + Optional catalog = + FactoryUtil.createOptionalCatalog( + "SqlServer", + ReadonlyConfig.fromMap(config), + Thread.currentThread().getContextClassLoader(), + "SqlServer"); + + Assertions.assertTrue(catalog.isPresent()); + catalog.ifPresent(Catalog::close); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalogFactory.java index ead206a230ff..5494d1b14735 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalogFactory.java @@ -51,6 +51,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/duckdb/DuckDBCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/duckdb/DuckDBCatalogFactory.java index 53999ace8fea..b6232fe2c58b 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/duckdb/DuckDBCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/duckdb/DuckDBCatalogFactory.java @@ -28,13 +28,6 @@ import com.google.auto.service.AutoService; -import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.DECIMAL_TYPE_NARROWING; -import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.HANDLE_BLOB_AS_STRING; -import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.PASSWORD; -import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.SCHEMA; -import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.URL; -import static org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions.USERNAME; - /** Factory for {@link DuckDBCatalog} */ @AutoService(Factory.class) public class DuckDBCatalogFactory implements CatalogFactory { @@ -58,8 +51,13 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig config) { @Override public OptionRule optionRule() { return OptionRule.builder() - .required(URL) - .optional(USERNAME, PASSWORD, SCHEMA, DECIMAL_TYPE_NARROWING, HANDLE_BLOB_AS_STRING) + .required(JdbcCommonOptions.URL) + .optional( + JdbcCommonOptions.USERNAME, + JdbcCommonOptions.PASSWORD, + JdbcCommonOptions.SCHEMA, + JdbcCommonOptions.DECIMAL_TYPE_NARROWING, + JdbcCommonOptions.HANDLE_BLOB_AS_STRING) .build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/highgo/HighGoCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/highgo/HighGoCatalogFactory.java index e61254107588..caa8e59b1011 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/highgo/HighGoCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/highgo/HighGoCatalogFactory.java @@ -45,6 +45,6 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalogFactory.java index 33ed340f2790..e951b07695f7 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalogFactory.java @@ -50,6 +50,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/kingbase/KingbaseCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/kingbase/KingbaseCatalogFactory.java index b535ce5f50f3..9453115d137a 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/kingbase/KingbaseCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/kingbase/KingbaseCatalogFactory.java @@ -51,6 +51,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalogFactory.java index 052f8763ab93..537b8c768a77 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalogFactory.java @@ -51,6 +51,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/opengauss/OpenGaussCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/opengauss/OpenGaussCatalogFactory.java index c80b163e740c..bffb1800bf67 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/opengauss/OpenGaussCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/opengauss/OpenGaussCatalogFactory.java @@ -51,6 +51,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogFactory.java index 782130bd9642..bd2794a7516e 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogFactory.java @@ -17,8 +17,12 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oracle; +import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; + import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -53,6 +57,31 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule(new OracleUrlValidator()).build(); + } + + static class OracleUrlValidator implements ConditionExtension { + @Override + public String description() { + return "Oracle JDBC URL must contain a service name (e.g. jdbc:oracle:thin:@host:port/service)"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String url) { + if (url == null || url.trim().isEmpty()) { + return false; + } + try { + JdbcUrlUtil.UrlInfo info = OracleURLParser.parse(url); + return StringUtils.isNotBlank(info.getHost()) + && info.getDefaultDatabase().isPresent(); + } catch (IllegalArgumentException e) { + throw new OptionValidationException( + String.format( + "Invalid Oracle JDBC URL format: [%s], " + + "expected pattern: jdbc:oracle:thin:@host:port/service", + url)); + } + } } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogFactory.java index 7217432eb6d1..3215eace0200 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogFactory.java @@ -51,6 +51,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalogFactory.java index 7931ef382636..728bc4897749 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalogFactory.java @@ -51,6 +51,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalogFactory.java index ba7dd361b649..0e1f73abaea8 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalogFactory.java @@ -17,8 +17,12 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.saphana; +import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; + import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -51,6 +55,30 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule(new SapHanaUrlValidator()).build(); + } + + static class SapHanaUrlValidator implements ConditionExtension { + @Override + public String description() { + return "SAP HANA JDBC URL must be a valid format (e.g. jdbc:sap://host:port)"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String url) { + if (url == null || url.trim().isEmpty()) { + return false; + } + try { + JdbcUrlUtil.UrlInfo info = SapHanaURLParser.parse(url); + return StringUtils.isNotBlank(info.getHost()); + } catch (IllegalArgumentException e) { + throw new OptionValidationException( + String.format( + "Invalid SAP HANA JDBC URL format: [%s], " + + "expected pattern: jdbc:sap://host:port", + url)); + } + } } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaURLParser.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaURLParser.java index 6c6e66fbfefd..704947d481d1 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaURLParser.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaURLParser.java @@ -25,7 +25,8 @@ public class SapHanaURLParser { private static final Pattern HANA_URL_PATTERN = - Pattern.compile("^(?jdbc:sap://(?[^:]+):(?\\d+)/\\?(?.*?))$"); + Pattern.compile( + "^(?jdbc:sap://(?[^:]+):(?\\d+)(/\\?(?.*?))?)$"); public static JdbcUrlUtil.UrlInfo parse(String url) { Matcher matcher = HANA_URL_PATTERN.matcher(url); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sqlserver/SqlServerCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sqlserver/SqlServerCatalogFactory.java index aa345997d279..0624347c3e92 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sqlserver/SqlServerCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sqlserver/SqlServerCatalogFactory.java @@ -17,8 +17,12 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.sqlserver; +import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; + import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; @@ -51,6 +55,31 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule(new SqlServerUrlValidator()).build(); + } + + /** Validates URL format only; database may be provided via separate config option. */ + static class SqlServerUrlValidator implements ConditionExtension { + @Override + public String description() { + return "SqlServer JDBC URL must be a valid format (e.g. jdbc:sqlserver://host:port;databaseName=db)"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String url) { + if (url == null || url.trim().isEmpty()) { + return false; + } + try { + JdbcUrlUtil.UrlInfo info = SqlServerURLParser.parse(url); + return info != null && StringUtils.isNotBlank(info.getHost()); + } catch (IllegalArgumentException e) { + throw new OptionValidationException( + String.format( + "Invalid SqlServer JDBC URL format: [%s], " + + "expected pattern: jdbc:sqlserver://host:port[;databaseName=db]", + url)); + } + } } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalogFactory.java index beb2945b8762..153062851e8f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalogFactory.java @@ -50,6 +50,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalogFactory.java index 63f35bfd358c..cd239e7e9ba5 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalogFactory.java @@ -23,7 +23,6 @@ import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; import org.apache.seatunnel.common.utils.JdbcUrlUtil; -import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oracle.OracleURLParser; import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier; @@ -40,7 +39,7 @@ public String factoryIdentifier() { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig options) { String urlWithDatabase = options.get(JdbcCommonOptions.URL); - JdbcUrlUtil.UrlInfo urlInfo = OracleURLParser.parse(urlWithDatabase); + JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(urlWithDatabase); return new XuguCatalog( catalogName, options.get(JdbcCommonOptions.USERNAME), @@ -52,6 +51,6 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) { @Override public OptionRule optionRule() { - return JdbcCommonOptions.BASE_CATALOG_RULE.build(); + return JdbcCommonOptions.baseCatalogRule().build(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcCommonOptions.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcCommonOptions.java index 2c636be5c4b6..bff276da2230 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcCommonOptions.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcCommonOptions.java @@ -17,12 +17,15 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.config; +import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; + import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.Options; import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.ConditionExtension; import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.common.utils.JdbcUrlUtil; import java.util.Map; @@ -162,38 +165,42 @@ public class JdbcCommonOptions { public static final Option REGION = Options.key("region").stringType().noDefaultValue().withDescription("region"); - /** @deprecated Use {@link #baseCatalogRule()} instead to avoid shared mutable state. */ - @Deprecated public static final OptionRule.Builder BASE_CATALOG_RULE = baseCatalogRule(); - /** - * Returns a fresh {@link OptionRule.Builder} with the base validation rules shared by all JDBC - * catalog factories (MySQL, PostgreSQL, Oracle, etc.). + * Returns a fresh {@link OptionRule.Builder} with the base validation rules shared by standard + * JDBC catalog factories (MySQL, PostgreSQL, etc.) that use generic {@code host:port/database} + * URL format and require username/password authentication. * - *

These rules are evaluated at submission time via {@code - * ConfigValidator.validate(factory.optionRule())} in the {@code FactoryUtil} entry path. They - * enforce that the JDBC URL contains a database name, and that username/password are provided. + *

Catalog factories with non-standard URL formats (SqlServer, Oracle, SapHana) should use + * {@link #baseCatalogRule(ConditionExtension)} with their own URL validator. * - *

Individual catalog factories may append additional rules (e.g. OceanBase requires {@code - * compatible_mode}) before calling {@code .build()}. + *

Catalog factories that do not require authentication (e.g. DuckDB) should define their own + * {@code optionRule()} directly. */ public static OptionRule.Builder baseCatalogRule() { + return baseCatalogRule(new UrlContainsDatabaseValidator()); + } + + /** + * Returns a fresh {@link OptionRule.Builder} with a custom URL validator. Use this for + * databases whose JDBC URL does not follow the standard {@code host:port/database} format (e.g. + * SqlServer, Oracle, SapHana). + */ + public static OptionRule.Builder baseCatalogRule(ConditionExtension urlValidator) { return OptionRule.builder() - .required(URL, Conditions.extension(URL, new UrlContainsDatabaseValidator())) + .required(URL, Conditions.extension(URL, urlValidator)) .required(USERNAME, PASSWORD) .optional(SCHEMA, DECIMAL_TYPE_NARROWING, HANDLE_BLOB_AS_STRING); } /** - * Submission-time validator that ensures the JDBC URL contains a database name. - * - *

This validator is attached to the {@code url} option via {@link - * Conditions#extension(Option, ConditionExtension)} and is evaluated by {@code ConfigValidator} - * before the catalog/source/sink factory creates its connector instance. + * Validates that the JDBC URL has a valid format with at least a host component. Database name + * is optional to maintain backward compatibility with connectors (e.g. StarRocks, Doris) that + * specify the database in the query or table_path instead of the URL. */ public static class UrlContainsDatabaseValidator implements ConditionExtension { @Override public String description() { - return "JDBC URL must contain a database name"; + return "JDBC URL must be a valid format: jdbc:://host:port[/database]"; } @Override @@ -202,9 +209,14 @@ public boolean evaluate(ReadonlyConfig config, String url) { return false; } try { - return JdbcUrlUtil.getUrlInfo(url).getDefaultDatabase().isPresent(); + JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(url); + return StringUtils.isNotBlank(urlInfo.getHost()); } catch (IllegalArgumentException e) { - return false; + throw new OptionValidationException( + String.format( + "Invalid JDBC URL format: [%s], " + + "expected pattern: jdbc:://host:port[/database]", + url)); } } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/JdbcCatalogFactoryTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/JdbcCatalogFactoryTest.java index 5aadf30058d6..2b827e3248b8 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/JdbcCatalogFactoryTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/JdbcCatalogFactoryTest.java @@ -21,9 +21,14 @@ import org.apache.seatunnel.api.configuration.util.ConfigValidator; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.dm.DamengCatalogFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.duckdb.DuckDBCatalogFactory; import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.mysql.MySqlCatalogFactory; import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oceanbase.OceanBaseCatalogFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oracle.OracleCatalogFactory; import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.psql.PostgresCatalogFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.saphana.SapHanaCatalogFactory; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.sqlserver.SqlServerCatalogFactory; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -40,6 +45,8 @@ private void validate(OptionRule rule, Map config) { ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); } + // ==================== Standard URL validators (MySQL / Postgres) ==================== + @Test void testValidCatalogConfig() { Map cfg = new HashMap<>(); @@ -68,39 +75,42 @@ void testPostgresCatalogValidConfig() { } @Test - void testUrlWithoutDatabaseFails() { + void testBlankUrlFails() { Map cfg = new HashMap<>(); - cfg.put("url", "jdbc:mysql://host:3306"); + cfg.put("url", ""); cfg.put("username", "root"); cfg.put("password", "pass"); Assertions.assertThrows(OptionValidationException.class, () -> validate(mysqlRule, cfg)); } @Test - void testBlankUrlFails() { + void testMissingCredentialsFails() { Map cfg = new HashMap<>(); - cfg.put("url", ""); - cfg.put("username", "root"); - cfg.put("password", "pass"); + cfg.put("url", "jdbc:mysql://localhost:3306/mydb"); Assertions.assertThrows(OptionValidationException.class, () -> validate(mysqlRule, cfg)); } @Test - void testMissingUsernameFails() { + void testMissingPasswordFails() { Map cfg = new HashMap<>(); - cfg.put("url", "jdbc:mysql://host:3306/mydb"); - cfg.put("password", "pass"); + cfg.put("url", "jdbc:mysql://localhost:3306/mydb"); + cfg.put("username", "root"); Assertions.assertThrows(OptionValidationException.class, () -> validate(mysqlRule, cfg)); } @Test - void testMissingPasswordFails() { + void testCatalogConfigMimicsExtractCatalogConfig() { Map cfg = new HashMap<>(); - cfg.put("url", "jdbc:mysql://host:3306/mydb"); + cfg.put("url", "jdbc:mysql://localhost:3306/mydb"); cfg.put("username", "root"); - Assertions.assertThrows(OptionValidationException.class, () -> validate(mysqlRule, cfg)); + cfg.put("password", "pass"); + cfg.put("decimal_type_narrowing", true); + cfg.put("handle_blob_as_string", false); + Assertions.assertDoesNotThrow(() -> validate(mysqlRule, cfg)); } + // ==================== OceanBase ==================== + @Test void testOceanBaseWithoutCompatibleModeFails() { OptionRule obRule = new OceanBaseCatalogFactory().optionRule(); @@ -110,4 +120,92 @@ void testOceanBaseWithoutCompatibleModeFails() { cfg.put("password", "pass"); Assertions.assertThrows(OptionValidationException.class, () -> validate(obRule, cfg)); } + + @Test + void testOceanBaseValidConfig() { + OptionRule obRule = new OceanBaseCatalogFactory().optionRule(); + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:oceanbase://localhost:2881/mydb"); + cfg.put("username", "root"); + cfg.put("password", "pass"); + cfg.put("compatible_mode", "mysql"); + Assertions.assertDoesNotThrow(() -> validate(obRule, cfg)); + } + + // ==================== Dameng (no database in URL) ==================== + + @Test + void testDamengCatalogUrlWithoutDatabase() { + OptionRule dmRule = new DamengCatalogFactory().optionRule(); + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:dm://e2e_dmdb:5236"); + cfg.put("username", "SYSDBA"); + cfg.put("password", "SYSDBA"); + Assertions.assertDoesNotThrow(() -> validate(dmRule, cfg)); + } + + // ==================== DuckDB (no credentials required) ==================== + + @Test + void testDuckDBCatalogConfigNoCredentials() { + OptionRule rule = new DuckDBCatalogFactory().optionRule(); + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:duckdb:/tmp/test.db"); + Assertions.assertDoesNotThrow(() -> validate(rule, cfg)); + } + + // ==================== SqlServer dialect validator ==================== + + @Test + void testSqlServerCatalogUrlWithDatabaseNameProperty() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:sqlserver://localhost:1433;databaseName=seatunnel"); + cfg.put("username", "sa"); + cfg.put("password", "Password!"); + Assertions.assertDoesNotThrow( + () -> validate(new SqlServerCatalogFactory().optionRule(), cfg)); + } + + @Test + void testSqlServerCatalogUrlWithoutDatabasePasses() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:sqlserver://localhost:1433;encrypt=false"); + cfg.put("username", "sa"); + cfg.put("password", "Password!"); + Assertions.assertDoesNotThrow( + () -> validate(new SqlServerCatalogFactory().optionRule(), cfg)); + } + + // ==================== Oracle dialect validator ==================== + + @Test + void testOracleCatalogThinUrlWithoutDoubleSlash() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:oracle:thin:@localhost:1521/ORCLCDB"); + cfg.put("username", "system"); + cfg.put("password", "oracle"); + Assertions.assertDoesNotThrow(() -> validate(new OracleCatalogFactory().optionRule(), cfg)); + } + + // ==================== SapHana dialect validator ==================== + + @Test + void testSapHanaCatalogHostOnlyUrl() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:sap://localhost:39017"); + cfg.put("username", "SYSTEM"); + cfg.put("password", "Password1"); + Assertions.assertDoesNotThrow( + () -> validate(new SapHanaCatalogFactory().optionRule(), cfg)); + } + + @Test + void testSapHanaCatalogWithDatabaseParam() { + Map cfg = new HashMap<>(); + cfg.put("url", "jdbc:sap://localhost:39017/?databaseName=HXE"); + cfg.put("username", "SYSTEM"); + cfg.put("password", "Password1"); + Assertions.assertDoesNotThrow( + () -> validate(new SapHanaCatalogFactory().optionRule(), cfg)); + } } diff --git a/seatunnel-connectors-v2/connector-lance/src/main/java/org/apache/seatunnel/connectors/seatunnel/lance/catalog/LanceCatalogFactory.java b/seatunnel-connectors-v2/connector-lance/src/main/java/org/apache/seatunnel/connectors/seatunnel/lance/catalog/LanceCatalogFactory.java index 7d26b4e8ef13..a96529bcd6c3 100644 --- a/seatunnel-connectors-v2/connector-lance/src/main/java/org/apache/seatunnel/connectors/seatunnel/lance/catalog/LanceCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-lance/src/main/java/org/apache/seatunnel/connectors/seatunnel/lance/catalog/LanceCatalogFactory.java @@ -21,7 +21,11 @@ import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; +import org.apache.seatunnel.api.table.factory.Factory; +import com.google.auto.service.AutoService; + +@AutoService(Factory.class) public class LanceCatalogFactory implements CatalogFactory { @Override public Catalog createCatalog(String catalogName, ReadonlyConfig readonlyConfig) { From e7309efb4aa09886680da60112bbf62cdcf53bad Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 25 Jun 2026 21:47:58 +0800 Subject: [PATCH 063/375] [Fix][Connector-V2] Add Pulsar multi-table sink replica option (#11168) --- .../pulsar/sink/PulsarSinkFactoryTest.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactoryTest.java b/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactoryTest.java index 69ba2888f2c1..4859f2aa390a 100644 --- a/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactoryTest.java +++ b/seatunnel-connectors-v2/connector-pulsar/src/test/java/org/apache/seatunnel/connectors/seatunnel/pulsar/sink/PulsarSinkFactoryTest.java @@ -18,6 +18,8 @@ package org.apache.seatunnel.connectors.seatunnel.pulsar.sink; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.options.SinkConnectorCommonOptions; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; @@ -25,7 +27,9 @@ import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext; import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.connectors.seatunnel.pulsar.config.PulsarSinkOptions; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -33,16 +37,14 @@ import java.util.List; import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; - +/** Verifies the Pulsar sink factory metadata required by connector specification checks. */ public class PulsarSinkFactoryTest { @Test public void testCreateSinkRequiresTopicForSingleTable() { PulsarSinkFactory factory = new PulsarSinkFactory(); - assertThrows( + Assertions.assertThrows( IllegalArgumentException.class, () -> factory.createSink( @@ -54,13 +56,32 @@ public void testCreateSinkRequiresTopicForSingleTable() { public void testCreateSinkAllowsMissingTopicForMultiTable() { PulsarSinkFactory factory = new PulsarSinkFactory(); - assertDoesNotThrow( + Assertions.assertDoesNotThrow( () -> factory.createSink( new TableSinkFactoryContext( null, config(), getClass().getClassLoader()))); } + /** Ensures the factory still exposes the documented Pulsar identifier. */ + @Test + void factoryIdentifier() { + PulsarSinkFactory pulsarSinkFactory = new PulsarSinkFactory(); + Assertions.assertEquals( + PulsarSinkOptions.IDENTIFIER, pulsarSinkFactory.factoryIdentifier()); + } + + /** Guards the option metadata that connector specification checks validate in CI. */ + @Test + void optionRuleContainsMultiTableReplica() { + PulsarSinkFactory pulsarSinkFactory = new PulsarSinkFactory(); + OptionRule optionRule = pulsarSinkFactory.optionRule(); + Assertions.assertTrue( + optionRule + .getOptionalOptions() + .contains(SinkConnectorCommonOptions.MULTI_TABLE_SINK_REPLICA)); + } + private ReadonlyConfig config() { Map options = new HashMap<>(); options.put("client.service-url", "pulsar://localhost:6650"); From 634330a790f25c625da6bb8a5ff94f7ed91d1e8c Mon Sep 17 00:00:00 2001 From: Jast Date: Sat, 27 Jun 2026 14:26:02 +0800 Subject: [PATCH 064/375] [Fix][CI] Use workflow token for backend website build (#11192) --- .github/workflows/backend.yml | 201 +++++++++++---------- .github/workflows/build_main.yml | 1 + .github/workflows/notify_test_workflow.yml | 32 ++-- .github/workflows/schedule_backend.yml | 3 +- 4 files changed, 121 insertions(+), 116 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index dd7676043a6e..3c6378ffcedd 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -38,7 +38,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Check license header @@ -49,7 +49,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Check code style @@ -94,8 +94,11 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Helm - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 - id: install + run: | + if ! command -v helm >/dev/null 2>&1; then + curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + fi + helm version - name: Lint Chart run: helm lint deploy/kubernetes/seatunnel @@ -106,7 +109,7 @@ jobs: # Temporarily ignore this job to avoid blocking PRs continue-on-error: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - run: sudo npm install -g markdown-link-check@3.8.7 - run: | for file in $(find . -name "*.md"); do @@ -146,7 +149,7 @@ jobs: /usr/bin/git checkout apache/dev /usr/bin/git checkout '${{ github.ref }}' echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11.0' - name: Check for file changes by python @@ -289,9 +292,9 @@ jobs: - name: Make integration test modules id: it-modules timeout-minutes: 60 - if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.cv2-e2e-modules.outputs.modules != '' || steps.cv2-flink-e2e-modules.outputs.modules != '' || steps.cv2-spark-e2e-modules.outputs.modules != '') }} + if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.cv2-e2e-modules.outputs.modules != '' || steps.engine-e2e-modules.outputs.modules != '') }} run: | - modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' + modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' modules=${modules: 1} pl_modules=`python tools/update_modules_check/update_modules_check.py replace "$modules"` # remove deleted modules @@ -307,8 +310,6 @@ jobs: engine_modules='${{ steps.engine-modules.outputs.modules }}' connector_modules='${{ steps.cv2-modules.outputs.modules }}' connector_modules="$connector_modules"'${{ steps.cv2-e2e-modules.outputs.modules }}' - connector_modules="$connector_modules"'${{ steps.cv2-flink-e2e-modules.outputs.modules }}' - connector_modules="$connector_modules"'${{ steps.cv2-spark-e2e-modules.outputs.modules }}' engine_e2e_modules='${{ steps.engine-e2e-modules.outputs.modules }}' if [[ "zz${connector_modules}${engine_e2e_modules}" == "zz" && "zz"$engine_modules != "zz" ]];then # Engine changes already trigger the broad downstream integration jobs through engine=true. @@ -334,16 +335,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '8' cache: 'maven' - name: Install - uses: nick-fields/retry@v2 + uses: nick-fields/retry@v3 with: timeout_minutes: 40 max_attempts: 3 @@ -361,24 +362,24 @@ jobs: timeout-minutes: 90 steps: - name: Checkout PR - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: seatunnel-pr - name: Checkout website repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: apache/seatunnel-website path: seatunnel-website - name: Sync PR changes to website run: | bash seatunnel-pr/tools/documents/sync.sh seatunnel-pr seatunnel-website - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v4 with: node-version: 18.20.7 - name: Run docusaurus build env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ github.token }} run: | cd seatunnel-website npm set strict-ssl false @@ -393,8 +394,8 @@ jobs: timeout-minutes: 60 steps: - name: Checkout PR - uses: actions/checkout@v3 - - uses: actions/setup-node@v2 + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: 20.x - name: Install Dependencies and Check Code Style @@ -421,9 +422,9 @@ jobs: os: [ 'ubuntu-latest', 'windows-latest' ] timeout-minutes: 90 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -444,9 +445,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -475,9 +476,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -506,9 +507,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -537,9 +538,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 200 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -567,9 +568,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -597,9 +598,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -627,9 +628,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -658,9 +659,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -689,9 +690,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 150 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -722,11 +723,11 @@ jobs: cp /etc/rancher/k3s/k3s.yaml ~/.kube/config env: KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: free disk space run: tools/github/free_disk_space.sh - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -751,9 +752,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -779,9 +780,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 150 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -807,9 +808,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -838,9 +839,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -876,9 +877,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -907,9 +908,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 270 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -938,9 +939,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 270 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -969,9 +970,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1000,9 +1001,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1031,9 +1032,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1060,9 +1061,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1088,9 +1089,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1116,9 +1117,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1144,9 +1145,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1172,9 +1173,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1200,9 +1201,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1228,9 +1229,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1256,9 +1257,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1282,9 +1283,9 @@ jobs: # Kudu E2E expands each @TestTemplate case across several PR test containers. timeout-minutes: 90 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1307,9 +1308,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1332,9 +1333,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1357,9 +1358,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1383,9 +1384,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1408,9 +1409,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1434,9 +1435,9 @@ jobs: timeout-minutes: 210 steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1463,9 +1464,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1488,9 +1489,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1513,9 +1514,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1538,9 +1539,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml index f4816940a888..d5110c50a458 100644 --- a/.github/workflows/build_main.yml +++ b/.github/workflows/build_main.yml @@ -27,6 +27,7 @@ on: jobs: call-build-and-test: permissions: + contents: read packages: write name: Run uses: ./.github/workflows/backend.yml diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml index a517a03c168a..2a0d1729f35d 100644 --- a/.github/workflows/notify_test_workflow.yml +++ b/.github/workflows/notify_test_workflow.yml @@ -36,7 +36,7 @@ jobs: checks: write steps: - name: "Notify test workflow" - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -109,31 +109,33 @@ jobs: } }) } else { - const run_id = runs.data.workflow_runs[0].id + const build_run = runs.data.workflow_runs[0] + const run_id = build_run.id - if (runs.data.workflow_runs[0].head_sha != context.payload.pull_request.head.sha) { + if (build_run.head_sha != context.payload.pull_request.head.sha) { throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.'); } // Here we get check run ID to provide Check run view instead of Actions view, see also SPARK-37879. const check_runs = await github.request(check_run_endpoint, check_run_params) console.log('check_runs: ' + JSON.stringify(check_runs)) - const check_run_head = check_runs.data.check_runs.filter(r => r.name === "Run / License header")[0] - - console.log('check_run_head: ' + JSON.stringify(check_run_head)) - if (check_run_head.head_sha != context.payload.pull_request.head.sha) { - throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.'); - } - - const check_run_url = 'https://github.com/' - + context.payload.pull_request.head.repo.full_name - + '/runs/' - + check_run_head.id - const actions_url = 'https://github.com/' + context.payload.pull_request.head.repo.full_name + '/actions/runs/' + run_id + const run_url_part = '/actions/runs/' + run_id + const check_run_head = check_runs.data.check_runs.find(r => + r.head_sha === build_run.head_sha + && ( + (r.details_url && r.details_url.includes(run_url_part)) + || (r.html_url && r.html_url.includes(run_url_part)) + ) + ) + + console.log('check_run_head: ' + JSON.stringify(check_run_head)) + const check_run_url = check_run_head + ? (check_run_head.html_url || check_run_head.details_url || actions_url) + : actions_url await github.rest.checks.create({ owner: context.repo.owner, diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml index 9a6aa1cb90a2..b513ba62874d 100644 --- a/.github/workflows/schedule_backend.yml +++ b/.github/workflows/schedule_backend.yml @@ -27,8 +27,9 @@ concurrency: jobs: call-build-and-test: permissions: + contents: read packages: write name: Run uses: ./.github/workflows/backend.yml with: - TEST_IN_PR: false + TEST_IN_PR: ${{ 'false' }} From b90b066ce6f0c892bcf2d084c13242d9a1954a8e Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 27 Jun 2026 17:15:53 +0800 Subject: [PATCH 065/375] Add heye1005 in ASF collaborators (#11199) --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index 1b02ecc60c0d..4445166e3c5d 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -39,7 +39,7 @@ github: - CosmosNi - fcb-xiaobo - LeonYoah - - silenceland + - heye1005 - SEZ9 - boy-xiaozhang - nzw921rx From 2b16eedb97171f058053a462b63a93bee96d674d Mon Sep 17 00:00:00 2001 From: Jast Date: Sat, 27 Jun 2026 20:50:04 +0800 Subject: [PATCH 066/375] [Docs] Fix inconsistent transform documentation (#11197) --- docs/en/transforms/encrypt.md | 33 ++++++++++++++++++---------- docs/en/transforms/jsonpath.md | 25 ++++++++++++--------- docs/en/transforms/split.md | 7 +++--- docs/en/transforms/sql.md | 6 ++++- docs/zh/transforms/data-validator.md | 2 +- docs/zh/transforms/encrypt.md | 21 +++++++++++++----- docs/zh/transforms/jsonpath.md | 24 ++++++++++++-------- docs/zh/transforms/split.md | 5 ++--- docs/zh/transforms/sql.md | 6 ++++- 9 files changed, 82 insertions(+), 47 deletions(-) diff --git a/docs/en/transforms/encrypt.md b/docs/en/transforms/encrypt.md index add8af07e9fd..611f828464cc 100644 --- a/docs/en/transforms/encrypt.md +++ b/docs/en/transforms/encrypt.md @@ -8,12 +8,13 @@ The Encrypt transform plugin is used to encrypt or decrypt specified fields in r ## Options -| name | type | required | default value | description | -|-------------|--------|----------|---------------|-----------------------------------| -| `fields` | Array | Yes | - | List of fields to encrypt/decrypt | -| `algorithm` | String | No | `AES_GCM` | Encryption algorithm | -| `key` | String | Yes | - | Base64-encoded encryption key | -| `mode` | String | No | `ENCRYPT` | `ENCRYPT`or `DECRYPT` | +| name | type | required | default value | description | +|--------------------|---------|----------|---------------|--------------------------------------------------| +| `fields` | Array | Yes | - | List of fields to encrypt/decrypt | +| `algorithm` | String | No | `AES_GCM` | Encryption algorithm | +| `key` | String | Yes | - | Base64-encoded encryption key | +| `mode` | String | No | `encrypt` | `encrypt` or `decrypt` | +| `max_field_length` | Integer | No | `10485760` | Maximum string field length before processing | ### algorithm [string] @@ -37,30 +38,38 @@ For both `AES_GCM` and `AES_CBC`, valid key lengths are 16, 24, or 32 bytes (cor - `base64:AAAAAAAAAAAAAAAAAAAAAA==` - `AAAAAAAAAAAAAAAAAAAAAA==` +### mode [string] + +The transform mode. Supported values are `encrypt` and `decrypt`. The comparison is case-insensitive, but new examples should use lowercase values to match the default. + +### max_field_length [int] + +The maximum string length that can be encrypted or decrypted for each configured field. If a field value exceeds this limit, the transform fails fast instead of processing an unexpectedly large value. + ### common options [string] Transform plugin common parameters, please refer to [Transform Plugin](common-options/common-options.md) for details ## Example -``` +```hocon transform { FieldEncrypt { - fields = ["name"] + fields = ["name"] key = "base64:AAAAAAAAAAAAAAAAAAAAAA==" algorithm = "AES_CBC" - mode = "ENCRYPT" + mode = "encrypt" } } ``` -``` +```hocon transform { FieldEncrypt { - fields = ["name"] + fields = ["name"] key = "base64:AAAAAAAAAAAAAAAAAAAAAA==" algorithm = "AES_CBC" - mode = "DECRYPT" + mode = "decrypt" } } ``` diff --git a/docs/en/transforms/jsonpath.md b/docs/en/transforms/jsonpath.md index 3a99e247b2b6..8c866335a129 100644 --- a/docs/en/transforms/jsonpath.md +++ b/docs/en/transforms/jsonpath.md @@ -28,13 +28,13 @@ This option is used to specify the processing method when an error occurs in the #### option -| name | type | required | default value | -|-------------------------|--------|----------|---------------| -| src_field | String | Yes | | -| dest_field | String | Yes | | -| path | String | Yes | | -| dest_type | String | No | String | -| column_error_handle_way | Enum | No | | +| name | type | required | default value | +|-------------------------|-----------------|----------|---------------| +| src_field | String | Yes | | +| dest_field | String or Array | Yes | | +| path | String or Array | Yes | | +| dest_type | String or Array | No | String | +| column_error_handle_way | Enum | No | | #### src_field @@ -52,14 +52,20 @@ Support SeatunnelDateType > after use jsonpath output field +This can be a single field name or an array of field names when extracting multiple values from the same source field. + #### dest_type > the type of dest field +This can be a single type or an array of types. If omitted for a single output field, the default type is `string`. + #### path > Jsonpath +This can be a single JSONPath expression or an array of JSONPath expressions. + #### column_error_handle_way [Enum] This option is used to specify the processing method when an error occurs in the column. @@ -91,7 +97,7 @@ The data read from source is a table like this json: Assuming we want to use JsonPath to extract properties. -```json +```hocon transform { JsonPath { plugin_input = "fake" @@ -180,7 +186,7 @@ transform { } ``` -**Important:** When using batch field extraction (multiple paths, dest_fields, and dest_types), the `dest_type` parameter is **required** and cannot be omitted. Each extracted field must have a corresponding type specified. The array format provides better readability and is less error-prone than string-based configurations. +**Important:** When using batch field extraction, `path`, `dest_field`, and `dest_type` must have the same number of items. If you omit `dest_type`, the transform uses the single default type `string`, so multiple output fields should provide a `dest_type` array explicitly. Then the data result table `fake1` will like this @@ -325,4 +331,3 @@ transform { ## Changelog * Add JsonPath Transform - diff --git a/docs/en/transforms/split.md b/docs/en/transforms/split.md index d56cf08cae7d..de7d5a084783 100644 --- a/docs/en/transforms/split.md +++ b/docs/en/transforms/split.md @@ -16,7 +16,7 @@ Split a field to more than one field. ### separator [string] -The list of fields that need to be kept. Fields not in the list will be deleted +The delimiter used to split the source field. ### split_field [string] @@ -41,7 +41,7 @@ The data read from source is a table like this: | Kin Dom | 20 | 123 | | Joy Dom | 20 | 123 | -We want split `name` field to `first_name` and `second name`, we can add `Split` transform like this +We want to split the `name` field into `first_name` and `last_name`, so we can add a `Split` transform like this: ``` transform { @@ -50,7 +50,7 @@ transform { plugin_output = "fake1" separator = " " split_field = "name" - output_fields = [first_name, second_name] + output_fields = [first_name, last_name] } } ``` @@ -69,4 +69,3 @@ Then the data in result table `fake1` will like this ### new version - Add Split Transform Connector - diff --git a/docs/en/transforms/sql.md b/docs/en/transforms/sql.md index 4c5dfd7c205b..9173cb96454a 100644 --- a/docs/en/transforms/sql.md +++ b/docs/en/transforms/sql.md @@ -15,6 +15,7 @@ SQL transform use memory SQL engine, we can via SQL functions and ability of SQL | plugin_input | string | yes | - | | plugin_output | string | yes | - | | query | string | yes | - | +| engine | string | no | ZETA | ### plugin_input [string] @@ -27,6 +28,10 @@ The query SQL, it's a simple SQL supported base function and criteria filter ope the query expression can be `select [table_name.]column_a` to query the column that named `column_a`. and the table name is optional. or `select c_row.c_inner_row.column_b` to query the inline struct column that named `column_b` within `c_row` column and `c_inner_row` column. **In this query expression, can't have table name.** +### engine [string] + +The SQL engine used by this transform. Supported values are `ZETA` and `INTERNAL`. If this option is not configured, `ZETA` is used. + ## Example The data read from source is a table like this: @@ -157,4 +162,3 @@ sink { ### new version - Add SQL Transform Connector - diff --git a/docs/zh/transforms/data-validator.md b/docs/zh/transforms/data-validator.md index a4c02a9c6b4b..28ee4bd8af44 100644 --- a/docs/zh/transforms/data-validator.md +++ b/docs/zh/transforms/data-validator.md @@ -10,7 +10,7 @@ DataValidator 转换插件会根据配置规则校验字段值，并按照指定 | 名称 | 类型 | 是否必需 | 默认值 | |-----------------|--------|----------|--------| -| error_handle_way| enum | 否 | FAIL | +| row_error_handle_way| enum | 否 | FAIL | | row_error_handle_way.error_table | string | 否 | | | field_rules | array | 是 | | diff --git a/docs/zh/transforms/encrypt.md b/docs/zh/transforms/encrypt.md index 4829ae253768..e51a4af9025e 100644 --- a/docs/zh/transforms/encrypt.md +++ b/docs/zh/transforms/encrypt.md @@ -8,12 +8,13 @@ FieldEncrypt 转换插件用于使用对称加密算法，对记录中的指定 ## 参数说明 -| 参数名 | 类型 | 是否必填 | 默认值 | 描述 | -|-------------|--------|------|-----------|----------------------------| -| `fields` | Array | 是 | - | 需要加密或解密的字段列表 | -| `algorithm` | String | 否 | `AES_CBC` | 加密算法 | -| `key` | String | 是 | - | Base64 编码的加密密钥 | -| `mode` | String | 否 | `ENCRYPT` | 操作模式：`ENCRYPT` 或 `DECRYPT` | +| 参数名 | 类型 | 是否必填 | 默认值 | 描述 | +|------------------|---------|------|------------|----------------------| +| `fields` | Array | 是 | - | 需要加密或解密的字段列表 | +| `algorithm` | String | 否 | `AES_GCM` | 加密算法 | +| `key` | String | 是 | - | Base64 编码的加密密钥 | +| `mode` | String | 否 | `encrypt` | 操作模式：`encrypt` 或 `decrypt` | +| `max_field_length` | Integer | 否 | `10485760` | 处理前允许的最大字符串字段长度 | ### algorithm [string] @@ -39,6 +40,14 @@ FieldEncrypt 转换插件用于使用对称加密算法，对记录中的指定 - `base64:AAAAAAAAAAAAAAAAAAAAAA==` - `AAAAAAAAAAAAAAAAAAAAAA==` +### mode [string] + +转换模式。支持 `encrypt` 和 `decrypt`。代码会忽略大小写，但新配置建议使用和默认值一致的小写写法。 + +### max_field_length [int] + +每个配置字段在加密或解密前允许的最大字符串长度。如果字段值超过该限制，Transform 会直接失败，避免处理异常大的字段值。 + ### common options [string] Transform 插件的通用参数，请参考 [Transform Plugin](common-options/common-options.md)。 diff --git a/docs/zh/transforms/jsonpath.md b/docs/zh/transforms/jsonpath.md index 02f7110db78a..e01c473f66ce 100644 --- a/docs/zh/transforms/jsonpath.md +++ b/docs/zh/transforms/jsonpath.md @@ -28,13 +28,13 @@ JsonPath 转换插件支持使用 JSONPath 选择数据。 #### 属性 -| 名称 | 类型 | 是否必须 | 默认值 | -|-------------------------|--------|------|--------| -| src_field | String | Yes | | -| dest_field | String | Yes | | -| path | String | Yes | | -| dest_type | String | No | String | -| column_error_handle_way | Enum | No | | +| 名称 | 类型 | 是否必须 | 默认值 | +|-------------------------|-----------------|------|--------| +| src_field | String | Yes | | +| dest_field | String or Array | Yes | | +| path | String or Array | Yes | | +| dest_type | String or Array | No | String | +| column_error_handle_way | Enum | No | | #### src_field @@ -52,14 +52,20 @@ JsonPath 转换插件支持使用 JSONPath 选择数据。 > 使用 JSONPath 后的输出字段 +可以是单个字段名；当需要从同一个源字段提取多个值时，也可以配置为字段名数组。 + #### dest_type > 目标字段的类型 +可以是单个类型；也可以在批量提取时配置为类型数组。单字段提取时如果省略，默认使用 `string`。 + #### path > Jsonpath +可以是单个 JSONPath 表达式，也可以是 JSONPath 表达式数组。 + #### column_error_handle_way [Enum] 该选项用于指定当列发生错误时的处理方式。 @@ -91,7 +97,7 @@ JsonPath 转换插件支持使用 JSONPath 选择数据。 假设我们想要使用 JsonPath 提取属性。 -```json +```hocon transform { JsonPath { plugin_input = "fake" @@ -179,7 +185,7 @@ transform { } } ``` -**重要提示：** 当使用批量字段提取（多个 paths、dest_fields 和 dest_types）时，`dest_type` 参数是必填的，不能省略。每个提取的字段都必须指定一个对应的类型。数组格式提供了更好的可读性，比基于字符串的配置更不容易出错。 +**重要提示：** 使用批量字段提取时，`path`、`dest_field` 和 `dest_type` 的数组长度必须一致。如果省略 `dest_type`，Transform 只会使用单个默认类型 `string`，因此多个输出字段应显式配置 `dest_type` 数组。 那么数据结果表 `fake1` 将会像这样 diff --git a/docs/zh/transforms/split.md b/docs/zh/transforms/split.md index 2c851a59b1c0..f30b946d0514 100644 --- a/docs/zh/transforms/split.md +++ b/docs/zh/transforms/split.md @@ -41,7 +41,7 @@ | Kin Dom | 20 | 123 | | Joy Dom | 20 | 123 | -我们想要将 `name` 字段拆分为 `first_name` 和 `second_name`，我们可以像这样添加 `Split` 转换： +我们想要将 `name` 字段拆分为 `first_name` 和 `last_name`，可以像这样添加 `Split` 转换： ``` transform { @@ -50,7 +50,7 @@ transform { plugin_output = "fake1" separator = " " split_field = "name" - output_fields = [first_name, second_name] + output_fields = [first_name, last_name] } } ``` @@ -69,4 +69,3 @@ transform { ### 新版本 - 添加拆分转换连接器 - diff --git a/docs/zh/transforms/sql.md b/docs/zh/transforms/sql.md index cc9ce8b2d66e..28957034286d 100644 --- a/docs/zh/transforms/sql.md +++ b/docs/zh/transforms/sql.md @@ -15,6 +15,7 @@ SQL 转换使用内存中的 SQL 引擎，我们可以通过 SQL 函数和 SQL | plugin_input | string | yes | - | | plugin_output | string | yes | - | | query | string | yes | - | +| engine | string | no | ZETA | ### plugin_input [string] @@ -27,6 +28,10 @@ SQL 转换使用内存中的 SQL 引擎，我们可以通过 SQL 函数和 SQL 查询表达式可以是`select [table_name.]column_a`，这时会去查询列为`column_a`的列，`table_name`为可选项 也可以是`select c_row.c_inner_row.column_b`，这时会去查询列`c_row`下的`c_inner_row`的`column_b`。**嵌套结构查询中，不能存在`table_name`** +### engine [string] + +该 Transform 使用的 SQL 引擎。支持 `ZETA` 和 `INTERNAL`。如果不配置，默认使用 `ZETA`。 + ## 示例 源端数据读取的表格如下： @@ -155,4 +160,3 @@ sink { ### 新版本 - 添加SQL转换连接器 - From 0f48aef4ffd8f19a77a1071e6efade3124149e3f Mon Sep 17 00:00:00 2001 From: Jast Date: Sat, 27 Jun 2026 20:50:08 +0800 Subject: [PATCH 067/375] [Fix][Connector-V2] Fix Hudi null struct field conversion (#11196) --- .../sink/convert/AvroSchemaConverter.java | 18 +++-- .../sink/convert/HudiRecordConverter.java | 14 +--- .../connectors/seatunnel/hudi/HudiTest.java | 65 +++++++++++++++++++ 3 files changed, 82 insertions(+), 15 deletions(-) diff --git a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/AvroSchemaConverter.java b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/AvroSchemaConverter.java index acb102127573..2f9ac4f2a751 100644 --- a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/AvroSchemaConverter.java +++ b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/AvroSchemaConverter.java @@ -64,6 +64,11 @@ public static Schema convertToSchema(SeaTunnelDataType schema) { * @return Avro's {@link Schema} matching this logical type. */ public static Schema convertToSchema(SeaTunnelDataType dataType, String rowName) { + return convertToSchema(dataType, rowName, false); + } + + private static Schema convertToSchema( + SeaTunnelDataType dataType, String rowName, boolean nullableRow) { switch (dataType.getSqlType()) { case BOOLEAN: Schema bool = SchemaBuilder.builder().booleanType(); @@ -126,25 +131,30 @@ public static Schema convertToSchema(SeaTunnelDataType dataType, String rowNa SeaTunnelDataType fieldType = rowType.getFieldType(i); SchemaBuilder.GenericDefault fieldBuilder = builder.name(fieldName) - .type(convertToSchema(fieldType, rowName + "." + fieldName)); + .type( + convertToSchema( + fieldType, rowName + "." + fieldName, true)); builder = fieldBuilder.withDefault(null); } - return builder.endRecord(); + Schema record = builder.endRecord(); + return nullableRow ? nullableSchema(record) : record; case MAP: Schema map = SchemaBuilder.builder() .map() .values( convertToSchema( - extractValueTypeToAvroMap(dataType), rowName)); + extractValueTypeToAvroMap(dataType), + rowName, + true)); return nullableSchema(map); case ARRAY: ArrayType arrayType = (ArrayType) dataType; Schema array = SchemaBuilder.builder() .array() - .items(convertToSchema(arrayType.getElementType(), rowName)); + .items(convertToSchema(arrayType.getElementType(), rowName, true)); return nullableSchema(array); default: throw new UnsupportedOperationException( diff --git a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/HudiRecordConverter.java b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/HudiRecordConverter.java index e8d23ab4eff4..0cc8ba557a3e 100644 --- a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/HudiRecordConverter.java +++ b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/HudiRecordConverter.java @@ -24,7 +24,6 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.avro.AvroSchemaUtils; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieKey; @@ -39,7 +38,6 @@ import java.util.UUID; import java.util.stream.Collectors; -import static org.apache.seatunnel.connectors.seatunnel.hudi.sink.convert.AvroSchemaConverter.convertToSchema; import static org.apache.seatunnel.connectors.seatunnel.hudi.sink.convert.RowDataToAvroConverters.createConverter; public class HudiRecordConverter implements Serializable { @@ -59,17 +57,11 @@ public HoodieRecord convertRow( HudiTableConfig hudiTableConfig) { GenericRecord rec = new GenericData.Record(schema); for (int i = 0; i < seaTunnelRowType.getTotalFields(); i++) { + String fieldName = seaTunnelRowType.getFieldNames()[i]; rec.put( - seaTunnelRowType.getFieldNames()[i], + fieldName, createConverter(seaTunnelRowType.getFieldType(i)) - .convert( - convertToSchema( - seaTunnelRowType.getFieldType(i), - AvroSchemaUtils.getAvroRecordQualifiedName( - hudiTableConfig.getTableName()) - + "." - + seaTunnelRowType.getFieldNames()[i]), - element.getField(i))); + .convert(schema.getField(fieldName).schema(), element.getField(i))); } return new HoodieAvroRecord<>( getHoodieKey(element, seaTunnelRowType, hudiTableConfig), diff --git a/seatunnel-connectors-v2/connector-hudi/src/test/java/org/apache/seatunnel/connectors/seatunnel/hudi/HudiTest.java b/seatunnel-connectors-v2/connector-hudi/src/test/java/org/apache/seatunnel/connectors/seatunnel/hudi/HudiTest.java index 7dbfc402b6f3..633b5e69aa18 100644 --- a/seatunnel-connectors-v2/connector-hudi/src/test/java/org/apache/seatunnel/connectors/seatunnel/hudi/HudiTest.java +++ b/seatunnel-connectors-v2/connector-hudi/src/test/java/org/apache/seatunnel/connectors/seatunnel/hudi/HudiTest.java @@ -23,10 +23,15 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.hudi.config.HudiTableConfig; +import org.apache.seatunnel.connectors.seatunnel.hudi.sink.convert.HudiRecordConverter; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.EncoderFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hudi.avro.AvroSchemaUtils; import org.apache.hudi.client.HoodieJavaWriteClient; @@ -37,6 +42,7 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; @@ -53,6 +59,7 @@ import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.io.TempDir; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.sql.Timestamp; @@ -127,6 +134,45 @@ void testSchema() { getSchema()); } + @Test + void testConvertNullableStructField() throws IOException { + SeaTunnelRowType structType = + new SeaTunnelRowType( + new String[] {"field1", "field2"}, + new SeaTunnelDataType[] {STRING_TYPE, INT_TYPE}); + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"id", "payload"}, + new SeaTunnelDataType[] {INT_TYPE, structType}); + Schema schema = + convertToSchema(rowType, AvroSchemaUtils.getAvroRecordQualifiedName(tableName)); + + Assertions.assertEquals(Schema.Type.RECORD, schema.getType()); + Assertions.assertEquals(Schema.Type.UNION, schema.getField("payload").schema().getType()); + + SeaTunnelRow nullStructRow = new SeaTunnelRow(2); + nullStructRow.setField(0, 1); + nullStructRow.setField(1, null); + + GenericRecord nullStructRecord = convertToAvroRecord(schema, rowType, nullStructRow); + Assertions.assertNull(nullStructRecord.get("payload")); + writeAvroRecord(schema, nullStructRecord); + + SeaTunnelRow struct = new SeaTunnelRow(2); + struct.setField(0, "value"); + struct.setField(1, 100); + + SeaTunnelRow structRow = new SeaTunnelRow(2); + structRow.setField(0, 2); + structRow.setField(1, struct); + + GenericRecord structRecord = convertToAvroRecord(schema, rowType, structRow); + GenericRecord payload = (GenericRecord) structRecord.get("payload"); + Assertions.assertEquals("value", payload.get("field1").toString()); + Assertions.assertEquals(100, payload.get("field2")); + writeAvroRecord(schema, structRecord); + } + @Test @DisabledOnOs(OS.WINDOWS) void testWriteData() throws IOException { @@ -210,6 +256,25 @@ private HoodieRecord convertRow(SeaTunnelRow element) { getHoodieKey(element, seaTunnelRowType), new HoodieAvroPayload(Option.of(rec))); } + private GenericRecord convertToAvroRecord( + Schema schema, SeaTunnelRowType rowType, SeaTunnelRow element) throws IOException { + HudiTableConfig tableConfig = + HudiTableConfig.builder() + .tableName(tableName) + .opType(WriteOperationType.INSERT) + .build(); + HoodieRecord record = + new HudiRecordConverter().convertRow(schema, rowType, element, tableConfig); + return (GenericRecord) record.getData().getInsertValue(schema).get(); + } + + private void writeAvroRecord(Schema schema, GenericRecord record) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); + new GenericDatumWriter(schema).write(record, encoder); + encoder.flush(); + } + private HoodieKey getHoodieKey(SeaTunnelRow element, SeaTunnelRowType seaTunnelRowType) { String partitionPath = getRecordPartitionPath(element, seaTunnelRowType); String rowKey = getRecordKey(element, seaTunnelRowType); From 7a591a04115a2dc15660e6774546cf8b679f47f4 Mon Sep 17 00:00:00 2001 From: Shuai Liu <390105636@qq.com> Date: Sat, 27 Jun 2026 21:10:49 +0800 Subject: [PATCH 068/375] [Improve][Connector-V2][Redis] Align OptionRule with docs and dedupe config validation (#11195) --- .../redis/config/RedisTableConfig.java | 101 +++++------------- .../redis/sink/RedisSinkFactory.java | 4 +- .../redis/source/RedisSourceFactory.java | 1 - .../redis/config/RedisTableConfigTest.java | 97 +++++++++++++++++ 4 files changed, 126 insertions(+), 77 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-redis/src/test/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisTableConfigTest.java diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisTableConfig.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisTableConfig.java index 525106d96b12..2f90ff5aadbd 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisTableConfig.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisTableConfig.java @@ -114,44 +114,44 @@ private static TablePath getTablePath(ReadonlyConfig tableConfig, String keys) { * @return List of RedisTableConfig */ public static List of(ReadonlyConfig config) { - // Check if using multi-table mode + // Multi-table mode: each table_configs item is parsed as an independent table. if (config.getOptional(RedisBaseOptions.TABLE_CONFIGS).isPresent()) { List> tableConfigMaps = config.get(RedisBaseOptions.TABLE_CONFIGS); return tableConfigMaps.stream() .map(ReadonlyConfig::fromMap) - .map(RedisTableConfig::buildFromConfig) + .map(RedisTableConfig::buildTableConfig) .collect(Collectors.toList()); - } else { - // Single table mode (backward compatibility) - return Collections.singletonList(buildSingleTableConfig(config)); } + // Single-table mode: top-level options describe one table. + return Collections.singletonList(buildTableConfig(config)); } /** - * Build RedisTableConfig from ReadonlyConfig (for multi-table mode). + * Build a fully initialized {@link RedisTableConfig} from a single table-level {@link + * ReadonlyConfig}. Shared by both single-table (top-level config) and multi-table (each {@code + * tables_configs} item) modes. * - * @param tableConfig ReadonlyConfig for a single table (table-level config) + * @param config ReadonlyConfig describing one table * @return Fully initialized RedisTableConfig with runtime objects */ - private static RedisTableConfig buildFromConfig(ReadonlyConfig tableConfig) { - // Validate required fields - validateRequiredFields(tableConfig); + private static RedisTableConfig buildTableConfig(ReadonlyConfig config) { + // Validate required fields are present + validateRequiredFields(config); // Build catalog table and deserialization schema - TableConfigResult result = - buildCatalogTableAndSchema(tableConfig, tableConfig.get(KEY_PATTERN)); + TableConfigResult result = buildCatalogTableAndSchema(config, config.get(KEY_PATTERN)); return RedisTableConfig.builder() - .keys(tableConfig.get(KEY_PATTERN)) - .dataType(tableConfig.get(DATA_TYPE)) - .batchSize(tableConfig.get(BATCH_SIZE)) - .format(tableConfig.get(FORMAT)) - .schema(tableConfig.getOptional(ConnectorCommonOptions.SCHEMA).orElse(null)) - .hashKeyParseMode(tableConfig.get(HASH_KEY_PARSE_MODE)) - .readKeyEnabled(tableConfig.get(READ_KEY_ENABLED)) + .keys(config.get(KEY_PATTERN)) + .dataType(config.get(DATA_TYPE)) + .batchSize(config.get(BATCH_SIZE)) + .format(config.get(FORMAT)) + .schema(config.getOptional(ConnectorCommonOptions.SCHEMA).orElse(null)) + .hashKeyParseMode(config.get(HASH_KEY_PARSE_MODE)) + .readKeyEnabled(config.get(READ_KEY_ENABLED)) .keyFieldName(result.keyFieldName) - .singleFieldName(tableConfig.getOptional(SINGLE_FIELD_NAME).orElse(null)) - .fieldDelimiter(tableConfig.get(FIELD_DELIMITER)) + .singleFieldName(config.getOptional(SINGLE_FIELD_NAME).orElse(null)) + .fieldDelimiter(config.get(FIELD_DELIMITER)) .tablePath(result.tablePath) .catalogTable(result.catalogTable) .deserializationSchema(result.deserializationSchema) @@ -159,52 +159,19 @@ private static RedisTableConfig buildFromConfig(ReadonlyConfig tableConfig) { } /** - * Build a single table configuration from global config (backward compatibility). - * - * @param config ReadonlyConfig instance - * @return RedisTableConfig - */ - private static RedisTableConfig buildSingleTableConfig(ReadonlyConfig config) { - // Validate that required fields for single table mode are present - validateRequiredFields(config); - - // Build catalog table and deserialization schema - TableConfigResult result = buildCatalogTableAndSchema(config, config.get(KEY_PATTERN)); - - RedisTableConfig tableConfig = - RedisTableConfig.builder() - .keys(config.get(KEY_PATTERN)) - .dataType(config.get(DATA_TYPE)) - .batchSize(config.get(BATCH_SIZE)) - .format(config.get(FORMAT)) - .schema(config.getOptional(ConnectorCommonOptions.SCHEMA).orElse(null)) - .hashKeyParseMode(config.get(HASH_KEY_PARSE_MODE)) - .readKeyEnabled(config.get(READ_KEY_ENABLED)) - .keyFieldName(result.keyFieldName) - .singleFieldName(config.getOptional(SINGLE_FIELD_NAME).orElse(null)) - .fieldDelimiter(config.get(FIELD_DELIMITER)) - .tablePath(result.tablePath) - .catalogTable(result.catalogTable) - .deserializationSchema(result.deserializationSchema) - .build(); - - validateTableConfig(tableConfig); - return tableConfig; - } - - /** - * Validate required fields in configuration. + * Validate required fields in a (table-level) configuration. * * @param config ReadonlyConfig to validate */ private static void validateRequiredFields(ReadonlyConfig config) { - if (!config.getOptional(KEY_PATTERN).isPresent()) { + String keys = config.getOptional(KEY_PATTERN).orElse(null); + if (keys == null || keys.trim().isEmpty()) { throw new IllegalArgumentException( - "Redis table configuration requires 'keys' parameter. "); + "Redis table configuration requires 'keys' parameter."); } if (!config.getOptional(DATA_TYPE).isPresent()) { throw new IllegalArgumentException( - "Redis table configuration requires 'data_type' parameter. "); + "Redis table configuration requires 'data_type' parameter."); } } @@ -264,20 +231,4 @@ private static TableConfigResult buildCatalogTableAndSchema( return result; } - - /** - * Validate a single table configuration. - * - * @param tableConfig RedisTableConfig to validate - */ - private static void validateTableConfig(RedisTableConfig tableConfig) { - if (tableConfig.getKeys() == null || tableConfig.getKeys().trim().isEmpty()) { - throw new IllegalArgumentException( - "Redis configuration must specify 'keys' parameter."); - } - if (tableConfig.getDataType() == null) { - throw new IllegalArgumentException( - "Redis configuration must specify 'data_type' parameter."); - } - } } diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/sink/RedisSinkFactory.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/sink/RedisSinkFactory.java index 53c5b62b220a..2bfbd09733a6 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/sink/RedisSinkFactory.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/sink/RedisSinkFactory.java @@ -50,8 +50,10 @@ public OptionRule optionRule() { RedisBaseOptions.MODE, RedisBaseOptions.AUTH, RedisBaseOptions.USER, - RedisBaseOptions.KEY_PATTERN, + RedisBaseOptions.DB_NUM, + RedisBaseOptions.BATCH_SIZE, RedisBaseOptions.FORMAT, + RedisBaseOptions.FIELD_DELIMITER, RedisSinkOptions.EXPIRE, RedisSinkOptions.SUPPORT_CUSTOM_KEY, RedisSinkOptions.VALUE_FIELD, diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java index 4007cc939524..ad4dd6a79cfb 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java @@ -54,7 +54,6 @@ public OptionRule optionRule() { RedisSourceOptions.HASH_KEY_PARSE_MODE, RedisBaseOptions.AUTH, RedisBaseOptions.USER, - RedisBaseOptions.KEY, RedisSourceOptions.READ_KEY_ENABLED, RedisSourceOptions.SINGLE_FIELD_NAME, RedisSourceOptions.KEY_FIELD_NAME, diff --git a/seatunnel-connectors-v2/connector-redis/src/test/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisTableConfigTest.java b/seatunnel-connectors-v2/connector-redis/src/test/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisTableConfigTest.java new file mode 100644 index 000000000000..8d9edfb35f20 --- /dev/null +++ b/seatunnel-connectors-v2/connector-redis/src/test/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisTableConfigTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.redis.config; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class RedisTableConfigTest { + + private static ReadonlyConfig config(Map map) { + return ReadonlyConfig.fromMap(map); + } + + @Test + void singleTableBuildsOneTable() { + Map map = new HashMap<>(); + map.put("keys", "key_test*"); + map.put("data_type", "string"); + + List tables = RedisTableConfig.of(config(map)); + + Assertions.assertEquals(1, tables.size()); + Assertions.assertEquals("key_test*", tables.get(0).getKeys()); + Assertions.assertEquals(RedisDataType.STRING, tables.get(0).getDataType()); + } + + @Test + void singleTableMissingDataTypeThrows() { + Map map = new HashMap<>(); + map.put("keys", "key_test*"); + + Assertions.assertThrows( + IllegalArgumentException.class, () -> RedisTableConfig.of(config(map))); + } + + @Test + void singleTableBlankKeysThrows() { + Map map = new HashMap<>(); + map.put("keys", " "); + map.put("data_type", "string"); + + Assertions.assertThrows( + IllegalArgumentException.class, () -> RedisTableConfig.of(config(map))); + } + + @Test + void multiTableAppliesRequiredFieldValidationPerItem() { + Map tableItem = new HashMap<>(); + tableItem.put("keys", "key_test*"); + Map map = new HashMap<>(); + map.put("tables_configs", Collections.singletonList(tableItem)); + + Assertions.assertThrows( + IllegalArgumentException.class, () -> RedisTableConfig.of(config(map))); + } + + @Test + void multiTableBuildsEachItem() { + Map t1 = new HashMap<>(); + t1.put("keys", "k1*"); + t1.put("data_type", "string"); + Map t2 = new HashMap<>(); + t2.put("keys", "k2*"); + t2.put("data_type", "hash"); + Map map = new HashMap<>(); + map.put("tables_configs", Arrays.asList(t1, t2)); + + List tables = RedisTableConfig.of(config(map)); + + Assertions.assertEquals(2, tables.size()); + Assertions.assertEquals(RedisDataType.STRING, tables.get(0).getDataType()); + Assertions.assertEquals(RedisDataType.HASH, tables.get(1).getDataType()); + } +} From 02aa837898c821961abbef8ea567bfe2bcea2d2f Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Sat, 27 Jun 2026 21:10:53 +0800 Subject: [PATCH 069/375] [Improve][Connector-V2] Migrate Elasticsearch validation to declarative OptionRule (#11122) --- .../catalog/ElasticSearchCatalogFactory.java | 49 ++- .../client/auth/ApiKeyAuthProvider.java | 29 +- .../auth/ApiKeyEncodedAuthProvider.java | 32 +- .../client/auth/BasicAuthProvider.java | 38 +- .../config/ElasticsearchValidators.java | 66 ++++ .../sink/ElasticsearchSinkFactory.java | 20 +- .../source/ElasticsearchSource.java | 20 +- .../source/ElasticsearchSourceFactory.java | 50 ++- .../ElasticSearchCatalogFactoryTest.java | 136 +++++++ .../sink/ElasticsearchSinkFactoryTest.java | 217 +++++++++++ .../ElasticsearchSourceFactoryTest.java | 342 ++++++++++++++++++ 11 files changed, 896 insertions(+), 103 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/config/ElasticsearchValidators.java create mode 100644 seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchCatalogFactoryTest.java create mode 100644 seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/sink/ElasticsearchSinkFactoryTest.java create mode 100644 seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSourceFactoryTest.java diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchCatalogFactory.java b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchCatalogFactory.java index 76623f84fc6b..ecf463c01137 100644 --- a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchCatalogFactory.java +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchCatalogFactory.java @@ -18,13 +18,30 @@ package org.apache.seatunnel.connectors.seatunnel.elasticsearch.catalog; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.factory.CatalogFactory; import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.AuthTypeEnum; +import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchValidators.ApiKeyEncodedFormatValidator; import com.google.auto.service.AutoService; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.API_KEY; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.API_KEY_ENCODED; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.API_KEY_ID; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.AUTH_TYPE; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.HOSTS; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.PASSWORD; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.TLS_KEY_STORE_PASSWORD; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.TLS_KEY_STORE_PATH; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.TLS_TRUST_STORE_PASSWORD; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.TLS_TRUST_STORE_PATH; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.TLS_VERIFY_CERTIFICATE; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.TLS_VERIFY_HOSTNAME; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.USERNAME; + @AutoService(Factory.class) public class ElasticSearchCatalogFactory implements CatalogFactory { @@ -40,6 +57,36 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { - return OptionRule.builder().build(); + return OptionRule.builder() + .required(HOSTS) + .optional( + USERNAME, + PASSWORD, + TLS_VERIFY_CERTIFICATE, + TLS_VERIFY_HOSTNAME, + TLS_KEY_STORE_PATH, + TLS_KEY_STORE_PASSWORD, + TLS_TRUST_STORE_PATH, + TLS_TRUST_STORE_PASSWORD) + .optional(AUTH_TYPE) + .conditionalRule( + AUTH_TYPE, + AuthTypeEnum.BASIC, + OptionRule.builder().bundled(USERNAME, PASSWORD).build()) + .conditional(AUTH_TYPE, AuthTypeEnum.BASIC, Conditions.notBlank(USERNAME)) + .conditional(AUTH_TYPE, AuthTypeEnum.BASIC, Conditions.notBlank(PASSWORD)) + .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, API_KEY_ID, API_KEY) + .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, Conditions.notBlank(API_KEY_ID)) + .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, Conditions.notBlank(API_KEY)) + .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY_ENCODED, API_KEY_ENCODED) + .conditional( + AUTH_TYPE, + AuthTypeEnum.API_KEY_ENCODED, + Conditions.notBlank(API_KEY_ENCODED)) + .conditional( + AUTH_TYPE, + AuthTypeEnum.API_KEY_ENCODED, + Conditions.extension(API_KEY_ENCODED, new ApiKeyEncodedFormatValidator())) + .build(); } } diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/ApiKeyAuthProvider.java b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/ApiKeyAuthProvider.java index bbd5def6317f..f89a1324d496 100644 --- a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/ApiKeyAuthProvider.java +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/ApiKeyAuthProvider.java @@ -62,20 +62,14 @@ public String getAuthType() { return AUTH_TYPE; } + /** + * No-op. Presence and non-blankness of {@code auth.api_key_id} / {@code auth.api_key} are now + * enforced declaratively via {@code OptionRule} (conditional on {@code auth_type=api_key}). + * This method is kept to honor the {@link AuthenticationProvider} contract. + */ @Override public void validate(ReadonlyConfig config) { - Optional apiKeyId = config.getOptional(ElasticsearchBaseOptions.API_KEY_ID); - Optional apiKey = config.getOptional(ElasticsearchBaseOptions.API_KEY); - Optional apiKeyEncoded = - config.getOptional(ElasticsearchBaseOptions.API_KEY_ENCODED); - - if (!apiKeyId.isPresent() || !apiKey.isPresent()) { - throw new IllegalArgumentException( - "API key authentication with auth_type='api_key' requires both api_key_id and api_key"); - } - validateApiKeyIdAndSecret(apiKeyId.get(), apiKey.get()); - - log.debug("API key authentication configuration validated"); + // intentionally empty - validation handled by OptionRule } /** @@ -95,15 +89,4 @@ private String getEncodedApiKey(ReadonlyConfig config) { return null; } - - /** Validate API key ID and secret. */ - private void validateApiKeyIdAndSecret(String apiKeyId, String apiKey) { - if (apiKeyId == null || apiKeyId.trim().isEmpty()) { - throw new IllegalArgumentException("API key ID cannot be null or empty"); - } - - if (apiKey == null || apiKey.trim().isEmpty()) { - throw new IllegalArgumentException("API key cannot be null or empty"); - } - } } diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/ApiKeyEncodedAuthProvider.java b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/ApiKeyEncodedAuthProvider.java index 200bc13705ac..d3910303f6ef 100644 --- a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/ApiKeyEncodedAuthProvider.java +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/ApiKeyEncodedAuthProvider.java @@ -24,8 +24,6 @@ import lombok.extern.slf4j.Slf4j; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import java.util.Optional; @Slf4j @@ -66,34 +64,6 @@ public String getAuthType() { @Override public void validate(ReadonlyConfig config) { - Optional apiKeyEncoded = - config.getOptional(ElasticsearchBaseOptions.API_KEY_ENCODED); - if (!apiKeyEncoded.isPresent()) { - throw new IllegalArgumentException( - "API key authentication with auth_type='api_key_encoded' requires api_key_encoded"); - } - validateEncodedApiKey(apiKeyEncoded.get()); - - log.debug("Encoded API key authentication configuration validated"); - } - - /** Validate encoded API key. */ - private void validateEncodedApiKey(String apiKeyEncoded) { - if (apiKeyEncoded == null || apiKeyEncoded.trim().isEmpty()) { - throw new IllegalArgumentException("Encoded API key cannot be null or empty"); - } - - try { - byte[] decoded = Base64.getDecoder().decode(apiKeyEncoded); - String decodedStr = new String(decoded, StandardCharsets.UTF_8); - - if (!decodedStr.contains(":")) { - throw new IllegalArgumentException( - "Encoded API key must be Base64 encoded 'id:key' format"); - } - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - "Invalid encoded API key format: " + e.getMessage(), e); - } + // intentionally empty - validation handled by OptionRule } } diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/BasicAuthProvider.java b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/BasicAuthProvider.java index 7e4d0ce010f7..ee169ad1fc80 100644 --- a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/BasicAuthProvider.java +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/auth/BasicAuthProvider.java @@ -60,38 +60,14 @@ public String getAuthType() { return AUTH_TYPE; } + /** + * No-op. Username/password presence and pairing are now enforced declaratively via {@code + * OptionRule} (see {@code ElasticsearchSourceFactory} / {@code ElasticsearchSinkFactory} / + * {@code ElasticSearchCatalogFactory}). This method is kept to honor the {@link + * AuthenticationProvider} contract. + */ @Override public void validate(ReadonlyConfig config) { - Optional username = config.getOptional(ElasticsearchBaseOptions.USERNAME); - Optional password = config.getOptional(ElasticsearchBaseOptions.PASSWORD); - - // For backward compatibility, we allow basic auth to be optional - // If username is provided, password must also be provided - if (username.isPresent() && !password.isPresent()) { - throw new IllegalArgumentException( - "Password is required when username is provided for basic authentication"); - } - - if (!username.isPresent() && password.isPresent()) { - throw new IllegalArgumentException( - "Username is required when password is provided for basic authentication"); - } - - if (username.isPresent()) { - String usernameValue = username.get(); - if (usernameValue == null || usernameValue.trim().isEmpty()) { - throw new IllegalArgumentException("Username cannot be null or empty"); - } - - String passwordValue = password.get(); - if (passwordValue == null || passwordValue.trim().isEmpty()) { - throw new IllegalArgumentException("Password cannot be null or empty"); - } - - log.debug("Basic authentication configuration validated for user: {}", usernameValue); - } else { - log.debug( - "No basic authentication credentials provided - authentication will be skipped"); - } + // intentionally empty - validation handled by OptionRule } } diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/config/ElasticsearchValidators.java b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/config/ElasticsearchValidators.java new file mode 100644 index 000000000000..6d429527bc87 --- /dev/null +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/config/ElasticsearchValidators.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.elasticsearch.config; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; + +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +/** + * Reusable {@link ConditionExtension} validators shared by Elasticsearch Source / Sink / Catalog + * factories. + * + *

All validators return {@code false} on failure (instead of throwing) so that {@code + * ConfigValidator} can aggregate errors across the entire rule. + */ +@UtilityClass +public final class ElasticsearchValidators { + + /** + * Validates that {@code auth.api_key_encoded} is a Base64-encoded {@code id:key} string. + * + *

Skips when the value is null/blank — presence and non-blankness are enforced separately by + * the corresponding conditional rules. + */ + @Slf4j + public static class ApiKeyEncodedFormatValidator implements ConditionExtension { + @Override + public String description() { + return "'auth.api_key_encoded' must be a Base64-encoded 'id:key' string"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String value) { + if (value == null || value.trim().isEmpty()) { + return true; + } + try { + byte[] decoded = Base64.getDecoder().decode(value); + return new String(decoded, StandardCharsets.UTF_8).contains(":"); + } catch (IllegalArgumentException e) { + log.warn("Failed to decode 'auth.api_key_encoded' as Base64", e); + return false; + } + } + } +} diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/sink/ElasticsearchSinkFactory.java b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/sink/ElasticsearchSinkFactory.java index 31b1438eec0b..0bbab15e35fc 100644 --- a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/sink/ElasticsearchSinkFactory.java +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/sink/ElasticsearchSinkFactory.java @@ -18,6 +18,7 @@ package org.apache.seatunnel.connectors.seatunnel.elasticsearch.sink; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.options.SinkConnectorCommonOptions; import org.apache.seatunnel.api.table.catalog.CatalogTable; @@ -28,6 +29,7 @@ import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext; import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.AuthTypeEnum; import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSinkOptions; +import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchValidators.ApiKeyEncodedFormatValidator; import com.google.auto.service.AutoService; @@ -63,8 +65,8 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() + .required(HOSTS) .required( - HOSTS, INDEX, ElasticsearchSinkOptions.SCHEMA_SAVE_MODE, ElasticsearchSinkOptions.DATA_SAVE_MODE) @@ -86,8 +88,24 @@ public OptionRule optionRule() { VECTORIZATION_FIELDS, VECTOR_DIMENSIONS) .optional(AUTH_TYPE) + .conditionalRule( + AUTH_TYPE, + AuthTypeEnum.BASIC, + OptionRule.builder().bundled(USERNAME, PASSWORD).build()) + .conditional(AUTH_TYPE, AuthTypeEnum.BASIC, Conditions.notBlank(USERNAME)) + .conditional(AUTH_TYPE, AuthTypeEnum.BASIC, Conditions.notBlank(PASSWORD)) .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, API_KEY_ID, API_KEY) + .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, Conditions.notBlank(API_KEY_ID)) + .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, Conditions.notBlank(API_KEY)) .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY_ENCODED, API_KEY_ENCODED) + .conditional( + AUTH_TYPE, + AuthTypeEnum.API_KEY_ENCODED, + Conditions.notBlank(API_KEY_ENCODED)) + .conditional( + AUTH_TYPE, + AuthTypeEnum.API_KEY_ENCODED, + Conditions.extension(API_KEY_ENCODED, new ApiKeyEncodedFormatValidator())) .build(); } diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSource.java b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSource.java index 46b01f2e89f2..6785b25cdbdd 100644 --- a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSource.java +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSource.java @@ -43,8 +43,6 @@ import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions; import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.SearchApiTypeEnum; import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.SearchTypeEnum; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.exception.ElasticsearchConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.exception.ElasticsearchConnectorException; import org.apache.commons.collections4.CollectionUtils; @@ -76,23 +74,10 @@ public ElasticsearchSource(ReadonlyConfig config) { boolean multiSource = config.getOptional(ElasticsearchSourceOptions.INDEX_LIST).isPresent(); boolean singleSource = config.getOptional(ElasticsearchSourceOptions.INDEX).isPresent(); - boolean sqlQuery = config.getOptional(SQL_QUERY).isPresent(); - - if (SearchTypeEnum.SQL.equals(config.get(SEARCH_TYPE)) && !sqlQuery) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.SOURCE_CONFIG_ERROR_02, - ElasticsearchConnectorErrorCode.SOURCE_CONFIG_ERROR_02.getDescription()); - } - if (multiSource && singleSource) { log.warn( "Elasticsearch Source config warn: when both 'index' and 'index_list' are present in the configuration, only the 'index_list' configuration will take effect"); } - if (!multiSource && !singleSource) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.SOURCE_CONFIG_ERROR_01, - ElasticsearchConnectorErrorCode.SOURCE_CONFIG_ERROR_01.getDescription()); - } if (multiSource) { this.elasticsearchConfigList = createMultiSource(config); } else { @@ -107,6 +92,11 @@ private List createMultiSource(ReadonlyConfig config) { configMaps.stream().map(ReadonlyConfig::fromMap).collect(Collectors.toList()); List elasticsearchConfigList = new ArrayList<>(configList.size()); for (ReadonlyConfig readonlyConfig : configList) { + // NOTE: per-entry configs inside `index_list` are NOT validated by the factory's + // OptionRule (which only validates the top-level config). If an entry is missing + // `index`, or sets `search_type=SQL` without `sql_query`, parseOneIndexQueryConfig + // below will fail at runtime. This is a pre-existing limitation; revisit if/when + // per-entry declarative validation is supported. ElasticsearchConfig elasticsearchConfig = parseOneIndexQueryConfig(readonlyConfig); elasticsearchConfigList.add(elasticsearchConfig); } diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSourceFactory.java b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSourceFactory.java index fc13b74d252f..cbf9c4757a87 100644 --- a/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSourceFactory.java +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSourceFactory.java @@ -17,6 +17,9 @@ package org.apache.seatunnel.connectors.seatunnel.elasticsearch.source; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.source.SourceSplit; @@ -25,10 +28,13 @@ import org.apache.seatunnel.api.table.factory.TableSourceFactory; import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.AuthTypeEnum; +import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchValidators.ApiKeyEncodedFormatValidator; +import org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.SearchTypeEnum; import com.google.auto.service.AutoService; import java.io.Serializable; +import java.util.List; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.API_KEY; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.API_KEY_ENCODED; @@ -44,6 +50,7 @@ import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.TLS_VERIFY_CERTIFICATE; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.TLS_VERIFY_HOSTNAME; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchBaseOptions.USERNAME; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.ARRAY_COLUMN; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.INDEX_LIST; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.PIT_BATCH_SIZE; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.PIT_KEEP_ALIVE; @@ -53,6 +60,9 @@ import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.SCROLL_TIME; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.SEARCH_API_TYPE; import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.SEARCH_TYPE; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.SLICE_MAX; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.SOURCE; +import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.config.ElasticsearchSourceOptions.SQL_QUERY; @AutoService(Factory.class) public class ElasticsearchSourceFactory implements TableSourceFactory { @@ -64,7 +74,7 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .required(HOSTS) + .required(HOSTS, Conditions.extension(HOSTS, new RequireIndexValidator())) .optional( INDEX, INDEX_LIST, @@ -78,6 +88,9 @@ public OptionRule optionRule() { PIT_BATCH_SIZE, SEARCH_API_TYPE, SEARCH_TYPE, + SOURCE, + ARRAY_COLUMN, + SLICE_MAX, TLS_VERIFY_CERTIFICATE, TLS_VERIFY_HOSTNAME, TLS_KEY_STORE_PATH, @@ -85,8 +98,26 @@ public OptionRule optionRule() { TLS_TRUST_STORE_PATH, TLS_TRUST_STORE_PASSWORD) .optional(AUTH_TYPE) + .conditionalRule( + AUTH_TYPE, + AuthTypeEnum.BASIC, + OptionRule.builder().bundled(USERNAME, PASSWORD).build()) + .conditional(AUTH_TYPE, AuthTypeEnum.BASIC, Conditions.notBlank(USERNAME)) + .conditional(AUTH_TYPE, AuthTypeEnum.BASIC, Conditions.notBlank(PASSWORD)) .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, API_KEY_ID, API_KEY) + .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, Conditions.notBlank(API_KEY_ID)) + .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY, Conditions.notBlank(API_KEY)) .conditional(AUTH_TYPE, AuthTypeEnum.API_KEY_ENCODED, API_KEY_ENCODED) + .conditional( + AUTH_TYPE, + AuthTypeEnum.API_KEY_ENCODED, + Conditions.notBlank(API_KEY_ENCODED)) + .conditional( + AUTH_TYPE, + AuthTypeEnum.API_KEY_ENCODED, + Conditions.extension(API_KEY_ENCODED, new ApiKeyEncodedFormatValidator())) + .conditional(SEARCH_TYPE, SearchTypeEnum.SQL, SQL_QUERY) + .conditional(SEARCH_TYPE, SearchTypeEnum.SQL, Conditions.notBlank(SQL_QUERY)) .build(); } @@ -101,4 +132,21 @@ TableSource createSource(TableSourceFactoryContext context) { public Class getSourceClass() { return ElasticsearchSource.class; } + + /** + * Validates that at least one of {@code index} or {@code index_list} is provided. Attached to + * the always-present {@code HOSTS} so the rule runs even when both index options are missing. + */ + static class RequireIndexValidator implements ConditionExtension> { + @Override + public String description() { + return "at least one of 'index' or 'index_list' must be provided"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List value) { + return config.getOptional(INDEX).isPresent() + || config.getOptional(INDEX_LIST).isPresent(); + } + } } diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchCatalogFactoryTest.java b/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchCatalogFactoryTest.java new file mode 100644 index 000000000000..10b044492830 --- /dev/null +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchCatalogFactoryTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.elasticsearch.catalog; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +class ElasticSearchCatalogFactoryTest { + + private OptionRule rule; + + @BeforeEach + void setUp() { + rule = new ElasticSearchCatalogFactory().optionRule(); + } + + @Test + void testValidConfig() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testMissingHostsFails() { + Map config = new HashMap<>(); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("hosts")); + } + + @Test + void testUsernameWithoutPasswordFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("username", "admin"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("username") || ex.getMessage().contains("password")); + } + + @Test + void testValidBasicAuth() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("username", "admin"); + config.put("password", "secret"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testBasicAuthBlankUsernameFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("username", " "); + config.put("password", "secret"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("username") || ex.getMessage().contains("blank")); + } + + @Test + void testBasicAuthBlankPasswordFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("username", "admin"); + config.put("password", " "); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("password") || ex.getMessage().contains("blank")); + } + + @Test + void testApiKeyAuthMissingKeysFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("auth_type", "API_KEY"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("api_key_id") || ex.getMessage().contains("api_key")); + } + + @Test + void testApiKeyEncodedInvalidFormatFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("auth_type", "API_KEY_ENCODED"); + config.put("auth.api_key_encoded", "not-base64-!@#"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("Base64")); + } +} diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/sink/ElasticsearchSinkFactoryTest.java b/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/sink/ElasticsearchSinkFactoryTest.java new file mode 100644 index 000000000000..38fd8e8d8019 --- /dev/null +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/sink/ElasticsearchSinkFactoryTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.elasticsearch.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +class ElasticsearchSinkFactoryTest { + + private OptionRule rule; + + @BeforeEach + void setUp() { + rule = new ElasticsearchSinkFactory().optionRule(); + } + + private Map validSinkConfig() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("schema_save_mode", "CREATE_SCHEMA_WHEN_NOT_EXIST"); + config.put("data_save_mode", "APPEND_DATA"); + return config; + } + + @Test + void testValidConfig() { + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(validSinkConfig())).validate(rule)); + } + + @Test + void testMissingHostsFails() { + Map config = validSinkConfig(); + config.remove("hosts"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("hosts")); + } + + @Test + void testMissingIndexFails() { + Map config = validSinkConfig(); + config.remove("index"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("index")); + } + + @Test + void testUsernameWithoutPasswordFails() { + Map config = validSinkConfig(); + config.put("username", "admin"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("username") || ex.getMessage().contains("password")); + } + + @Test + void testPasswordWithoutUsernameFails() { + Map config = validSinkConfig(); + config.put("password", "secret"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("username") || ex.getMessage().contains("password")); + } + + @Test + void testValidBasicAuth() { + Map config = validSinkConfig(); + config.put("username", "admin"); + config.put("password", "secret"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testBasicAuthBlankUsernameFails() { + Map config = validSinkConfig(); + config.put("username", " "); + config.put("password", "secret"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("username") || ex.getMessage().contains("blank")); + } + + @Test + void testBasicAuthBlankPasswordFails() { + Map config = validSinkConfig(); + config.put("username", "admin"); + config.put("password", " "); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("password") || ex.getMessage().contains("blank")); + } + + @Test + void testApiKeyAuthMissingKeysFails() { + Map config = validSinkConfig(); + config.put("auth_type", "API_KEY"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("api_key_id") || ex.getMessage().contains("api_key")); + } + + @Test + void testApiKeyAuthBlankValueFails() { + Map config = validSinkConfig(); + config.put("auth_type", "API_KEY"); + config.put("auth.api_key_id", "valid_id"); + config.put("auth.api_key", " "); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("api_key")); + } + + @Test + void testApiKeyEncodedAuthBlankFails() { + Map config = validSinkConfig(); + config.put("auth_type", "API_KEY_ENCODED"); + config.put("auth.api_key_encoded", " "); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("api_key_encoded")); + } + + @Test + void testApiKeyEncodedInvalidFormatFails() { + Map config = validSinkConfig(); + config.put("auth_type", "API_KEY_ENCODED"); + // valid Base64 but decoded value lacks the required ':' separator + config.put("auth.api_key_encoded", "bm9Db2xvbkhlcmU="); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("id:key")); + } + + @Test + void testApiKeyEncodedValidFormatPasses() { + Map config = validSinkConfig(); + config.put("auth_type", "API_KEY_ENCODED"); + config.put("auth.api_key_encoded", "aWQ6a2V5"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testValidApiKeyAuth() { + Map config = validSinkConfig(); + config.put("auth_type", "API_KEY"); + config.put("auth.api_key_id", "my_key_id"); + config.put("auth.api_key", "my_secret"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testApiKeyAuthWithResidualUsernameDoesNotFail() { + Map config = validSinkConfig(); + config.put("auth_type", "API_KEY"); + config.put("auth.api_key_id", "my_key_id"); + config.put("auth.api_key", "my_secret"); + config.put("username", "residual_user"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } +} diff --git a/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSourceFactoryTest.java b/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSourceFactoryTest.java new file mode 100644 index 000000000000..d1b6d2f4db41 --- /dev/null +++ b/seatunnel-connectors-v2/connector-elasticsearch/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/source/ElasticsearchSourceFactoryTest.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.elasticsearch.source; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +class ElasticsearchSourceFactoryTest { + + private OptionRule rule; + + @BeforeEach + void setUp() { + rule = new ElasticsearchSourceFactory().optionRule(); + } + + @Test + void testValidConfigWithIndex() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testValidConfigWithIndexList() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put( + "index_list", + Arrays.asList( + new HashMap() { + { + put("index", "idx1"); + } + })); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testMissingHostsFails() { + Map config = new HashMap<>(); + config.put("index", "test_index"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("hosts")); + } + + @Test + void testMissingIndexAndIndexListFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("index") || ex.getMessage().contains("index_list")); + } + + @Test + void testSqlSearchTypeMissingSqlQueryFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("search_type", "SQL"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("sql_query")); + } + + @Test + void testSqlSearchTypeBlankSqlQueryFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("search_type", "SQL"); + config.put("sql_query", " "); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("sql_query")); + } + + @Test + void testSqlSearchTypeWithValidSqlQuery() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("search_type", "SQL"); + config.put("sql_query", "SELECT * FROM test_index"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testUsernameWithoutPasswordFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("username", "admin"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("username") || ex.getMessage().contains("password")); + } + + @Test + void testPasswordWithoutUsernameFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("password", "secret"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("username") || ex.getMessage().contains("password")); + } + + @Test + void testValidBasicAuth() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("username", "admin"); + config.put("password", "secret"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testBasicAuthBlankUsernameFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("username", " "); + config.put("password", "secret"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("username") || ex.getMessage().contains("blank")); + } + + @Test + void testBasicAuthBlankPasswordFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("username", "admin"); + config.put("password", " "); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("password") || ex.getMessage().contains("blank")); + } + + @Test + void testApiKeyAuthMissingKeysFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue( + ex.getMessage().contains("api_key_id") || ex.getMessage().contains("api_key")); + } + + @Test + void testApiKeyAuthBlankKeyIdFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY"); + config.put("auth.api_key_id", " "); + config.put("auth.api_key", "my_secret"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("api_key_id")); + } + + @Test + void testApiKeyEncodedAuthMissingFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY_ENCODED"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("api_key_encoded")); + } + + @Test + void testApiKeyEncodedAuthBlankFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY_ENCODED"); + config.put("auth.api_key_encoded", " "); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("api_key_encoded")); + } + + @Test + void testApiKeyEncodedInvalidBase64Fails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY_ENCODED"); + config.put("auth.api_key_encoded", "not-valid-base64!@#"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("Base64")); + } + + @Test + void testApiKeyEncodedBase64WithoutColonFails() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY_ENCODED"); + // "noColonHere" Base64-encoded + config.put("auth.api_key_encoded", "bm9Db2xvbkhlcmU="); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + Assertions.assertTrue(ex.getMessage().contains("id:key")); + } + + @Test + void testApiKeyEncodedValidFormatPasses() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY_ENCODED"); + // "id:key" Base64-encoded + config.put("auth.api_key_encoded", "aWQ6a2V5"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + @Test + void testValidApiKeyAuth() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY"); + config.put("auth.api_key_id", "my_key_id"); + config.put("auth.api_key", "my_secret"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } + + /** + * Verify the error-aggregation contract: when multiple independent rules fail in one config, + * all errors are reported together in a single exception message instead of fail-fast on the + * first one. Triggers (1) username-without-password (auth_type=BASIC), (2) search_type=SQL + * without sql_query. + */ + @Test + void testMultipleValidationErrorsAreAggregated() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("username", "admin"); + config.put("search_type", "SQL"); + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + String msg = ex.getMessage(); + Assertions.assertTrue( + msg.contains("password") || msg.contains("username"), + "expected basic-auth pair error in: " + msg); + Assertions.assertTrue(msg.contains("sql_query"), "expected sql_query error in: " + msg); + } + + @Test + void testApiKeyAuthWithResidualUsernameDoesNotFail() { + Map config = new HashMap<>(); + config.put("hosts", Arrays.asList("localhost:9200")); + config.put("index", "test_index"); + config.put("auth_type", "API_KEY"); + config.put("auth.api_key_id", "my_key_id"); + config.put("auth.api_key", "my_secret"); + config.put("username", "residual_user"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + } +} From c8fb49358a46cf295ef308438756c2e946f99464 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Sat, 27 Jun 2026 21:10:56 +0800 Subject: [PATCH 070/375] [Improve][Connector-V2] Migrate RocketMQ Source validation to declarative OptionRule (#11158) --- .../rocketmq/source/RocketMqSourceConfig.java | 31 +--- .../source/RocketMqSourceFactory.java | 79 +++++++++ .../rocketmq/source/RocketMqFactoryTest.java | 157 ++++++++++++++++++ 3 files changed, 242 insertions(+), 25 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-rocketmq/src/test/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqFactoryTest.java diff --git a/seatunnel-connectors-v2/connector-rocketmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqSourceConfig.java b/seatunnel-connectors-v2/connector-rocketmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqSourceConfig.java index e94d5d13da1a..633b44461371 100644 --- a/seatunnel-connectors-v2/connector-rocketmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqSourceConfig.java +++ b/seatunnel-connectors-v2/connector-rocketmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqSourceConfig.java @@ -103,11 +103,6 @@ public RocketMqSourceConfig(ReadonlyConfig readonlyConfig) { private void parseTableConfig(ReadonlyConfig tableConfig, List allTopics) { String topicsStr = tableConfig.get(RocketMqSourceOptions.TOPICS); - if (topicsStr == null || topicsStr.trim().isEmpty()) { - throw new IllegalArgumentException( - "'topics' must be configured in each tables_configs entry, but got: " - + tableConfig); - } List topics = Arrays.stream(topicsStr.split(RocketMqSourceOptions.DEFAULT_FIELD_DELIMITER)) .map(String::trim) @@ -135,30 +130,16 @@ private void parseTableConfig(ReadonlyConfig tableConfig, List allTopics switch (startMode) { case CONSUME_FROM_TIMESTAMP: startTimestamp = tableConfig.get(RocketMqSourceOptions.START_MODE_TIMESTAMP); - if (startTimestamp == null) { - throw new IllegalArgumentException( - "When 'start.mode' is set to 'CONSUME_FROM_TIMESTAMP' in tables_configs, " - + "'start.mode.timestamp' must also be specified in the same table config entry. " - + "Topics: " - + topicsStr); - } + // Runtime check: cannot be declarative (depends on current time) long currentTimestamp = System.currentTimeMillis(); - if (startTimestamp < 0 || startTimestamp > currentTimestamp) { + if (startTimestamp > currentTimestamp) { throw new IllegalArgumentException( - "The offsets timestamp value is smaller than 0 or larger" - + " than the current time"); + "start.mode.timestamp must not be greater than the current time"); } break; case CONSUME_FROM_SPECIFIC_OFFSETS: Map offsetConfigMap = tableConfig.get(RocketMqSourceOptions.START_MODE_OFFSETS); - if (offsetConfigMap == null || offsetConfigMap.isEmpty()) { - throw new IllegalArgumentException( - "When 'start.mode' is set to 'CONSUME_FROM_SPECIFIC_OFFSETS' in tables_configs, " - + "'start.mode.offsets' must also be specified in the same table config entry. " - + "Topics: " - + topicsStr); - } Map specificOffsets = metadata.getSpecificStartOffsets(); if (specificOffsets == null) { specificOffsets = new HashMap<>(); @@ -233,10 +214,10 @@ private ConsumerMetadata buildConsumerMetadata( long startOffsetsTimestamp = readonlyConfig.get(RocketMqSourceOptions.START_MODE_TIMESTAMP); long currentTimestamp = System.currentTimeMillis(); - if (startOffsetsTimestamp < 0 || startOffsetsTimestamp > currentTimestamp) { + // Runtime check: cannot be declarative (depends on current time) + if (startOffsetsTimestamp > currentTimestamp) { throw new IllegalArgumentException( - "The offsets timestamp value is smaller than 0 or larger" - + " than the current time"); + "start.mode.timestamp must not be greater than the current time"); } consumerMetadata.setStartOffsetsTimestamp(startOffsetsTimestamp); break; diff --git a/seatunnel-connectors-v2/connector-rocketmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqSourceFactory.java b/seatunnel-connectors-v2/connector-rocketmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqSourceFactory.java index 47cd1e19c19c..1d6a4b833158 100644 --- a/seatunnel-connectors-v2/connector-rocketmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqSourceFactory.java +++ b/seatunnel-connectors-v2/connector-rocketmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqSourceFactory.java @@ -17,7 +17,11 @@ package org.apache.seatunnel.connectors.seatunnel.rocketmq.source; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.source.SourceSplit; import org.apache.seatunnel.api.table.connector.TableSource; @@ -31,6 +35,8 @@ import com.google.auto.service.AutoService; import java.io.Serializable; +import java.util.List; +import java.util.Map; @AutoService(Factory.class) public class RocketMqSourceFactory implements TableSourceFactory { @@ -65,10 +71,26 @@ public OptionRule optionRule() { RocketMqSourceOptions.START_MODE, StartMode.CONSUME_FROM_TIMESTAMP, RocketMqSourceOptions.START_MODE_TIMESTAMP) + .conditional( + RocketMqSourceOptions.START_MODE, + StartMode.CONSUME_FROM_TIMESTAMP, + Conditions.greaterOrEqual(RocketMqSourceOptions.START_MODE_TIMESTAMP, 0L)) .conditional( RocketMqSourceOptions.START_MODE, StartMode.CONSUME_FROM_SPECIFIC_OFFSETS, RocketMqSourceOptions.START_MODE_OFFSETS) + .conditional( + RocketMqSourceOptions.START_MODE, + StartMode.CONSUME_FROM_SPECIFIC_OFFSETS, + Conditions.mapNotEmpty(RocketMqSourceOptions.START_MODE_OFFSETS)) + .optional( + RocketMqSourceOptions.TABLE_CONFIGS, + Conditions.extension( + RocketMqSourceOptions.TABLE_CONFIGS, new TableConfigsValidator())) + .optional( + RocketMqSourceOptions.TABLE_LIST, + Conditions.extension( + RocketMqSourceOptions.TABLE_LIST, new TableConfigsValidator())) .conditional( RocketMqBaseOptions.ACL_ENABLED, true, @@ -87,4 +109,61 @@ public Class getSourceClass() { TableSource createSource(TableSourceFactoryContext context) { return () -> (SeaTunnelSource) new RocketMqSource(context.getOptions()); } + + static class TableConfigsValidator implements ConditionExtension>> { + + @Override + public String description() { + return "each tables_configs entry must have valid topics, " + + "start.mode.timestamp (>= 0) when CONSUME_FROM_TIMESTAMP, " + + "and non-empty start.mode.offsets when CONSUME_FROM_SPECIFIC_OFFSETS"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List> entries) + throws OptionValidationException { + if (entries == null || entries.isEmpty()) { + return true; + } + for (int i = 0; i < entries.size(); i++) { + ReadonlyConfig tableConfig = ReadonlyConfig.fromMap(entries.get(i)); + String topics = tableConfig.get(RocketMqSourceOptions.TOPICS); + if (topics == null || topics.trim().isEmpty()) { + throw new OptionValidationException( + "tables_configs[%d]: 'topics' must not be empty", i); + } + StartMode startMode = + tableConfig.getOptional(RocketMqSourceOptions.START_MODE).orElse(null); + if (startMode == StartMode.CONSUME_FROM_TIMESTAMP) { + Long ts = + tableConfig + .getOptional(RocketMqSourceOptions.START_MODE_TIMESTAMP) + .orElse(null); + if (ts == null) { + throw new OptionValidationException( + "tables_configs[%d]: 'start.mode.timestamp' required " + + "when start.mode=CONSUME_FROM_TIMESTAMP", + i); + } + if (ts < 0) { + throw new OptionValidationException( + "tables_configs[%d]: 'start.mode.timestamp' must be >= 0, got: %d", + i, ts); + } + } else if (startMode == StartMode.CONSUME_FROM_SPECIFIC_OFFSETS) { + Map offsets = + tableConfig + .getOptional(RocketMqSourceOptions.START_MODE_OFFSETS) + .orElse(null); + if (offsets == null || offsets.isEmpty()) { + throw new OptionValidationException( + "tables_configs[%d]: 'start.mode.offsets' must not be empty " + + "when start.mode=CONSUME_FROM_SPECIFIC_OFFSETS", + i); + } + } + } + return true; + } + } } diff --git a/seatunnel-connectors-v2/connector-rocketmq/src/test/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqFactoryTest.java b/seatunnel-connectors-v2/connector-rocketmq/src/test/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqFactoryTest.java new file mode 100644 index 000000000000..95c8f5a175c3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rocketmq/src/test/java/org/apache/seatunnel/connectors/seatunnel/rocketmq/source/RocketMqFactoryTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rocketmq.source; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class RocketMqFactoryTest { + + private final OptionRule sourceRule = new RocketMqSourceFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(sourceRule); + } + + private Map validTimestampConfig() { + Map cfg = new HashMap<>(); + cfg.put("name.srv.addr", "localhost:9876"); + cfg.put("topics", "test-topic"); + cfg.put("start.mode", "CONSUME_FROM_TIMESTAMP"); + cfg.put("start.mode.timestamp", 1000L); + return cfg; + } + + private Map validSpecificOffsetsConfig() { + Map cfg = new HashMap<>(); + cfg.put("name.srv.addr", "localhost:9876"); + cfg.put("topics", "test-topic"); + cfg.put("start.mode", "CONSUME_FROM_SPECIFIC_OFFSETS"); + Map offsets = new HashMap<>(); + offsets.put("test-topic-0", 100L); + cfg.put("start.mode.offsets", offsets); + return cfg; + } + + private Map validMultiTableConfig() { + Map cfg = new HashMap<>(); + cfg.put("name.srv.addr", "localhost:9876"); + List> tableConfigs = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("topics", "topic-a,topic-b"); + tableConfigs.add(entry); + cfg.put("tables_configs", tableConfigs); + return cfg; + } + + @Test + void testValidTimestampConfig() { + Assertions.assertDoesNotThrow(() -> validate(validTimestampConfig())); + } + + @Test + void testNegativeTimestampRejected() { + Map cfg = validTimestampConfig(); + cfg.put("start.mode.timestamp", -1L); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testValidSpecificOffsetsConfig() { + Assertions.assertDoesNotThrow(() -> validate(validSpecificOffsetsConfig())); + } + + @Test + void testEmptyOffsetsMapRejected() { + Map cfg = new HashMap<>(); + cfg.put("name.srv.addr", "localhost:9876"); + cfg.put("topics", "test-topic"); + cfg.put("start.mode", "CONSUME_FROM_SPECIFIC_OFFSETS"); + cfg.put("start.mode.offsets", Collections.emptyMap()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMultiTableValidConfig() { + Assertions.assertDoesNotThrow(() -> validate(validMultiTableConfig())); + } + + @Test + void testMultiTableMissingTopicsRejected() { + Map cfg = new HashMap<>(); + cfg.put("name.srv.addr", "localhost:9876"); + List> tableConfigs = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("topics", ""); + tableConfigs.add(entry); + cfg.put("tables_configs", tableConfigs); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMultiTableTimestampModeWithoutTimestampRejected() { + Map cfg = new HashMap<>(); + cfg.put("name.srv.addr", "localhost:9876"); + List> tableConfigs = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("topics", "topic-a"); + entry.put("start.mode", "CONSUME_FROM_TIMESTAMP"); + tableConfigs.add(entry); + cfg.put("tables_configs", tableConfigs); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMultiTableNegativeTimestampRejected() { + Map cfg = new HashMap<>(); + cfg.put("name.srv.addr", "localhost:9876"); + List> tableConfigs = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("topics", "topic-a"); + entry.put("start.mode", "CONSUME_FROM_TIMESTAMP"); + entry.put("start.mode.timestamp", -5L); + tableConfigs.add(entry); + cfg.put("tables_configs", tableConfigs); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMultiTableEmptyOffsetsRejected() { + Map cfg = new HashMap<>(); + cfg.put("name.srv.addr", "localhost:9876"); + List> tableConfigs = new ArrayList<>(); + Map entry = new HashMap<>(); + entry.put("topics", "topic-a"); + entry.put("start.mode", "CONSUME_FROM_SPECIFIC_OFFSETS"); + entry.put("start.mode.offsets", Collections.emptyMap()); + tableConfigs.add(entry); + cfg.put("tables_configs", tableConfigs); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} From dc64df48e6b529d8e2bb9318116e0d52d0890cdd Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Sat, 27 Jun 2026 21:11:01 +0800 Subject: [PATCH 071/375] [Improve][Transform-V2] Migrate transforms validation to declarative OptionRule (#11095) --- .../concepts/incompatible-changes.md | 11 ++ .../concepts/incompatible-changes.md | 12 ++ .../DefineSinkTypeTransformConfig.java | 12 -- .../DefineSinkTypeTransformFactory.java | 52 ++++++++- .../copy/CopyFieldTransformFactory.java | 27 ++++- .../DynamicCompileTransformFactory.java | 14 ++- .../encrypt/FieldEncryptTransformFactory.java | 12 +- .../FieldMapperTransformFactory.java | 5 +- .../filter/FilterFieldTransform.java | 10 -- .../filter/FilterFieldTransformFactory.java | 9 +- .../jsonpath/JsonPathTransformFactory.java | 55 ++++++++- .../metadata/MetadataTransformFactory.java | 5 +- .../RegexExtractTransformFactory.java | 33 +++++- .../transform/replace/ReplaceTransform.java | 34 +----- .../replace/ReplaceTransformFactory.java | 8 +- .../RowKindExtractorTransformFactory.java | 1 + .../split/SplitTransformFactory.java | 10 +- .../transform/sql/SQLTransformFactory.java | 2 +- .../transform/table/TableFilterConfig.java | 8 -- .../table/TableFilterTransformFactory.java | 35 +++++- .../table/TableMergeTransformFactory.java | 3 +- .../DataValidatorTransformFactory.java | 47 +++++++- .../DefineSinkTypeTransformFactoryTest.java | 97 +++++++++++++++ .../copy/CopyFieldTransformFactoryTest.java | 83 +++++++++++++ .../DynamicCompileTransformFactoryTest.java | 105 +++++++++++++++++ .../FieldEncryptTransformFactoryTest.java | 101 ++++++++++++++++ .../FieldMapperTransformFactoryTest.java | 61 ++++++++++ .../FilterFieldTransformFactoryTest.java | 82 +++++++++++++ .../filter/FilterFieldTransformTest.java | 44 +++---- .../JsonPathTransformFactoryTest.java | 110 ++++++++++++++++++ .../MetadataTransformFactoryTest.java | 61 ++++++++++ .../RegexExtractTransformFactoryTest.java | 97 +++++++++++++++ .../replace/ReplaceTransformFactoryTest.java | 90 ++++++++++++++ .../replace/ReplaceTransformTest.java | 33 +++--- .../split/SplitTransformFactoryTest.java | 81 +++++++++++++ .../sql/SQLTransformFactoryTest.java | 22 ++++ .../TableFilterTransformFactoryTest.java | 88 ++++++++++++++ .../DataValidatorTransformFactoryTest.java | 103 ++++++++++++++++ 38 files changed, 1538 insertions(+), 125 deletions(-) create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/copy/CopyFieldTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/dynamiccompile/DynamicCompileTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/encrypt/FieldEncryptTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/fieldmapper/FieldMapperTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/filter/FilterFieldTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/jsonpath/JsonPathTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/MetadataTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/regexextract/RegexExtractTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/replace/ReplaceTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/split/SplitTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/table/TableFilterTransformFactoryTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/validator/DataValidatorTransformFactoryTest.java diff --git a/docs/en/introduction/concepts/incompatible-changes.md b/docs/en/introduction/concepts/incompatible-changes.md index 6ea2dc2ac1d3..c0fe5c18240f 100644 --- a/docs/en/introduction/concepts/incompatible-changes.md +++ b/docs/en/introduction/concepts/incompatible-changes.md @@ -102,6 +102,17 @@ You need to check this document before you upgrade to related version. **Migration Guide**: If you are using custom datetime format patterns in `PARSEDATETIME`, `TO_DATE`, or `IS_DATE` functions, you must update your queries to use one of the supported patterns above. If your data uses a different format, you may need to preprocess the input data to match a supported format, or use string manipulation functions to transform the format before parsing. - DataValidator transform: In `row_error_handle_way = ROUTE_TO_TABLE` mode, the routed error row `table_id` now includes the upstream database/schema prefix (for example, `db1.ffp` / `db1.schema1.ffp` instead of `ffp`). +- **[BREAKING]** Several transform plugins now perform stricter submission-time config validation via declarative `OptionRule`. Configs that previously passed submission but failed at runtime will now be rejected at submission time with a descriptive `OptionValidationException`: + + | Transform | Newly Rejected Config | Previous Behavior | Migration | + |-----------|----------------------|-------------------|-----------| + | `DefineSinkType` | `columns` entries with null/empty `column` or `type` | Runtime NPE or undefined behavior | Ensure every entry has non-empty `column` and `type` fields | + | `DefineSinkType` | `columns` with duplicate column names | Silent override or runtime conflict | Remove duplicate column entries | + | `FieldEncrypt` | `max_field_length` set to ≤ 0 | Ignored or unexpected truncation | Set `max_field_length` to a positive integer, or remove the option to use the default | + | `DynamicCompile` | `compile_pattern = SOURCE_CODE` without a non-blank `source_code` | Runtime compilation failure | Provide `source_code` when using `SOURCE_CODE` pattern | + | `DynamicCompile` | `compile_pattern = ABSOLUTE_PATH` without a non-blank `absolute_path` | Runtime file-read failure | Provide `absolute_path` when using `ABSOLUTE_PATH` pattern | + + **Migration Guide**: Review your transform configs against the table above. If any of your existing configs match a "Newly Rejected" pattern, update them before upgrading. The error messages at submission time now clearly identify which option is invalid and why. - Adjusted SQL Transform date & time functions: - `DATEDIFF(, , 'MONTH')` now returns the total number of months between the two dates across years (for example, from `2023-01-01` to `2024-03-01` returns `14` instead of `15`). - `WEEK()` now returns the ISO week number directly (previous behavior added an extra `+1` to the ISO week value). diff --git a/docs/zh/introduction/concepts/incompatible-changes.md b/docs/zh/introduction/concepts/incompatible-changes.md index 8a6d7b0a5198..8c046178f267 100644 --- a/docs/zh/introduction/concepts/incompatible-changes.md +++ b/docs/zh/introduction/concepts/incompatible-changes.md @@ -98,6 +98,18 @@ **迁移指南**: 如果您在 `PARSEDATETIME`、`TO_DATE` 或 `IS_DATE` 函数中使用自定义日期时间格式模式，您必须更新查询以使用上述支持的模式之一。如果您的数据使用不同的格式，您可能需要预处理输入数据以匹配支持的格式，或使用字符串操作函数在解析之前转换格式。 - DataValidator 转换：当 `row_error_handle_way = ROUTE_TO_TABLE` 时，路由到错误表的行 `table_id` 现在会携带上游的 database/schema 前缀（例如从 `ffp` 变为 `db1.ffp` / `db1.schema1.ffp`）。 +- **[BREAKING]** 多个转换插件现在通过声明式 `OptionRule` 在提交时执行更严格的配置校验。以前在提交时能通过但运行时失败的配置，现在会在提交时被拒绝，并抛出描述清晰的 `OptionValidationException`： + + | 转换插件 | 新增拒绝的配置 | 以前的行为 | 迁移方式 | + |---------|--------------|-----------|---------| + | `DefineSinkType` | `columns` 条目中 `column` 或 `type` 为空 | 运行时 NPE 或未定义行为 | 确保每个条目都有非空的 `column` 和 `type` 字段 | + | `DefineSinkType` | `columns` 中存在重复列名 | 静默覆盖或运行时冲突 | 移除重复的列条目 | + | `FieldEncrypt` | `max_field_length` 设置为 ≤ 0 | 被忽略或产生意外截断 | 设置为正整数，或移除该选项以使用默认值 | + | `DynamicCompile` | `compile_pattern = SOURCE_CODE` 但 `source_code` 为空 | 运行时编译失败 | 使用 `SOURCE_CODE` 模式时提供 `source_code` | + | `DynamicCompile` | `compile_pattern = ABSOLUTE_PATH` 但 `absolute_path` 为空 | 运行时文件读取失败 | 使用 `ABSOLUTE_PATH` 模式时提供 `absolute_path` | + + **迁移指南**：升级前请对照上表检查您的转换配置。如果现有配置匹配了"新增拒绝的配置"中的情况，请在升级前修改。提交时的错误消息会清楚标明哪个选项无效及原因。 + ### 引擎行为变更 ### 依赖升级 diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformConfig.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformConfig.java index 06eda0e78372..fa37191e9656 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformConfig.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformConfig.java @@ -33,8 +33,6 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.apache.seatunnel.shade.com.google.common.base.Preconditions.checkArgument; - @Data @AllArgsConstructor public class DefineSinkTypeTransformConfig implements Serializable { @@ -82,16 +80,6 @@ public static class TableTransforms implements Serializable { public static DefineSinkTypeTransformConfig of(ReadonlyConfig config) { List columns = config.get(COLUMNS); - - checkArgument(columns != null && !columns.isEmpty(), "The columns must be set"); - columns.forEach( - defineColumnType -> { - checkArgument( - defineColumnType.getColumn() != null, "The column name must be set"); - checkArgument( - defineColumnType.getType() != null, "The column type must be set"); - }); - return new DefineSinkTypeTransformConfig(columns); } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformFactory.java index e583ca2b4432..2f0d986b2216 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformFactory.java @@ -17,15 +17,24 @@ package org.apache.seatunnel.transform.adaptsink; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; import org.apache.seatunnel.api.table.factory.TableTransformFactory; import org.apache.seatunnel.api.table.factory.TableTransformFactoryContext; +import org.apache.seatunnel.transform.adaptsink.DefineSinkTypeTransformConfig.DefineColumnType; import org.apache.seatunnel.transform.common.TransformCommonOptions; import com.google.auto.service.AutoService; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + @AutoService(Factory.class) public class DefineSinkTypeTransformFactory implements TableTransformFactory { @Override @@ -36,7 +45,13 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .required(DefineSinkTypeTransformConfig.COLUMNS) + .required( + DefineSinkTypeTransformConfig.COLUMNS, + Conditions.notEmpty(DefineSinkTypeTransformConfig.COLUMNS) + .and( + Conditions.extension( + DefineSinkTypeTransformConfig.COLUMNS, + new ColumnsStructureValidator()))) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); @@ -48,4 +63,39 @@ public TableTransform createTransform(TableTransformFactoryContext context) { new DefineSinkTypeMultiCatalogTransform( context.getCatalogTables(), context.getOptions()); } + + static class ColumnsStructureValidator implements ConditionExtension> { + @Override + public String description() { + return "each column entry must contain non-null 'column' and 'type'"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List value) + throws OptionValidationException { + if (value == null) { + return false; + } + Set seen = new HashSet<>(); + for (int i = 0; i < value.size(); i++) { + DefineColumnType entry = value.get(i); + if (entry.getColumn() == null || entry.getColumn().trim().isEmpty()) { + throw new OptionValidationException( + String.format( + "columns[%d]: 'column' name must not be null or empty", i)); + } + if (entry.getType() == null || entry.getType().trim().isEmpty()) { + throw new OptionValidationException( + String.format("columns[%d]: 'type' must not be null or empty", i)); + } + if (!seen.add(entry.getColumn())) { + throw new OptionValidationException( + String.format( + "columns[%d]: duplicate column name '%s'", + i, entry.getColumn())); + } + } + return true; + } + } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/copy/CopyFieldTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/copy/CopyFieldTransformFactory.java index 428623ff2e17..5de39766ffb1 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/copy/CopyFieldTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/copy/CopyFieldTransformFactory.java @@ -17,6 +17,9 @@ package org.apache.seatunnel.transform.copy; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -36,8 +39,15 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .bundled(CopyTransformConfig.SRC_FIELD, CopyTransformConfig.DEST_FIELD) - .bundled(CopyTransformConfig.FIELDS) + .exclusive(CopyTransformConfig.FIELDS, CopyTransformConfig.SRC_FIELD) + .optional( + CopyTransformConfig.FIELDS, + Conditions.mapNotEmpty(CopyTransformConfig.FIELDS)) + .optional( + CopyTransformConfig.SRC_FIELD, + Conditions.extension( + CopyTransformConfig.SRC_FIELD, new RequireDestFieldValidator())) + .optional(CopyTransformConfig.DEST_FIELD) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); @@ -49,4 +59,17 @@ public TableTransform createTransform(TableTransformFactoryContext context) { new CopyFieldMultiCatalogTransform( context.getCatalogTables(), context.getOptions()); } + + static class RequireDestFieldValidator implements ConditionExtension { + @Override + public String description() { + return "'dest_field' is required when 'src_field' is provided"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String value) { + String destField = config.get(CopyTransformConfig.DEST_FIELD); + return destField != null && !destField.trim().isEmpty(); + } + } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/dynamiccompile/DynamicCompileTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/dynamiccompile/DynamicCompileTransformFactory.java index 593e940cb621..02141fd0a985 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/dynamiccompile/DynamicCompileTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/dynamiccompile/DynamicCompileTransformFactory.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.transform.dynamiccompile; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -36,17 +37,24 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .optional( - DynamicCompileTransformConfig.COMPILE_LANGUAGE, - DynamicCompileTransformConfig.COMPILE_PATTERN) + .required(DynamicCompileTransformConfig.COMPILE_LANGUAGE) + .optional(DynamicCompileTransformConfig.COMPILE_PATTERN) .conditional( DynamicCompileTransformConfig.COMPILE_PATTERN, CompilePattern.SOURCE_CODE, DynamicCompileTransformConfig.SOURCE_CODE) + .conditional( + DynamicCompileTransformConfig.COMPILE_PATTERN, + CompilePattern.SOURCE_CODE, + Conditions.notBlank(DynamicCompileTransformConfig.SOURCE_CODE)) .conditional( DynamicCompileTransformConfig.COMPILE_PATTERN, CompilePattern.ABSOLUTE_PATH, DynamicCompileTransformConfig.ABSOLUTE_PATH) + .conditional( + DynamicCompileTransformConfig.COMPILE_PATTERN, + CompilePattern.ABSOLUTE_PATH, + Conditions.notBlank(DynamicCompileTransformConfig.ABSOLUTE_PATH)) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/encrypt/FieldEncryptTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/encrypt/FieldEncryptTransformFactory.java index d12d4292a852..3c33a095ffff 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/encrypt/FieldEncryptTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/encrypt/FieldEncryptTransformFactory.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.transform.encrypt; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -38,10 +39,17 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .required(FieldEncryptTransformConfig.FIELDS) - .required(FieldEncryptTransformConfig.KEY) + .required( + FieldEncryptTransformConfig.FIELDS, + Conditions.notEmpty(FieldEncryptTransformConfig.FIELDS)) + .required( + FieldEncryptTransformConfig.KEY, + Conditions.notBlank(FieldEncryptTransformConfig.KEY)) .optional(FieldEncryptTransformConfig.ALGORITHM) .optional(FieldEncryptTransformConfig.MODE) + .optional( + FieldEncryptTransformConfig.MAX_FIELD_LENGTH, + Conditions.greaterThan(FieldEncryptTransformConfig.MAX_FIELD_LENGTH, 0)) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/fieldmapper/FieldMapperTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/fieldmapper/FieldMapperTransformFactory.java index ade69dd3db89..8edb4419e3e9 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/fieldmapper/FieldMapperTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/fieldmapper/FieldMapperTransformFactory.java @@ -18,6 +18,7 @@ package org.apache.seatunnel.transform.fieldmapper; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -37,7 +38,9 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .optional(FieldMapperTransformConfig.FIELD_MAPPER) + .required( + FieldMapperTransformConfig.FIELD_MAPPER, + Conditions.mapNotEmpty(FieldMapperTransformConfig.FIELD_MAPPER)) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/filter/FilterFieldTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/filter/FilterFieldTransform.java index 53b66af3d842..0fee01c378b8 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/filter/FilterFieldTransform.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/filter/FilterFieldTransform.java @@ -18,8 +18,6 @@ package org.apache.seatunnel.transform.filter; import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.configuration.util.ConfigValidator; -import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.ConstraintKey; @@ -58,14 +56,6 @@ public FilterFieldTransform( SeaTunnelRowType seaTunnelRowType = catalogTable.getTableSchema().toPhysicalRowDataType(); includeFields = config.get(FilterFieldTransformConfig.INCLUDE_FIELDS); excludeFields = config.get(FilterFieldTransformConfig.EXCLUDE_FIELDS); - // exactly only one should be set - ConfigValidator.of(config) - .validate( - OptionRule.builder() - .exclusive( - FilterFieldTransformConfig.INCLUDE_FIELDS, - FilterFieldTransformConfig.EXCLUDE_FIELDS) - .build()); List canNotFoundFields = Stream.concat( Optional.ofNullable(includeFields).orElse(new ArrayList<>()) diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/filter/FilterFieldTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/filter/FilterFieldTransformFactory.java index f390f4842494..bdc273ed2758 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/filter/FilterFieldTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/filter/FilterFieldTransformFactory.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.transform.filter; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -38,9 +39,15 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .optional( + .exclusive( FilterFieldTransformConfig.INCLUDE_FIELDS, FilterFieldTransformConfig.EXCLUDE_FIELDS) + .optional( + FilterFieldTransformConfig.INCLUDE_FIELDS, + Conditions.notEmpty(FilterFieldTransformConfig.INCLUDE_FIELDS)) + .optional( + FilterFieldTransformConfig.EXCLUDE_FIELDS, + Conditions.notEmpty(FilterFieldTransformConfig.EXCLUDE_FIELDS)) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/jsonpath/JsonPathTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/jsonpath/JsonPathTransformFactory.java index 0220303fd55a..d0411fddb3a1 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/jsonpath/JsonPathTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/jsonpath/JsonPathTransformFactory.java @@ -17,7 +17,11 @@ package org.apache.seatunnel.transform.jsonpath; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; import org.apache.seatunnel.api.table.factory.TableTransformFactory; @@ -26,6 +30,9 @@ import com.google.auto.service.AutoService; +import java.util.List; +import java.util.Map; + @AutoService(Factory.class) public class JsonPathTransformFactory implements TableTransformFactory { @Override @@ -36,7 +43,13 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .optional(JsonPathTransformConfig.COLUMNS) + .required( + JsonPathTransformConfig.COLUMNS, + Conditions.notEmpty(JsonPathTransformConfig.COLUMNS) + .and( + Conditions.extension( + JsonPathTransformConfig.COLUMNS, + new ColumnsValidator()))) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .optional(TransformCommonOptions.ROW_ERROR_HANDLE_WAY_OPTION) @@ -48,4 +61,44 @@ public TableTransform createTransform(TableTransformFactoryContext context) { return () -> new JsonPathMultiCatalogTransform(context.getCatalogTables(), context.getOptions()); } + + static class ColumnsValidator implements ConditionExtension>> { + @Override + public String description() { + return "each column entry must contain non-empty 'path' and 'dest_field'"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List> value) + throws OptionValidationException { + if (value == null) { + return false; + } + for (int i = 0; i < value.size(); i++) { + Map entry = value.get(i); + Object path = entry.get("path"); + if (path == null + || (path instanceof String && ((String) path).trim().isEmpty()) + || (path instanceof List && ((List) path).isEmpty())) { + throw new OptionValidationException( + String.format("columns[%d]: 'path' must not be null or empty", i)); + } + Object srcField = entry.get("src_field"); + if (srcField == null + || (srcField instanceof String && ((String) srcField).trim().isEmpty())) { + throw new OptionValidationException( + String.format("columns[%d]: 'src_field' must not be null or empty", i)); + } + Object destField = entry.get("dest_field"); + if (destField == null + || (destField instanceof String && ((String) destField).trim().isEmpty()) + || (destField instanceof List && ((List) destField).isEmpty())) { + throw new OptionValidationException( + String.format( + "columns[%d]: 'dest_field' must not be null or empty", i)); + } + } + return true; + } + } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/metadata/MetadataTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/metadata/MetadataTransformFactory.java index b9233cd1d338..479376b8885e 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/metadata/MetadataTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/metadata/MetadataTransformFactory.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.transform.metadata; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -36,7 +37,9 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .optional(MetadataTransformConfig.METADATA_FIELDS) + .required( + MetadataTransformConfig.METADATA_FIELDS, + Conditions.mapNotEmpty(MetadataTransformConfig.METADATA_FIELDS)) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/regexextract/RegexExtractTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/regexextract/RegexExtractTransformFactory.java index d6c6fd3a49b8..0502bd27fc7f 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/regexextract/RegexExtractTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/regexextract/RegexExtractTransformFactory.java @@ -17,6 +17,9 @@ package org.apache.seatunnel.transform.regexextract; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -26,6 +29,8 @@ import com.google.auto.service.AutoService; +import java.util.List; + @AutoService(Factory.class) public class RegexExtractTransformFactory implements TableTransformFactory { @@ -37,13 +42,17 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() + .required(RegexExtractTransformConfig.KEY_SOURCE_FIELD) + .required(RegexExtractTransformConfig.KEY_REGEX_PATTERN) .required( - RegexExtractTransformConfig.KEY_SOURCE_FIELD, - RegexExtractTransformConfig.KEY_REGEX_PATTERN, - RegexExtractTransformConfig.KEY_OUTPUT_FIELDS) + RegexExtractTransformConfig.KEY_OUTPUT_FIELDS, + Conditions.notEmpty(RegexExtractTransformConfig.KEY_OUTPUT_FIELDS)) .optional( RegexExtractTransformConfig.KEY_DEFAULT_VALUES, - TransformCommonOptions.MULTI_TABLES) + Conditions.extension( + RegexExtractTransformConfig.KEY_DEFAULT_VALUES, + new DefaultValuesLengthValidator())) + .optional(TransformCommonOptions.MULTI_TABLES) .build(); } @@ -53,4 +62,20 @@ public TableTransform createTransform(TableTransformFactoryContext context) { new RegexExtractMultiCatalogTransform( context.getCatalogTables(), context.getOptions()); } + + static class DefaultValuesLengthValidator implements ConditionExtension> { + @Override + public String description() { + return "'default_values' length must equal 'output_fields' length"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List value) { + if (value == null) { + return true; + } + List outputFields = config.get(RegexExtractTransformConfig.KEY_OUTPUT_FIELDS); + return outputFields == null || value.size() == outputFields.size(); + } + } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/replace/ReplaceTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/replace/ReplaceTransform.java index 0228e1f0ecfa..0ceb74966753 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/replace/ReplaceTransform.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/replace/ReplaceTransform.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.transform.replace; -import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.TableIdentifier; @@ -32,7 +31,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -49,19 +47,9 @@ public class ReplaceTransform extends AbstractCatalogSupportMapTransform { public ReplaceTransform( @NonNull ReadonlyConfig config, @NonNull CatalogTable inputCatalogTable) { super(inputCatalogTable); - validateConflictingReplaceFieldKeys(config); - this.replaceFields.addAll( - getRequiredOption(config, ReplaceTransformConfig.KEY_REPLACE_FIELDS)); - - if (replaceFields.isEmpty()) { - throw TransformCommonError.validationFailed( - String.format( - "Option '%s' must not be empty.", - ReplaceTransformConfig.KEY_REPLACE_FIELDS.key())); - } - - this.pattern = getRequiredOption(config, ReplaceTransformConfig.KEY_PATTERN); - this.replacement = getRequiredOption(config, ReplaceTransformConfig.KEY_REPLACEMENT); + this.replaceFields.addAll(config.get(ReplaceTransformConfig.KEY_REPLACE_FIELDS)); + this.pattern = config.get(ReplaceTransformConfig.KEY_PATTERN); + this.replacement = config.get(ReplaceTransformConfig.KEY_REPLACEMENT); this.isRegex = config.get(ReplaceTransformConfig.KEY_IS_REGEX); this.replaceFirst = config.get(ReplaceTransformConfig.KEY_REPLACE_FIRST); this.regexPattern = initializeRegexPattern(); @@ -73,22 +61,6 @@ public String getPluginName() { return PLUGIN_NAME; } - private void validateConflictingReplaceFieldKeys(ReadonlyConfig config) { - Map sourceMap = config.getSourceMap(); - if (sourceMap.containsKey("replace_field") && sourceMap.containsKey("replace_fields")) { - throw TransformCommonError.validationFailed( - "Options 'replace_field' and 'replace_fields' cannot be configured together."); - } - } - - private T getRequiredOption(ReadonlyConfig config, Option option) { - return config.getOptional(option) - .orElseThrow( - () -> - TransformCommonError.validationFailed( - String.format("Option '%s' is required.", option.key()))); - } - private void initializeFieldIndexes() { SeaTunnelRowType physicalRowType = inputCatalogTable.getTableSchema().toPhysicalRowDataType(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/replace/ReplaceTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/replace/ReplaceTransformFactory.java index 4fc22cd11064..c7c7e1ff6430 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/replace/ReplaceTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/replace/ReplaceTransformFactory.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.transform.replace; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -36,10 +37,11 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .optional( + .required( ReplaceTransformConfig.KEY_REPLACE_FIELDS, - ReplaceTransformConfig.KEY_PATTERN, - ReplaceTransformConfig.KEY_REPLACEMENT) + Conditions.notEmpty(ReplaceTransformConfig.KEY_REPLACE_FIELDS)) + .required(ReplaceTransformConfig.KEY_PATTERN) + .required(ReplaceTransformConfig.KEY_REPLACEMENT) .optional(ReplaceTransformConfig.KEY_IS_REGEX) .optional(ReplaceTransformConfig.KEY_REPLACE_FIRST) .optional(TransformCommonOptions.MULTI_TABLES) diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/rowkind/RowKindExtractorTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/rowkind/RowKindExtractorTransformFactory.java index 8694b72b194f..99011726fc00 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/rowkind/RowKindExtractorTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/rowkind/RowKindExtractorTransformFactory.java @@ -37,6 +37,7 @@ public String factoryIdentifier() { public OptionRule optionRule() { return OptionRule.builder() .optional(RowKindExtractorTransformConfig.CUSTOM_FIELD_NAME) + .optional(RowKindExtractorTransformConfig.TRANSFORM_TYPE) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/split/SplitTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/split/SplitTransformFactory.java index 517d3ca5ceae..d5385a4f59ea 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/split/SplitTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/split/SplitTransformFactory.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.transform.split; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -36,10 +37,11 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .optional( - SplitTransformConfig.KEY_SEPARATOR, - SplitTransformConfig.KEY_SPLIT_FIELD, - SplitTransformConfig.KEY_OUTPUT_FIELDS) + .required(SplitTransformConfig.KEY_SEPARATOR) + .required(SplitTransformConfig.KEY_SPLIT_FIELD) + .required( + SplitTransformConfig.KEY_OUTPUT_FIELDS, + Conditions.notEmpty(SplitTransformConfig.KEY_OUTPUT_FIELDS)) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransformFactory.java index f1310cfbc0ad..4a3b6db9c566 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransformFactory.java @@ -38,7 +38,7 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .optional(KEY_QUERY) + .required(KEY_QUERY) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .build(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableFilterConfig.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableFilterConfig.java index 7498f4636365..7ee7203aba97 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableFilterConfig.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableFilterConfig.java @@ -18,7 +18,6 @@ package org.apache.seatunnel.transform.table; import org.apache.seatunnel.shade.com.fasterxml.jackson.annotation.JsonAlias; -import org.apache.seatunnel.shade.com.google.common.base.Preconditions; import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.Options; @@ -107,13 +106,6 @@ public static TableFilterConfig of(ReadonlyConfig config) { filterConfig.setSchemaPattern(config.get(SCHEMA_PATTERN)); filterConfig.setTablePattern(config.get(TABLE_PATTERN)); filterConfig.setPatternMode(config.get(PATTERN_MODE)); - - Preconditions.checkArgument( - filterConfig.getDatabasePattern() != null - || filterConfig.getSchemaPattern() != null - || filterConfig.getTablePattern() != null - || filterConfig.getPatternMode() != null, - "At least one of database_pattern, schema_pattern, table_pattern or pattern_mode must be specified."); return filterConfig; } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableFilterTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableFilterTransformFactory.java index 987168d9cd90..701d9f56976e 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableFilterTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableFilterTransformFactory.java @@ -17,6 +17,9 @@ package org.apache.seatunnel.transform.table; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -26,6 +29,9 @@ import com.google.auto.service.AutoService; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + @AutoService(Factory.class) public class TableFilterTransformFactory implements TableTransformFactory { @Override @@ -38,8 +44,15 @@ public OptionRule optionRule() { return OptionRule.builder() .optional( TableFilterConfig.DATABASE_PATTERN, + Conditions.extension( + TableFilterConfig.DATABASE_PATTERN, new RegexValidator())) + .optional( TableFilterConfig.SCHEMA_PATTERN, - TableFilterConfig.TABLE_PATTERN) + Conditions.extension( + TableFilterConfig.SCHEMA_PATTERN, new RegexValidator())) + .optional( + TableFilterConfig.TABLE_PATTERN, + Conditions.extension(TableFilterConfig.TABLE_PATTERN, new RegexValidator())) .optional(TableFilterConfig.PATTERN_MODE) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) @@ -52,4 +65,24 @@ public TableTransform createTransform(TableTransformFactoryContext context) { new TableFilterMultiCatalogTransform( context.getCatalogTables(), context.getOptions()); } + + static class RegexValidator implements ConditionExtension { + @Override + public String description() { + return "must be a valid regular expression"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, String value) { + if (value == null || value.isEmpty()) { + return true; + } + try { + Pattern.compile(value); + return true; + } catch (PatternSyntaxException e) { + return false; + } + } + } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableMergeTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableMergeTransformFactory.java index 3db9cb31250f..6bf876a23574 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableMergeTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/table/TableMergeTransformFactory.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.transform.table; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; @@ -36,7 +37,7 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .required(TableMergeConfig.TABLE) + .required(TableMergeConfig.TABLE, Conditions.notBlank(TableMergeConfig.TABLE)) .optional(TableMergeConfig.DATABASE, TableMergeConfig.SCHEMA) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/DataValidatorTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/DataValidatorTransformFactory.java index f3d3760d46ae..b391d8339a17 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/DataValidatorTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/DataValidatorTransformFactory.java @@ -17,7 +17,11 @@ package org.apache.seatunnel.transform.validator; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.Conditions; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.connector.TableTransform; import org.apache.seatunnel.api.table.factory.Factory; import org.apache.seatunnel.api.table.factory.TableTransformFactory; @@ -26,6 +30,9 @@ import com.google.auto.service.AutoService; +import java.util.List; +import java.util.Map; + import static org.apache.seatunnel.transform.validator.DataValidatorTransformConfig.FIELD_RULES; /** Factory for creating DataValidator Transform instances. */ @@ -40,7 +47,10 @@ public String factoryIdentifier() { @Override public OptionRule optionRule() { return OptionRule.builder() - .required(FIELD_RULES) + .required( + FIELD_RULES, + Conditions.notEmpty(FIELD_RULES) + .and(Conditions.extension(FIELD_RULES, new FieldRulesValidator()))) .optional(TransformCommonOptions.MULTI_TABLES) .optional(TransformCommonOptions.TABLE_MATCH_REGEX) .optional(TransformCommonOptions.ROW_ERROR_HANDLE_WAY_OPTION) @@ -53,4 +63,39 @@ public TableTransform createTransform(TableTransformFactoryContext context) { return () -> new DataValidatorTransform(context.getOptions(), context.getCatalogTables().get(0)); } + + static class FieldRulesValidator implements ConditionExtension>> { + @Override + public String description() { + return "each field_rules entry must contain 'field_name' and either 'rule_type' or 'rules'"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, List> value) + throws OptionValidationException { + if (value == null) { + return false; + } + for (int i = 0; i < value.size(); i++) { + Map entry = value.get(i); + Object fieldName = entry.get("field_name"); + if (fieldName == null + || (fieldName instanceof String && ((String) fieldName).trim().isEmpty())) { + throw new OptionValidationException( + String.format( + "field_rules[%d]: 'field_name' must not be null or empty", i)); + } + boolean hasRuleType = entry.containsKey("rule_type"); + Object rulesObj = entry.get("rules"); + boolean hasRules = rulesObj instanceof List && !((List) rulesObj).isEmpty(); + if (!hasRuleType && !hasRules) { + throw new OptionValidationException( + String.format( + "field_rules[%d]: must contain 'rule_type' or non-empty 'rules'", + i)); + } + } + return true; + } + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformFactoryTest.java new file mode 100644 index 000000000000..a3974a35f15f --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/adaptsink/DefineSinkTypeTransformFactoryTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.adaptsink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class DefineSinkTypeTransformFactoryTest { + + private final OptionRule rule = new DefineSinkTypeTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + private Map columnEntry(String column, String type) { + Map entry = new HashMap<>(); + entry.put("column", column); + entry.put("type", type); + return entry; + } + + @Test + void testValidConfig() { + Map cfg = new HashMap<>(); + cfg.put( + "columns", + Arrays.asList(columnEntry("id", "bigint"), columnEntry("name", "string"))); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testMissingColumnsFails() { + Map cfg = new HashMap<>(); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyColumnsFails() { + Map cfg = new HashMap<>(); + cfg.put("columns", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testColumnWithNullNameFails() { + Map cfg = new HashMap<>(); + cfg.put("columns", Arrays.asList(columnEntry(null, "bigint"))); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testColumnWithNullTypeFails() { + Map cfg = new HashMap<>(); + cfg.put("columns", Arrays.asList(columnEntry("id", null))); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testColumnWithEmptyNameFails() { + Map cfg = new HashMap<>(); + cfg.put("columns", Arrays.asList(columnEntry("", "bigint"))); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testDuplicateColumnNameFails() { + Map cfg = new HashMap<>(); + cfg.put("columns", Arrays.asList(columnEntry("id", "bigint"), columnEntry("id", "string"))); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/copy/CopyFieldTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/copy/CopyFieldTransformFactoryTest.java new file mode 100644 index 000000000000..39937cca5d96 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/copy/CopyFieldTransformFactoryTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.copy; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class CopyFieldTransformFactoryTest { + + private final OptionRule rule = new CopyFieldTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + @Test + void testValidFieldsMapConfig() { + Map cfg = new HashMap<>(); + Map fields = new HashMap<>(); + fields.put("new_col", "old_col"); + cfg.put("fields", fields); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testValidSrcDestConfig() { + Map cfg = new HashMap<>(); + cfg.put("src_field", "old_col"); + cfg.put("dest_field", "new_col"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testEmptyFieldsMapFails() { + Map cfg = new HashMap<>(); + cfg.put("fields", Collections.emptyMap()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testNeitherFieldsNorSrcFieldProvidedFails() { + Map cfg = new HashMap<>(); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testSrcFieldWithoutDestFieldFails() { + Map cfg = new HashMap<>(); + cfg.put("src_field", "old_col"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testDestFieldWithoutSrcFieldFails() { + Map cfg = new HashMap<>(); + cfg.put("dest_field", "new_col"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/dynamiccompile/DynamicCompileTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/dynamiccompile/DynamicCompileTransformFactoryTest.java new file mode 100644 index 000000000000..704c80aadd55 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/dynamiccompile/DynamicCompileTransformFactoryTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.dynamiccompile; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class DynamicCompileTransformFactoryTest { + + private final OptionRule rule = new DynamicCompileTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + @Test + void testValidSourceCodeConfig() { + Map cfg = new HashMap<>(); + cfg.put("compile_language", "JAVA"); + cfg.put("compile_pattern", "SOURCE_CODE"); + cfg.put("source_code", "public class Transform {}"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testValidAbsolutePathConfig() { + Map cfg = new HashMap<>(); + cfg.put("compile_language", "JAVA"); + cfg.put("compile_pattern", "ABSOLUTE_PATH"); + cfg.put("absolute_path", "/tmp/Transform.java"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testMissingCompileLanguageFails() { + Map cfg = new HashMap<>(); + cfg.put("compile_pattern", "SOURCE_CODE"); + cfg.put("source_code", "public class Transform {}"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testSourceCodeBlankFails() { + Map cfg = new HashMap<>(); + cfg.put("compile_language", "JAVA"); + cfg.put("compile_pattern", "SOURCE_CODE"); + cfg.put("source_code", " "); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testAbsolutePathBlankFails() { + Map cfg = new HashMap<>(); + cfg.put("compile_language", "JAVA"); + cfg.put("compile_pattern", "ABSOLUTE_PATH"); + cfg.put("absolute_path", " "); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testSourceCodeMissingKeyFails() { + Map cfg = new HashMap<>(); + cfg.put("compile_language", "JAVA"); + cfg.put("compile_pattern", "SOURCE_CODE"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testAbsolutePathMissingKeyFails() { + Map cfg = new HashMap<>(); + cfg.put("compile_language", "JAVA"); + cfg.put("compile_pattern", "ABSOLUTE_PATH"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testDefaultPatternMissingSourceCodeFails() { + Map cfg = new HashMap<>(); + cfg.put("compile_language", "JAVA"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/encrypt/FieldEncryptTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/encrypt/FieldEncryptTransformFactoryTest.java new file mode 100644 index 000000000000..0d34a6a3ddda --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/encrypt/FieldEncryptTransformFactoryTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.encrypt; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class FieldEncryptTransformFactoryTest { + + private final OptionRule rule = new FieldEncryptTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + private Map validConfig() { + Map cfg = new HashMap<>(); + cfg.put("fields", Arrays.asList("password", "ssn")); + cfg.put("key", "1234567890abcdef"); + return cfg; + } + + @Test + void testValidConfig() { + Assertions.assertDoesNotThrow(() -> validate(validConfig())); + } + + @Test + void testMissingFieldsFails() { + Map cfg = validConfig(); + cfg.remove("fields"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyFieldsFails() { + Map cfg = validConfig(); + cfg.put("fields", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingKeyFails() { + Map cfg = validConfig(); + cfg.remove("key"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testBlankKeyFails() { + Map cfg = validConfig(); + cfg.put("key", " "); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMaxFieldLengthZeroFails() { + Map cfg = validConfig(); + cfg.put("max_field_length", 0); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMaxFieldLengthNegativeFails() { + Map cfg = validConfig(); + cfg.put("max_field_length", -1); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMaxFieldLengthPositiveValid() { + Map cfg = validConfig(); + cfg.put("max_field_length", 1024); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/fieldmapper/FieldMapperTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/fieldmapper/FieldMapperTransformFactoryTest.java new file mode 100644 index 000000000000..5ed2b0e2c517 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/fieldmapper/FieldMapperTransformFactoryTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.fieldmapper; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class FieldMapperTransformFactoryTest { + + private final OptionRule rule = new FieldMapperTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + @Test + void testValidConfig() { + Map cfg = new HashMap<>(); + Map fieldMapper = new HashMap<>(); + fieldMapper.put("old_name", "new_name"); + cfg.put("field_mapper", fieldMapper); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testMissingFieldMapperFails() { + Map cfg = new HashMap<>(); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyFieldMapperFails() { + Map cfg = new HashMap<>(); + cfg.put("field_mapper", Collections.emptyMap()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/filter/FilterFieldTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/filter/FilterFieldTransformFactoryTest.java new file mode 100644 index 000000000000..489ba5f8c9f0 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/filter/FilterFieldTransformFactoryTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.filter; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class FilterFieldTransformFactoryTest { + + private final OptionRule rule = new FilterFieldTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + @Test + void testIncludeFieldsValid() { + Map cfg = new HashMap<>(); + cfg.put("include_fields", Arrays.asList("id", "name")); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testExcludeFieldsValid() { + Map cfg = new HashMap<>(); + cfg.put("exclude_fields", Arrays.asList("password", "secret")); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testNeitherFieldsFails() { + Map cfg = new HashMap<>(); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testBothFieldsFails() { + Map cfg = new HashMap<>(); + cfg.put("include_fields", Arrays.asList("id")); + cfg.put("exclude_fields", Arrays.asList("password")); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testIncludeFieldsEmptyFails() { + Map cfg = new HashMap<>(); + cfg.put("include_fields", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testExcludeFieldsEmptyFails() { + Map cfg = new HashMap<>(); + cfg.put("exclude_fields", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/filter/FilterFieldTransformTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/filter/FilterFieldTransformTest.java index e1ca9b88276b..b33a83f7b905 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/filter/FilterFieldTransformTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/filter/FilterFieldTransformTest.java @@ -18,6 +18,8 @@ package org.apache.seatunnel.transform.filter; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; @@ -99,13 +101,15 @@ static void setUp() { @Test void testConfig() { - // test both not set + OptionRule rule = new FilterFieldTransformFactory().optionRule(); + + // test both not set — validated via Factory optionRule OptionValidationException noneSetEx = Assertions.assertThrows( OptionValidationException.class, () -> - new FilterFieldTransform( - ReadonlyConfig.fromMap(new HashMap<>()), catalogTable)); + ConfigValidator.of(ReadonlyConfig.fromMap(new HashMap<>())) + .validate(rule)); Assertions.assertTrue( noneSetEx.getMessage().contains("'include_fields'"), "Should mention include_fields: " + noneSetEx.getMessage()); @@ -121,23 +125,23 @@ void testConfig() { Assertions.assertThrows( OptionValidationException.class, () -> - new FilterFieldTransform( - ReadonlyConfig.fromMap( - new HashMap() { - { - put( - FilterFieldTransformConfig - .INCLUDE_FIELDS - .key(), - filterKeys); - put( - FilterFieldTransformConfig - .EXCLUDE_FIELDS - .key(), - filterKeys); - } - }), - catalogTable)); + ConfigValidator.of( + ReadonlyConfig.fromMap( + new HashMap() { + { + put( + FilterFieldTransformConfig + .INCLUDE_FIELDS + .key(), + filterKeys); + put( + FilterFieldTransformConfig + .EXCLUDE_FIELDS + .key(), + filterKeys); + } + })) + .validate(rule)); Assertions.assertTrue( bothSetEx.getMessage().contains("mutually exclusive"), "Should mention mutually exclusive: " + bothSetEx.getMessage()); diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/jsonpath/JsonPathTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/jsonpath/JsonPathTransformFactoryTest.java new file mode 100644 index 000000000000..8e1b4edfa2ec --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/jsonpath/JsonPathTransformFactoryTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.jsonpath; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class JsonPathTransformFactoryTest { + + private final OptionRule rule = new JsonPathTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + private Map columnEntry(String path, String srcField, String destField) { + Map entry = new HashMap<>(); + entry.put("path", path); + entry.put("src_field", srcField); + entry.put("dest_field", destField); + return entry; + } + + @Test + void testValidConfig() { + Map cfg = new HashMap<>(); + cfg.put("columns", Arrays.asList(columnEntry("$.data.name", "raw", "name"))); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testMissingColumnsFails() { + Map cfg = new HashMap<>(); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyColumnsFails() { + Map cfg = new HashMap<>(); + cfg.put("columns", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testColumnWithNullPathFails() { + Map cfg = new HashMap<>(); + Map entry = new HashMap<>(); + entry.put("src_field", "raw"); + entry.put("dest_field", "name"); + cfg.put("columns", Arrays.asList(entry)); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testColumnWithEmptyDestFieldFails() { + Map cfg = new HashMap<>(); + Map entry = new HashMap<>(); + entry.put("path", "$.data.name"); + entry.put("src_field", "raw"); + entry.put("dest_field", ""); + cfg.put("columns", Arrays.asList(entry)); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testColumnWithMissingSrcFieldFails() { + Map cfg = new HashMap<>(); + Map entry = new HashMap<>(); + entry.put("path", "$.data.name"); + entry.put("dest_field", "name"); + cfg.put("columns", Arrays.asList(entry)); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testColumnWithEmptySrcFieldFails() { + Map cfg = new HashMap<>(); + Map entry = new HashMap<>(); + entry.put("path", "$.data.name"); + entry.put("src_field", ""); + entry.put("dest_field", "name"); + cfg.put("columns", Arrays.asList(entry)); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/MetadataTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/MetadataTransformFactoryTest.java new file mode 100644 index 000000000000..f02f7bcf490e --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/MetadataTransformFactoryTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.metadata; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class MetadataTransformFactoryTest { + + private final OptionRule rule = new MetadataTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + @Test + void testValidConfig() { + Map cfg = new HashMap<>(); + Map metadataFields = new HashMap<>(); + metadataFields.put("database", "db_field"); + cfg.put("metadata_fields", metadataFields); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testMissingMetadataFieldsFails() { + Map cfg = new HashMap<>(); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyMetadataFieldsFails() { + Map cfg = new HashMap<>(); + cfg.put("metadata_fields", Collections.emptyMap()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/regexextract/RegexExtractTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/regexextract/RegexExtractTransformFactoryTest.java new file mode 100644 index 000000000000..28731c4076ea --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/regexextract/RegexExtractTransformFactoryTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.regexextract; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class RegexExtractTransformFactoryTest { + + private final OptionRule rule = new RegexExtractTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + private Map validConfig() { + Map cfg = new HashMap<>(); + cfg.put("source_field", "log_line"); + cfg.put("regex_pattern", "(\\d+)-(\\w+)"); + cfg.put("output_fields", Arrays.asList("id", "name")); + return cfg; + } + + @Test + void testValidConfig() { + Assertions.assertDoesNotThrow(() -> validate(validConfig())); + } + + @Test + void testValidConfigWithMatchingDefaults() { + Map cfg = validConfig(); + cfg.put("default_values", Arrays.asList("0", "unknown")); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testMissingSourceFieldFails() { + Map cfg = validConfig(); + cfg.remove("source_field"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingRegexPatternFails() { + Map cfg = validConfig(); + cfg.remove("regex_pattern"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingOutputFieldsFails() { + Map cfg = validConfig(); + cfg.remove("output_fields"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testDefaultValuesLengthMismatchFails() { + Map cfg = validConfig(); + cfg.put("default_values", Arrays.asList("0", "unknown", "extra")); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyOutputFieldsFails() { + Map cfg = new HashMap<>(); + cfg.put("source_field", "log_line"); + cfg.put("regex_pattern", "(\\d+)"); + cfg.put("output_fields", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/replace/ReplaceTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/replace/ReplaceTransformFactoryTest.java new file mode 100644 index 000000000000..bb93a6edbc85 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/replace/ReplaceTransformFactoryTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.replace; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class ReplaceTransformFactoryTest { + + private final OptionRule rule = new ReplaceTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + private Map validConfig() { + Map cfg = new HashMap<>(); + cfg.put("replace_fields", Arrays.asList("name")); + cfg.put("pattern", "old"); + cfg.put("replacement", "new"); + return cfg; + } + + @Test + void testValidConfig() { + Assertions.assertDoesNotThrow(() -> validate(validConfig())); + } + + @Test + void testMissingReplaceFieldsFails() { + Map cfg = validConfig(); + cfg.remove("replace_fields"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyReplaceFieldsFails() { + Map cfg = validConfig(); + cfg.put("replace_fields", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingPatternFails() { + Map cfg = validConfig(); + cfg.remove("pattern"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingReplacementFails() { + Map cfg = validConfig(); + cfg.remove("replacement"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testFallbackKeyValid() { + Map cfg = new HashMap<>(); + cfg.put("replace_field", "name"); + cfg.put("pattern", "old"); + cfg.put("replacement", "new"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/replace/ReplaceTransformTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/replace/ReplaceTransformTest.java index c716627c955b..a823c77c9f96 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/replace/ReplaceTransformTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/replace/ReplaceTransformTest.java @@ -18,6 +18,9 @@ package org.apache.seatunnel.transform.replace; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; import org.apache.seatunnel.api.table.catalog.TableIdentifier; @@ -114,22 +117,17 @@ void testMultipleFieldReplaceWithList() { } @Test - void testRejectConflictingReplaceFieldKeys() { + void testFallbackKeyUsedWhenPrimaryAbsent() { Map configMap = new HashMap<>(); configMap.put("replace_field", "name"); - configMap.put( - ReplaceTransformConfig.KEY_REPLACE_FIELDS.key(), Arrays.asList("name", "title")); configMap.put(ReplaceTransformConfig.KEY_PATTERN.key(), "before"); configMap.put(ReplaceTransformConfig.KEY_REPLACEMENT.key(), "after"); - TransformException exception = - Assertions.assertThrows( - TransformException.class, - () -> - new ReplaceTransform( - ReadonlyConfig.fromMap(configMap), catalogTable)); - - Assertions.assertTrue(exception.getMessage().contains("cannot be configured together")); + ReplaceTransform transform = + new ReplaceTransform(ReadonlyConfig.fromMap(configMap), catalogTable); + SeaTunnelRow input = new SeaTunnelRow(new Object[] {1, "before name", "title"}); + SeaTunnelRow output = transform.transformRow(input); + Assertions.assertEquals("after name", output.getField(1)); } @Test @@ -180,19 +178,20 @@ void testInvalidRegexPattern() { @Test void testEmptyReplaceFieldsValidation() { + OptionRule rule = new ReplaceTransformFactory().optionRule(); Map configMap = new HashMap<>(); configMap.put(ReplaceTransformConfig.KEY_REPLACE_FIELDS.key(), new ArrayList()); configMap.put(ReplaceTransformConfig.KEY_PATTERN.key(), "before"); configMap.put(ReplaceTransformConfig.KEY_REPLACEMENT.key(), "after"); - TransformException exception = + OptionValidationException exception = Assertions.assertThrows( - TransformException.class, - () -> - new ReplaceTransform( - ReadonlyConfig.fromMap(configMap), catalogTable)); + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(configMap)).validate(rule)); - Assertions.assertTrue(exception.getMessage().contains("must not be empty")); + Assertions.assertTrue( + exception.getMessage().contains("replace_fields"), + "Should mention replace_fields: " + exception.getMessage()); } @Test diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/split/SplitTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/split/SplitTransformFactoryTest.java new file mode 100644 index 000000000000..a7f53e00f5f6 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/split/SplitTransformFactoryTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.split; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class SplitTransformFactoryTest { + + private final OptionRule rule = new SplitTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + private Map validConfig() { + Map cfg = new HashMap<>(); + cfg.put("separator", " "); + cfg.put("split_field", "name"); + cfg.put("output_fields", Arrays.asList("first_name", "last_name")); + return cfg; + } + + @Test + void testValidConfig() { + Assertions.assertDoesNotThrow(() -> validate(validConfig())); + } + + @Test + void testMissingSeparatorFails() { + Map cfg = validConfig(); + cfg.remove("separator"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingSplitFieldFails() { + Map cfg = validConfig(); + cfg.remove("split_field"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingOutputFieldsFails() { + Map cfg = validConfig(); + cfg.remove("output_fields"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyOutputFieldsFails() { + Map cfg = validConfig(); + cfg.put("output_fields", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformFactoryTest.java index 5e79df60c137..5add6ab546ae 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformFactoryTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformFactoryTest.java @@ -18,7 +18,9 @@ package org.apache.seatunnel.transform.sql; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; import org.apache.seatunnel.api.table.connector.TableTransform; @@ -32,7 +34,9 @@ import org.junit.jupiter.api.Test; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class SQLTransformFactoryTest { @@ -73,4 +77,22 @@ public void testCreateTransformReturnsMultiCatalogTransform() { Assertions.assertNotNull(inner); Assertions.assertTrue(inner instanceof SQLMultiCatalogFlatMapTransform); } + + @Test + public void testValidConfigWithQuery() { + OptionRule rule = new SQLTransformFactory().optionRule(); + Map cfg = new HashMap<>(); + cfg.put("query", "SELECT id, name FROM table1"); + Assertions.assertDoesNotThrow( + () -> ConfigValidator.of(ReadonlyConfig.fromMap(cfg)).validate(rule)); + } + + @Test + public void testMissingQueryFails() { + OptionRule rule = new SQLTransformFactory().optionRule(); + Map cfg = new HashMap<>(); + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(cfg)).validate(rule)); + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/table/TableFilterTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/table/TableFilterTransformFactoryTest.java new file mode 100644 index 000000000000..3f0e665e99c7 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/table/TableFilterTransformFactoryTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.table; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class TableFilterTransformFactoryTest { + + private final OptionRule rule = new TableFilterTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + @Test + void testValidWithTablePattern() { + Map cfg = new HashMap<>(); + cfg.put("table_pattern", "user_.*"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testValidWithDatabasePattern() { + Map cfg = new HashMap<>(); + cfg.put("database_pattern", "prod_.*"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testValidWithSchemaPattern() { + Map cfg = new HashMap<>(); + cfg.put("schema_pattern", "public"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testInvalidDatabasePatternFails() { + Map cfg = new HashMap<>(); + cfg.put("database_pattern", "[unclosed"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testInvalidSchemaPatternFails() { + Map cfg = new HashMap<>(); + cfg.put("schema_pattern", "(missing_paren"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testInvalidTablePatternFails() { + Map cfg = new HashMap<>(); + cfg.put("table_pattern", "*invalid"); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testPatternModeWithTablePatternValid() { + Map cfg = new HashMap<>(); + cfg.put("pattern_mode", "EXCLUDE"); + cfg.put("table_pattern", "tmp_.*"); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } +} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/validator/DataValidatorTransformFactoryTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/validator/DataValidatorTransformFactoryTest.java new file mode 100644 index 000000000000..d2f99ec30f7f --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/validator/DataValidatorTransformFactoryTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.validator; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class DataValidatorTransformFactoryTest { + + private final OptionRule rule = new DataValidatorTransformFactory().optionRule(); + + private void validate(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule); + } + + @Test + void testValidConfig() { + Map cfg = new HashMap<>(); + Map fieldRule = new HashMap<>(); + fieldRule.put("field_name", "age"); + fieldRule.put("rule_type", "NOT_NULL"); + cfg.put("field_rules", Arrays.asList(fieldRule)); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testMissingFieldRulesFails() { + Map cfg = new HashMap<>(); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testEmptyFieldRulesFails() { + Map cfg = new HashMap<>(); + cfg.put("field_rules", Collections.emptyList()); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingFieldNameFails() { + Map cfg = new HashMap<>(); + Map fieldRule = new HashMap<>(); + fieldRule.put("rule_type", "NOT_NULL"); + cfg.put("field_rules", Arrays.asList(fieldRule)); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testMissingRuleTypeAndRulesFails() { + Map cfg = new HashMap<>(); + Map fieldRule = new HashMap<>(); + fieldRule.put("field_name", "age"); + cfg.put("field_rules", Arrays.asList(fieldRule)); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } + + @Test + void testNestedRulesValid() { + Map cfg = new HashMap<>(); + Map fieldRule = new HashMap<>(); + fieldRule.put("field_name", "email"); + Map nestedRule = new HashMap<>(); + nestedRule.put("rule_type", "NOT_NULL"); + fieldRule.put("rules", Arrays.asList(nestedRule)); + cfg.put("field_rules", Arrays.asList(fieldRule)); + Assertions.assertDoesNotThrow(() -> validate(cfg)); + } + + @Test + void testEmptyRulesListFails() { + Map cfg = new HashMap<>(); + Map fieldRule = new HashMap<>(); + fieldRule.put("field_name", "email"); + fieldRule.put("rules", Collections.emptyList()); + cfg.put("field_rules", Arrays.asList(fieldRule)); + Assertions.assertThrows(OptionValidationException.class, () -> validate(cfg)); + } +} From 6e94897c3261acff3464d551bcd91c255b0fec90 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Sat, 27 Jun 2026 21:11:04 +0800 Subject: [PATCH 072/375] [Docs][Connector-V2] STIP-23 Phase 3: Add timer flush feature flag to all sink connector docs (#10802) --- docs/en/connectors/sink/Aerospike.md | 1 + docs/en/connectors/sink/Airtable.md | 1 + docs/en/connectors/sink/AmazonDynamoDB.md | 1 + docs/en/connectors/sink/AmazonSqs.md | 1 + docs/en/connectors/sink/Assert.md | 1 + docs/en/connectors/sink/BigQuery.md | 1 + docs/en/connectors/sink/Clickhouse.md | 1 + docs/en/connectors/sink/Cloudberry.md | 1 + docs/en/connectors/sink/Console.md | 1 + docs/en/connectors/sink/CosFile.md | 1 + docs/en/connectors/sink/DB2.md | 1 + docs/en/connectors/sink/Databend.md | 1 + docs/en/connectors/sink/Doris.md | 1 + docs/en/connectors/sink/DuckDB.md | 1 + docs/en/connectors/sink/Feishu.md | 1 + docs/en/connectors/sink/Fluss.md | 1 + docs/en/connectors/sink/GraphQL.md | 1 + docs/en/connectors/sink/Greenplum.md | 1 + docs/en/connectors/sink/HdfsFile.md | 1 + docs/en/connectors/sink/Http.md | 1 + docs/en/connectors/sink/IoTDB.md | 1 + docs/en/connectors/sink/IoTDBv2.md | 1 + docs/en/connectors/sink/Kafka.md | 1 + docs/en/connectors/sink/Kingbase.md | 1 + docs/en/connectors/sink/Kudu.md | 1 + docs/en/connectors/sink/LocalFile.md | 1 + docs/en/connectors/sink/Milvus.md | 1 + docs/en/connectors/sink/Mysql.md | 1 + docs/en/connectors/sink/OceanBase.md | 1 + docs/en/connectors/sink/Oracle.md | 1 + docs/en/connectors/sink/PostgreSql.md | 1 + docs/en/connectors/sink/Prometheus.md | 1 + docs/en/connectors/sink/Redshift.md | 1 + docs/en/connectors/sink/RocketMQ.md | 1 + docs/en/connectors/sink/S3File.md | 1 + docs/en/connectors/sink/SelectDB-Cloud.md | 1 + docs/en/connectors/sink/Sls.md | 1 + docs/en/connectors/sink/Snowflake.md | 1 + docs/en/connectors/sink/SqlServer.md | 1 + docs/en/connectors/sink/StarRocks.md | 1 + docs/en/connectors/sink/Typesense.md | 1 + docs/en/connectors/sink/Vertica.md | 1 + docs/zh/connectors/sink/Aerospike.md | 1 + docs/zh/connectors/sink/Assert.md | 1 + docs/zh/connectors/sink/BigQuery.md | 1 + docs/zh/connectors/sink/Clickhouse.md | 1 + docs/zh/connectors/sink/ClickhouseFile.md | 1 + docs/zh/connectors/sink/Cloudberry.md | 1 + docs/zh/connectors/sink/Console.md | 1 + docs/zh/connectors/sink/Databend.md | 1 + docs/zh/connectors/sink/DingTalk.md | 1 + docs/zh/connectors/sink/Doris.md | 1 + docs/zh/connectors/sink/Elasticsearch.md | 1 + docs/zh/connectors/sink/Email.md | 1 + docs/zh/connectors/sink/Feishu.md | 1 + docs/zh/connectors/sink/Fluss.md | 1 + docs/zh/connectors/sink/FtpFile.md | 1 + docs/zh/connectors/sink/GoogleFirestore.md | 1 + docs/zh/connectors/sink/GraphQL.md | 1 + docs/zh/connectors/sink/Hbase.md | 1 + docs/zh/connectors/sink/HdfsFile.md | 1 + docs/zh/connectors/sink/Http.md | 1 + docs/zh/connectors/sink/Iceberg.md | 1 + docs/zh/connectors/sink/IoTDB.md | 1 + docs/zh/connectors/sink/IoTDBv2.md | 1 + docs/zh/connectors/sink/Kafka.md | 1 + docs/zh/connectors/sink/Kudu.md | 1 + docs/zh/connectors/sink/Lance.md | 1 + docs/zh/connectors/sink/LocalFile.md | 1 + docs/zh/connectors/sink/Milvus.md | 1 + docs/zh/connectors/sink/ObsFile.md | 1 + docs/zh/connectors/sink/Oracle.md | 1 + docs/zh/connectors/sink/Paimon.md | 1 + docs/zh/connectors/sink/Phoenix.md | 1 + docs/zh/connectors/sink/PostgreSql.md | 1 + docs/zh/connectors/sink/Prometheus.md | 1 + docs/zh/connectors/sink/Pulsar.md | 1 + docs/zh/connectors/sink/Rabbitmq.md | 1 + docs/zh/connectors/sink/RocketMQ.md | 1 + docs/zh/connectors/sink/S3-Redshift.md | 1 + docs/zh/connectors/sink/S3File.md | 1 + docs/zh/connectors/sink/SelectDB-Cloud.md | 1 + docs/zh/connectors/sink/SftpFile.md | 1 + docs/zh/connectors/sink/Sls.md | 1 + docs/zh/connectors/sink/Snowflake.md | 1 + docs/zh/connectors/sink/Socket.md | 1 + docs/zh/connectors/sink/SqlServer.md | 1 + docs/zh/connectors/sink/StarRocks.md | 1 + docs/zh/connectors/sink/TDengine.md | 1 + docs/zh/connectors/sink/Tablestore.md | 1 + docs/zh/connectors/sink/Typesense.md | 1 + docs/zh/connectors/sink/Vertica.md | 1 + 92 files changed, 92 insertions(+) diff --git a/docs/en/connectors/sink/Aerospike.md b/docs/en/connectors/sink/Aerospike.md index 1a7bc5f2b46a..db39777597fa 100644 --- a/docs/en/connectors/sink/Aerospike.md +++ b/docs/en/connectors/sink/Aerospike.md @@ -19,6 +19,7 @@ When using this connector, you need to comply with AGPL 3.0 license terms. - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Airtable.md b/docs/en/connectors/sink/Airtable.md index 5c50ae016f84..8d8cb1b959f6 100644 --- a/docs/en/connectors/sink/Airtable.md +++ b/docs/en/connectors/sink/Airtable.md @@ -13,6 +13,7 @@ Used to write data to Airtable. - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) - [ ] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Options diff --git a/docs/en/connectors/sink/AmazonDynamoDB.md b/docs/en/connectors/sink/AmazonDynamoDB.md index 1bae54cdbcab..4320cc1977a8 100644 --- a/docs/en/connectors/sink/AmazonDynamoDB.md +++ b/docs/en/connectors/sink/AmazonDynamoDB.md @@ -12,6 +12,7 @@ Write data to Amazon DynamoDB - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Options diff --git a/docs/en/connectors/sink/AmazonSqs.md b/docs/en/connectors/sink/AmazonSqs.md index 968db55fcf64..1951afc25511 100644 --- a/docs/en/connectors/sink/AmazonSqs.md +++ b/docs/en/connectors/sink/AmazonSqs.md @@ -22,6 +22,7 @@ Write data to Amazon SQS - [ ] [column projection](../../introduction/concepts/connector-v2-features.md) - [ ] [parallelism](../../introduction/concepts/connector-v2-features.md) - [ ] [support user-defined split](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Sink Options diff --git a/docs/en/connectors/sink/Assert.md b/docs/en/connectors/sink/Assert.md index a2930b1ae1c9..2f7e5ddfce4e 100644 --- a/docs/en/connectors/sink/Assert.md +++ b/docs/en/connectors/sink/Assert.md @@ -11,6 +11,7 @@ A sink plugin which can assert illegal data by user defined rules ## Key Features - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Options diff --git a/docs/en/connectors/sink/BigQuery.md b/docs/en/connectors/sink/BigQuery.md index 6baeb3b7a1a2..7018bcc09bea 100644 --- a/docs/en/connectors/sink/BigQuery.md +++ b/docs/en/connectors/sink/BigQuery.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-bigquery.md'; - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) for batch mode only - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Clickhouse.md b/docs/en/connectors/sink/Clickhouse.md index afb1760f89c8..cef70da88f92 100644 --- a/docs/en/connectors/sink/Clickhouse.md +++ b/docs/en/connectors/sink/Clickhouse.md @@ -18,6 +18,7 @@ import ChangeLog from '../changelog/connector-clickhouse.md'; > The Clickhouse sink plug-in can achieve accuracy once by implementing idempotent writing, and needs to cooperate with aggregatingmergetree and other engines that support deduplication. - [x] [support multiple table sink](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Cloudberry.md b/docs/en/connectors/sink/Cloudberry.md index f4e87a14ef02..7d3b5fe0a351 100644 --- a/docs/en/connectors/sink/Cloudberry.md +++ b/docs/en/connectors/sink/Cloudberry.md @@ -34,6 +34,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/Console.md b/docs/en/connectors/sink/Console.md index 38471c96698e..ad5b520dd9d6 100644 --- a/docs/en/connectors/sink/Console.md +++ b/docs/en/connectors/sink/Console.md @@ -23,6 +23,7 @@ Used to send data to Console. Both support streaming and batch mode. ## Key Features - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Options diff --git a/docs/en/connectors/sink/CosFile.md b/docs/en/connectors/sink/CosFile.md index a77b6b249c6d..6b00353e448b 100644 --- a/docs/en/connectors/sink/CosFile.md +++ b/docs/en/connectors/sink/CosFile.md @@ -40,6 +40,7 @@ To use this connector you need put hadoop-cos-{hadoop.version}-{version}.jar and - [x] canal_json - [x] debezium_json - [x] maxwell_json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Options diff --git a/docs/en/connectors/sink/DB2.md b/docs/en/connectors/sink/DB2.md index 7941dc136c15..22418f6565b8 100644 --- a/docs/en/connectors/sink/DB2.md +++ b/docs/en/connectors/sink/DB2.md @@ -32,6 +32,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/Databend.md b/docs/en/connectors/sink/Databend.md index 3cdbfdd762b4..92704b1632e8 100644 --- a/docs/en/connectors/sink/Databend.md +++ b/docs/en/connectors/sink/Databend.md @@ -16,6 +16,7 @@ import ChangeLog from '../changelog/connector-databend.md'; - [x] [Exactly-Once](../../introduction/concepts/connector-v2-features.md) - [x] [CDC](../../introduction/concepts/connector-v2-features.md) - [x] [Parallelism](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Doris.md b/docs/en/connectors/sink/Doris.md index c199340377e4..ab622f065704 100644 --- a/docs/en/connectors/sink/Doris.md +++ b/docs/en/connectors/sink/Doris.md @@ -21,6 +21,7 @@ import ChangeLog from '../changelog/connector-doris.md'; - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/DuckDB.md b/docs/en/connectors/sink/DuckDB.md index 8ce9ee1cb698..6b2c149a739c 100644 --- a/docs/en/connectors/sink/DuckDB.md +++ b/docs/en/connectors/sink/DuckDB.md @@ -36,6 +36,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/Feishu.md b/docs/en/connectors/sink/Feishu.md index 81ec5e9be5d7..5a28a5ee1c87 100644 --- a/docs/en/connectors/sink/Feishu.md +++ b/docs/en/connectors/sink/Feishu.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-http-feishu.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Fluss.md b/docs/en/connectors/sink/Fluss.md index ada08ff300c8..a6f8141e29a3 100644 --- a/docs/en/connectors/sink/Fluss.md +++ b/docs/en/connectors/sink/Fluss.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-fluss.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/GraphQL.md b/docs/en/connectors/sink/GraphQL.md index b7d0c51b559e..fe61172e32fe 100644 --- a/docs/en/connectors/sink/GraphQL.md +++ b/docs/en/connectors/sink/GraphQL.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-graphql.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Greenplum.md b/docs/en/connectors/sink/Greenplum.md index f034b17bb564..5d9e23dff90f 100644 --- a/docs/en/connectors/sink/Greenplum.md +++ b/docs/en/connectors/sink/Greenplum.md @@ -17,6 +17,7 @@ Write data to Greenplum using [Jdbc connector](Jdbc.md). Not support exactly-once semantics (XA transaction is not yet supported in Greenplum database). ::: +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Options diff --git a/docs/en/connectors/sink/HdfsFile.md b/docs/en/connectors/sink/HdfsFile.md index 2888722e05dc..208fb99f86eb 100644 --- a/docs/en/connectors/sink/HdfsFile.md +++ b/docs/en/connectors/sink/HdfsFile.md @@ -35,6 +35,7 @@ By default, we use 2PC commit to ensure `exactly-once` - [x] canal_json - [x] debezium_json - [x] maxwell_json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Http.md b/docs/en/connectors/sink/Http.md index 9d4ec378111c..b1c5e42ce1b0 100644 --- a/docs/en/connectors/sink/Http.md +++ b/docs/en/connectors/sink/Http.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-http.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/IoTDB.md b/docs/en/connectors/sink/IoTDB.md index 553260914882..b5f5efc8d529 100644 --- a/docs/en/connectors/sink/IoTDB.md +++ b/docs/en/connectors/sink/IoTDB.md @@ -19,6 +19,7 @@ Used to write data to IoTDB. - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) > IoTDB supports the `exactly-once` feature through idempotent writing. If multiple data have the same `key` and `timestamp`, the latest one will overwrite the previous one. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/IoTDBv2.md b/docs/en/connectors/sink/IoTDBv2.md index 3ac1708b17a1..7eab0a025c4f 100644 --- a/docs/en/connectors/sink/IoTDBv2.md +++ b/docs/en/connectors/sink/IoTDBv2.md @@ -19,6 +19,7 @@ Used to write data to IoTDB. - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) > IoTDB supports the `exactly-once` feature through idempotent writing. If multiple data have the same `key` and `timestamp`, the latest one will overwrite the previous one. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/Kafka.md b/docs/en/connectors/sink/Kafka.md index 77eeae39970c..3088cb685125 100644 --- a/docs/en/connectors/sink/Kafka.md +++ b/docs/en/connectors/sink/Kafka.md @@ -16,6 +16,7 @@ import ChangeLog from '../changelog/connector-kafka.md'; - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) > By default, we will use 2pc to guarantee the message is sent to kafka exactly once. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Kingbase.md b/docs/en/connectors/sink/Kingbase.md index db131d024ab1..7bf440d8adeb 100644 --- a/docs/en/connectors/sink/Kingbase.md +++ b/docs/en/connectors/sink/Kingbase.md @@ -18,6 +18,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Kudu.md b/docs/en/connectors/sink/Kudu.md index f0a8f44bf640..1946a8388328 100644 --- a/docs/en/connectors/sink/Kudu.md +++ b/docs/en/connectors/sink/Kudu.md @@ -19,6 +19,7 @@ import ChangeLog from '../changelog/connector-kudu.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Data Type Mapping diff --git a/docs/en/connectors/sink/LocalFile.md b/docs/en/connectors/sink/LocalFile.md index 661b0d439a23..4f615972f955 100644 --- a/docs/en/connectors/sink/LocalFile.md +++ b/docs/en/connectors/sink/LocalFile.md @@ -39,6 +39,7 @@ By default, we use 2PC commit to ensure `exactly-once` - [x] canal_json - [x] debezium_json - [x] maxwell_json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Options diff --git a/docs/en/connectors/sink/Milvus.md b/docs/en/connectors/sink/Milvus.md index 4868d3ba6f6a..434fec208428 100644 --- a/docs/en/connectors/sink/Milvus.md +++ b/docs/en/connectors/sink/Milvus.md @@ -16,6 +16,7 @@ This Milvus sink connector write data to Milvus or Zilliz Cloud, it has the foll - [x] [batch](../../introduction/concepts/connector-v2-features.md) - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [column projection](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Data Type Mapping diff --git a/docs/en/connectors/sink/Mysql.md b/docs/en/connectors/sink/Mysql.md index 393996efa74a..4207b489340e 100644 --- a/docs/en/connectors/sink/Mysql.md +++ b/docs/en/connectors/sink/Mysql.md @@ -37,6 +37,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/OceanBase.md b/docs/en/connectors/sink/OceanBase.md index 178480167ffd..4d622a091dcf 100644 --- a/docs/en/connectors/sink/OceanBase.md +++ b/docs/en/connectors/sink/OceanBase.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Oracle.md b/docs/en/connectors/sink/Oracle.md index 9617b0d44b7f..0fbdbd33f8e9 100644 --- a/docs/en/connectors/sink/Oracle.md +++ b/docs/en/connectors/sink/Oracle.md @@ -32,6 +32,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/PostgreSql.md b/docs/en/connectors/sink/PostgreSql.md index 6ec9fc16dbdf..c271ec7ba230 100644 --- a/docs/en/connectors/sink/PostgreSql.md +++ b/docs/en/connectors/sink/PostgreSql.md @@ -32,6 +32,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/Prometheus.md b/docs/en/connectors/sink/Prometheus.md index 43f9abe3d176..ecfc0bc3c24d 100644 --- a/docs/en/connectors/sink/Prometheus.md +++ b/docs/en/connectors/sink/Prometheus.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-prometheus.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Redshift.md b/docs/en/connectors/sink/Redshift.md index 5d1924c9d422..539a4dbd6d04 100644 --- a/docs/en/connectors/sink/Redshift.md +++ b/docs/en/connectors/sink/Redshift.md @@ -37,6 +37,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/RocketMQ.md b/docs/en/connectors/sink/RocketMQ.md index 6a6e09cb5fe9..f7dff4c70d7a 100644 --- a/docs/en/connectors/sink/RocketMQ.md +++ b/docs/en/connectors/sink/RocketMQ.md @@ -19,6 +19,7 @@ import ChangeLog from '../changelog/connector-rocketmq.md'; - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) By default, we will use 2pc to guarantee the message is sent to RocketMQ exactly once. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/S3File.md b/docs/en/connectors/sink/S3File.md index 4e85a346edab..a72a8fab59a3 100644 --- a/docs/en/connectors/sink/S3File.md +++ b/docs/en/connectors/sink/S3File.md @@ -35,6 +35,7 @@ import ChangeLog from '../changelog/connector-file-s3.md'; - [x] canal_json - [x] debezium_json - [x] maxwell_json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/SelectDB-Cloud.md b/docs/en/connectors/sink/SelectDB-Cloud.md index 64995637e765..b2369d1dbb8c 100644 --- a/docs/en/connectors/sink/SelectDB-Cloud.md +++ b/docs/en/connectors/sink/SelectDB-Cloud.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-selectdb-cloud.md'; - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Sls.md b/docs/en/connectors/sink/Sls.md index beee98b52b13..757aa446c7d1 100644 --- a/docs/en/connectors/sink/Sls.md +++ b/docs/en/connectors/sink/Sls.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-sls.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Snowflake.md b/docs/en/connectors/sink/Snowflake.md index ca0d55bdfe11..e68c083eb1e8 100644 --- a/docs/en/connectors/sink/Snowflake.md +++ b/docs/en/connectors/sink/Snowflake.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/SqlServer.md b/docs/en/connectors/sink/SqlServer.md index 2431f010c3ff..d967a343c80f 100644 --- a/docs/en/connectors/sink/SqlServer.md +++ b/docs/en/connectors/sink/SqlServer.md @@ -36,6 +36,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/en/connectors/sink/StarRocks.md b/docs/en/connectors/sink/StarRocks.md index 3c7584c4e8d5..90ad3197a1f5 100644 --- a/docs/en/connectors/sink/StarRocks.md +++ b/docs/en/connectors/sink/StarRocks.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-starrocks.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description diff --git a/docs/en/connectors/sink/Typesense.md b/docs/en/connectors/sink/Typesense.md index 18888d8df0e7..0b514c0c3be1 100644 --- a/docs/en/connectors/sink/Typesense.md +++ b/docs/en/connectors/sink/Typesense.md @@ -10,6 +10,7 @@ Outputs data to `Typesense`. - [ ] [Exactly Once](../../introduction/concepts/connector-v2-features.md) - [x] [CDC](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Options diff --git a/docs/en/connectors/sink/Vertica.md b/docs/en/connectors/sink/Vertica.md index cc50f370980d..9b01de06b808 100644 --- a/docs/en/connectors/sink/Vertica.md +++ b/docs/en/connectors/sink/Vertica.md @@ -32,6 +32,7 @@ semantics (using XA transaction guarantee). > Use `Xa transactions` to ensure `exactly-once`. So only support `exactly-once` for the database which is > support `Xa transactions`. You can set `is_exactly_once=true` to enable it. +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Supported DataSource Info diff --git a/docs/zh/connectors/sink/Aerospike.md b/docs/zh/connectors/sink/Aerospike.md index c8ee4ae2b622..9515ef6e537d 100644 --- a/docs/zh/connectors/sink/Aerospike.md +++ b/docs/zh/connectors/sink/Aerospike.md @@ -19,6 +19,7 @@ import ChangeLog from '../changelog/connector-aerospike.md'; - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) - [ ] [CDC](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Assert.md b/docs/zh/connectors/sink/Assert.md index d779c36357bd..fd2ce77ba3e6 100644 --- a/docs/zh/connectors/sink/Assert.md +++ b/docs/zh/connectors/sink/Assert.md @@ -11,6 +11,7 @@ Assert 数据接收器是一个用于断言数据是否符合用户定义规则 ## 核心特性 - [ ] [精准一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 配置 diff --git a/docs/zh/connectors/sink/BigQuery.md b/docs/zh/connectors/sink/BigQuery.md index 38806d9298b2..70d299821aca 100644 --- a/docs/zh/connectors/sink/BigQuery.md +++ b/docs/zh/connectors/sink/BigQuery.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-bigquery.md'; - [x] [精确一次](../../introduction/concepts/connector-v2-features.md) 仅适用于 batch 模式 - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Clickhouse.md b/docs/zh/connectors/sink/Clickhouse.md index 827721724f88..c4ff751f65b7 100644 --- a/docs/zh/connectors/sink/Clickhouse.md +++ b/docs/zh/connectors/sink/Clickhouse.md @@ -17,6 +17,7 @@ import ChangeLog from '../changelog/connector-clickhouse.md'; > Clickhouse sink 插件通过实现幂等写入可以达到精准一次，需要配合 aggregating merge tree 支持重复数据删除的引擎。 - [x] [支持多表写入](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/ClickhouseFile.md b/docs/zh/connectors/sink/ClickhouseFile.md index eaa566f7b7e6..348fc83b9774 100644 --- a/docs/zh/connectors/sink/ClickhouseFile.md +++ b/docs/zh/connectors/sink/ClickhouseFile.md @@ -17,6 +17,7 @@ import ChangeLog from '../changelog/connector-clickhouse.md'; 你也可以采用JDBC的方式将数据写入Clickhouse。 ::: +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 接收器选项 diff --git a/docs/zh/connectors/sink/Cloudberry.md b/docs/zh/connectors/sink/Cloudberry.md index a3a64284a541..a32d45e086ba 100644 --- a/docs/zh/connectors/sink/Cloudberry.md +++ b/docs/zh/connectors/sink/Cloudberry.md @@ -32,6 +32,7 @@ import ChangeLog from '../changelog/connector-cloudberry.md'; - [x] [cdc](../../introduction/concepts/connector-v2-features.md) > 使用 `XA 事务` 来确保 `精确一次`。因此，只有支持 `XA 事务` 的数据库才支持 `精确一次`。您可以设置 `is_exactly_once=true` 来启用它。 +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 支持的数据源信息 diff --git a/docs/zh/connectors/sink/Console.md b/docs/zh/connectors/sink/Console.md index f74bc50d71b9..4a9089913100 100644 --- a/docs/zh/connectors/sink/Console.md +++ b/docs/zh/connectors/sink/Console.md @@ -23,6 +23,7 @@ import ChangeLog from '../changelog/connector-console.md'; ## 主要特性 - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 接收器选项 diff --git a/docs/zh/connectors/sink/Databend.md b/docs/zh/connectors/sink/Databend.md index 0958a975cbd7..3d8670d7f189 100644 --- a/docs/zh/connectors/sink/Databend.md +++ b/docs/zh/connectors/sink/Databend.md @@ -16,6 +16,7 @@ import ChangeLog from '../changelog/connector-databend.md'; - [x] [精确一次](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [并行度](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/DingTalk.md b/docs/zh/connectors/sink/DingTalk.md index 1f6429f94276..39b27565b98e 100644 --- a/docs/zh/connectors/sink/DingTalk.md +++ b/docs/zh/connectors/sink/DingTalk.md @@ -13,6 +13,7 @@ import ChangeLog from '../changelog/connector-dingtalk.md'; ## 主要特性 - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Doris.md b/docs/zh/connectors/sink/Doris.md index 319bcbe613ba..589111552157 100644 --- a/docs/zh/connectors/sink/Doris.md +++ b/docs/zh/connectors/sink/Doris.md @@ -20,6 +20,7 @@ import ChangeLog from '../changelog/connector-doris.md'; - [x] [精确一次](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Elasticsearch.md b/docs/zh/connectors/sink/Elasticsearch.md index 482bcc7ee98f..7f21981559aa 100644 --- a/docs/zh/connectors/sink/Elasticsearch.md +++ b/docs/zh/connectors/sink/Elasticsearch.md @@ -18,6 +18,7 @@ import ChangeLog from '../changelog/connector-elasticsearch.md'; * 支持 `ElasticSearch 版本 >= 2.x 并且 <= 8.x` ::: +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/Email.md b/docs/zh/connectors/sink/Email.md index 69e625bf18de..3f2ca57e2b09 100644 --- a/docs/zh/connectors/sink/Email.md +++ b/docs/zh/connectors/sink/Email.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-email.md'; ## 主要特性 - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/Feishu.md b/docs/zh/connectors/sink/Feishu.md index 422338ddaa88..923e16ec4001 100644 --- a/docs/zh/connectors/sink/Feishu.md +++ b/docs/zh/connectors/sink/Feishu.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-http-feishu.md'; - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) - [ ] [变更数据捕获](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Fluss.md b/docs/zh/connectors/sink/Fluss.md index 1deefaae6e0c..73914f98d445 100644 --- a/docs/zh/connectors/sink/Fluss.md +++ b/docs/zh/connectors/sink/Fluss.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-fluss.md'; - [ ] [精准一次](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [支持多表写入](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/FtpFile.md b/docs/zh/connectors/sink/FtpFile.md index fa93f91d424b..d3a8660a70b3 100644 --- a/docs/zh/connectors/sink/FtpFile.md +++ b/docs/zh/connectors/sink/FtpFile.md @@ -35,6 +35,7 @@ import ChangeLog from '../changelog/connector-file-ftp.md'; - [x] excel - [x] xml - [x] binary +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/GoogleFirestore.md b/docs/zh/connectors/sink/GoogleFirestore.md index f6cf1506e1f9..00e6bede1c7b 100644 --- a/docs/zh/connectors/sink/GoogleFirestore.md +++ b/docs/zh/connectors/sink/GoogleFirestore.md @@ -11,6 +11,7 @@ import ChangeLog from '../changelog/connector-google-firestore.md'; ## 主要特性 - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/GraphQL.md b/docs/zh/connectors/sink/GraphQL.md index b88a7db874ac..8abf9854c227 100644 --- a/docs/zh/connectors/sink/GraphQL.md +++ b/docs/zh/connectors/sink/GraphQL.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-graphql.md'; - [ ] [[精确一次]](../../introduction/concepts/connector-v2-features.md) - [ ] [变更数据捕获](../../introduction/concepts/connector-v2-features.md) - [x] [支持多表写入](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Hbase.md b/docs/zh/connectors/sink/Hbase.md index f42916c5e0e8..15a9e9b25e0e 100644 --- a/docs/zh/connectors/sink/Hbase.md +++ b/docs/zh/connectors/sink/Hbase.md @@ -11,6 +11,7 @@ import ChangeLog from '../changelog/connector-hbase.md'; ## 主要特性 - [ ] [精准一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/HdfsFile.md b/docs/zh/connectors/sink/HdfsFile.md index bf2756d4a672..b187e0c3a876 100644 --- a/docs/zh/connectors/sink/HdfsFile.md +++ b/docs/zh/connectors/sink/HdfsFile.md @@ -33,6 +33,7 @@ import ChangeLog from '../changelog/connector-file-hadoop.md'; - [x] maxwell_json - [x] 压缩编解码器 - [x] lzo +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Http.md b/docs/zh/connectors/sink/Http.md index 8ccfe0f1a881..f2e5dda5ef75 100644 --- a/docs/zh/connectors/sink/Http.md +++ b/docs/zh/connectors/sink/Http.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-http.md'; - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Iceberg.md b/docs/zh/connectors/sink/Iceberg.md index 5df4c57680d6..0f75ea64df20 100644 --- a/docs/zh/connectors/sink/Iceberg.md +++ b/docs/zh/connectors/sink/Iceberg.md @@ -21,6 +21,7 @@ Apache Iceberg 目标连接器支持cdc模式、自动建表及表结构变更. ## 主要特性 - [x] [支持多表写入](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 支持的数据源信息 diff --git a/docs/zh/connectors/sink/IoTDB.md b/docs/zh/connectors/sink/IoTDB.md index a133a36bd5aa..db7988f9bdd1 100644 --- a/docs/zh/connectors/sink/IoTDB.md +++ b/docs/zh/connectors/sink/IoTDB.md @@ -19,6 +19,7 @@ import ChangeLog from '../changelog/connector-iotdb.md'; - [x] [精确一次](../../introduction/concepts/connector-v2-features.md) > IoTDB 通过幂等写支持`精确一次`功能。如果两条数据使用相同的`key`和`timestamp`，新数据将覆盖旧数据。 +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 支持的数据源信息 diff --git a/docs/zh/connectors/sink/IoTDBv2.md b/docs/zh/connectors/sink/IoTDBv2.md index dfca39a0fbce..4f28ad11397c 100644 --- a/docs/zh/connectors/sink/IoTDBv2.md +++ b/docs/zh/connectors/sink/IoTDBv2.md @@ -19,6 +19,7 @@ import ChangeLog from '../changelog/connector-iotdb.md'; - [x] [精确一次](../../introduction/concepts/connector-v2-features.md) > IoTDB 通过幂等写支持`精确一次`功能。如果两条数据使用相同的`key`和`timestamp`，新数据将覆盖旧数据。 +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 支持的数据源信息 diff --git a/docs/zh/connectors/sink/Kafka.md b/docs/zh/connectors/sink/Kafka.md index ddad0db0602b..50e3c6ceb93a 100644 --- a/docs/zh/connectors/sink/Kafka.md +++ b/docs/zh/connectors/sink/Kafka.md @@ -16,6 +16,7 @@ import ChangeLog from '../changelog/connector-kafka.md'; - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) > 默认情况下，我们将使用 2pc 来保证消息只发送一次到kafka +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Kudu.md b/docs/zh/connectors/sink/Kudu.md index 025ece1c4741..64e51f838abc 100644 --- a/docs/zh/connectors/sink/Kudu.md +++ b/docs/zh/connectors/sink/Kudu.md @@ -19,6 +19,7 @@ import ChangeLog from '../changelog/connector-kudu.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 数据类型映射 diff --git a/docs/zh/connectors/sink/Lance.md b/docs/zh/connectors/sink/Lance.md index 80c0439b290c..abb8caac4008 100644 --- a/docs/zh/connectors/sink/Lance.md +++ b/docs/zh/connectors/sink/Lance.md @@ -17,6 +17,7 @@ Lance 格式的 Sink 连接器。支持创建和写入数据集、Lance 命名 ## 主要特性 - [] [精确一次语义](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 依赖 diff --git a/docs/zh/connectors/sink/LocalFile.md b/docs/zh/connectors/sink/LocalFile.md index 26cf2450936f..23cb2045cce6 100644 --- a/docs/zh/connectors/sink/LocalFile.md +++ b/docs/zh/connectors/sink/LocalFile.md @@ -38,6 +38,7 @@ import ChangeLog from '../changelog/connector-file-local.md'; - [x] canal_json - [x] debezium_json - [x] maxwell_json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/Milvus.md b/docs/zh/connectors/sink/Milvus.md index c46a8e10be94..33bb0fd75e1b 100644 --- a/docs/zh/connectors/sink/Milvus.md +++ b/docs/zh/connectors/sink/Milvus.md @@ -36,6 +36,7 @@ Milvus sink连接器将数据写入Milvus或Zilliz Cloud，它具有以下功能 | FLOAT16_VECTOR | FLOAT16_VECTOR | | BFLOAT16_VECTOR | BFLOAT16_VECTOR | | SPARSE_FLOAT_VECTOR | SPARSE_FLOAT_VECTOR | +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Sink 选项 diff --git a/docs/zh/connectors/sink/ObsFile.md b/docs/zh/connectors/sink/ObsFile.md index df1ff7afd7fc..35ce836c4f60 100644 --- a/docs/zh/connectors/sink/ObsFile.md +++ b/docs/zh/connectors/sink/ObsFile.md @@ -32,6 +32,7 @@ import ChangeLog from '../changelog/connector-file-obs.md'; - [x] canal_json - [x] debezium_json - [x] maxwell_json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Oracle.md b/docs/zh/connectors/sink/Oracle.md index dd1cda550b9a..20b054713c2f 100644 --- a/docs/zh/connectors/sink/Oracle.md +++ b/docs/zh/connectors/sink/Oracle.md @@ -32,6 +32,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; >使用“Xa事务”来确保“精确一次”。因此，数据库只支持“精确一次”，即 >支持“Xa事务”。您可以设置`is_exactly_once=true `来启用它。 +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 支持的数据源信息 diff --git a/docs/zh/connectors/sink/Paimon.md b/docs/zh/connectors/sink/Paimon.md index 64623ab9dfbd..491c61d80df4 100644 --- a/docs/zh/connectors/sink/Paimon.md +++ b/docs/zh/connectors/sink/Paimon.md @@ -56,6 +56,7 @@ libfb303-xxx.jar ## 主要特性 - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 连接器选项 diff --git a/docs/zh/connectors/sink/Phoenix.md b/docs/zh/connectors/sink/Phoenix.md index 12255871b329..37fe5feb47dd 100644 --- a/docs/zh/connectors/sink/Phoenix.md +++ b/docs/zh/connectors/sink/Phoenix.md @@ -17,6 +17,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; ## 主要特性 - [ ] [精准一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 接收器选项 diff --git a/docs/zh/connectors/sink/PostgreSql.md b/docs/zh/connectors/sink/PostgreSql.md index 947437b664bb..6d81efbb9332 100644 --- a/docs/zh/connectors/sink/PostgreSql.md +++ b/docs/zh/connectors/sink/PostgreSql.md @@ -30,6 +30,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; - [x] [变更数据捕获（CDC）](../../introduction/concepts/connector-v2-features.md) > 使用 `XA 事务` 来确保 `精确一次`。因此，仅对支持 `XA 事务` 的数据库支持 `精确一次`。您可以设置 `is_exactly_once=true` 来启用此功能。 +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 支持的数据源信息 | 数据源 | 支持的版本 | 驱动 | URL | Maven | diff --git a/docs/zh/connectors/sink/Prometheus.md b/docs/zh/connectors/sink/Prometheus.md index 1eb6a1621ffb..0f90018970e7 100644 --- a/docs/zh/connectors/sink/Prometheus.md +++ b/docs/zh/connectors/sink/Prometheus.md @@ -15,6 +15,7 @@ import ChangeLog from '../changelog/connector-prometheus.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Pulsar.md b/docs/zh/connectors/sink/Pulsar.md index 955effa165b8..f58ba962ad39 100644 --- a/docs/zh/connectors/sink/Pulsar.md +++ b/docs/zh/connectors/sink/Pulsar.md @@ -13,6 +13,7 @@ import ChangeLog from '../changelog/connector-pulsar.md'; ## 核心特性 - [x] [精准一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Rabbitmq.md b/docs/zh/connectors/sink/Rabbitmq.md index 6bd256ee0df0..3a40dc4c1f95 100644 --- a/docs/zh/connectors/sink/Rabbitmq.md +++ b/docs/zh/connectors/sink/Rabbitmq.md @@ -11,6 +11,7 @@ import ChangeLog from '../changelog/connector-rabbitmq.md'; ## 主要特性 - [ ] [精准一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 接收器选项 diff --git a/docs/zh/connectors/sink/RocketMQ.md b/docs/zh/connectors/sink/RocketMQ.md index 5bd55404b16e..df334e7c9c16 100644 --- a/docs/zh/connectors/sink/RocketMQ.md +++ b/docs/zh/connectors/sink/RocketMQ.md @@ -19,6 +19,7 @@ import ChangeLog from '../changelog/connector-rocketmq.md'; - [x] [精确一次](../../introduction/concepts/connector-v2-features.md) 默认情况下，我们将使用2pc来保证消息精确一次到RocketMQ。 +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/S3-Redshift.md b/docs/zh/connectors/sink/S3-Redshift.md index 9f6719b8b399..0b80c5f8f2b6 100644 --- a/docs/zh/connectors/sink/S3-Redshift.md +++ b/docs/zh/connectors/sink/S3-Redshift.md @@ -26,6 +26,7 @@ import ChangeLog from '../changelog/connector-s3-redshift.md'; - [x] parquet - [x] orc - [x] json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 参数 diff --git a/docs/zh/connectors/sink/S3File.md b/docs/zh/connectors/sink/S3File.md index c73c78e2ddba..17808ae98082 100644 --- a/docs/zh/connectors/sink/S3File.md +++ b/docs/zh/connectors/sink/S3File.md @@ -34,6 +34,7 @@ import ChangeLog from '../changelog/connector-file-s3.md'; - [x] canal_json - [x] debezium_json - [x] maxwell_json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/SelectDB-Cloud.md b/docs/zh/connectors/sink/SelectDB-Cloud.md index 92cdadbded58..dab691982d91 100644 --- a/docs/zh/connectors/sink/SelectDB-Cloud.md +++ b/docs/zh/connectors/sink/SelectDB-Cloud.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-selectdb-cloud.md'; - [x] [精确一次](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/SftpFile.md b/docs/zh/connectors/sink/SftpFile.md index 884e5356fee7..6bdea36121f6 100644 --- a/docs/zh/connectors/sink/SftpFile.md +++ b/docs/zh/connectors/sink/SftpFile.md @@ -37,6 +37,7 @@ import ChangeLog from '../changelog/connector-file-sftp.md'; - [x] canal_json - [x] debezium_json - [x] maxwell_json +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 参数 diff --git a/docs/zh/connectors/sink/Sls.md b/docs/zh/connectors/sink/Sls.md index b2ff99950dcd..c6e7d4d2eea4 100644 --- a/docs/zh/connectors/sink/Sls.md +++ b/docs/zh/connectors/sink/Sls.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-sls.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Snowflake.md b/docs/zh/connectors/sink/Snowflake.md index 05fd64134c74..110daff6005a 100644 --- a/docs/zh/connectors/sink/Snowflake.md +++ b/docs/zh/connectors/sink/Snowflake.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) - [x] [（CDC）](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/Socket.md b/docs/zh/connectors/sink/Socket.md index 51494d6e71d8..5f992e3b377f 100644 --- a/docs/zh/connectors/sink/Socket.md +++ b/docs/zh/connectors/sink/Socket.md @@ -13,6 +13,7 @@ import ChangeLog from '../changelog/connector-socket.md'; ## 主要特性 - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/SqlServer.md b/docs/zh/connectors/sink/SqlServer.md index 6e65d6002fc1..45b119b05c30 100644 --- a/docs/zh/connectors/sink/SqlServer.md +++ b/docs/zh/connectors/sink/SqlServer.md @@ -34,6 +34,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; - [x] [cdc](../../introduction/concepts/connector-v2-features.md) > 使用 `Xa 事务` 来保证 `精确一次`。因此仅支持支持 `Xa 事务` 的数据库。可以通过设置 `is_exactly_once=true` 来启用。 +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 支持的数据源信息 diff --git a/docs/zh/connectors/sink/StarRocks.md b/docs/zh/connectors/sink/StarRocks.md index 575ed4b846d1..8fc68e0e2920 100644 --- a/docs/zh/connectors/sink/StarRocks.md +++ b/docs/zh/connectors/sink/StarRocks.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-starrocks.md'; - [ ] [精准一次](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 描述 diff --git a/docs/zh/connectors/sink/TDengine.md b/docs/zh/connectors/sink/TDengine.md index 4082f9c4867b..5d78d7c4aaec 100644 --- a/docs/zh/connectors/sink/TDengine.md +++ b/docs/zh/connectors/sink/TDengine.md @@ -12,6 +12,7 @@ import ChangeLog from '../changelog/connector-tdengine.md'; - [x] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/Tablestore.md b/docs/zh/connectors/sink/Tablestore.md index 1e83b6a62b2b..4fb0aa41f70f 100644 --- a/docs/zh/connectors/sink/Tablestore.md +++ b/docs/zh/connectors/sink/Tablestore.md @@ -11,6 +11,7 @@ import ChangeLog from '../changelog/connector-tablestore.md'; ## 主要特性 - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/Typesense.md b/docs/zh/connectors/sink/Typesense.md index 5056c91bc941..d6384dd917aa 100644 --- a/docs/zh/connectors/sink/Typesense.md +++ b/docs/zh/connectors/sink/Typesense.md @@ -10,6 +10,7 @@ import ChangeLog from '../changelog/connector-typesense.md'; - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) - [x] [cdc](../../introduction/concepts/connector-v2-features.md) +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 选项 diff --git a/docs/zh/connectors/sink/Vertica.md b/docs/zh/connectors/sink/Vertica.md index 91ea229bf58d..c34c9fedb5a6 100644 --- a/docs/zh/connectors/sink/Vertica.md +++ b/docs/zh/connectors/sink/Vertica.md @@ -30,6 +30,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) > 使用 `Xa 事务` 来保证 `精确一次`。因此仅支持支持 `Xa 事务` 的数据库。可以通过设置 `is_exactly_once=true` 来启用。 +- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) ## 支持的数据源信息 From 71ab292dafe821d08306434456551a8aebce5999 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 28 Jun 2026 22:32:14 +0800 Subject: [PATCH 073/375] [Feature][Connector-V2] Add Vitess CDC source connector (#11059) --- config/plugin_config | 3 +- .../changelog/connector-cdc-vitess.md | 7 + docs/en/connectors/source/Vitess-CDC.md | 148 ++++++ .../changelog/connector-cdc-vitess.md | 7 + docs/zh/connectors/source/Vitess-CDC.md | 145 ++++++ plugin-mapping.properties | 1 + .../cdc/base/utils/SourceRecordUtils.java | 20 +- .../cdc/base/utils/SourceRecordUtilsTest.java | 45 ++ .../connector-cdc-vitess/pom.xml | 101 ++++ .../cdc/vitess/config/VitessSourceConfig.java | 259 +++++++++++ .../vitess/config/VitessSourceOptions.java | 144 ++++++ .../cdc/vitess/source/VitessSource.java | 84 ++++ .../vitess/source/VitessSourceFactory.java | 252 ++++++++++ .../enumerator/VitessSourceEnumerator.java | 119 +++++ .../VitessSourceEnumeratorState.java | 54 +++ .../source/reader/VitessSourceReader.java | 379 +++++++++++++++ .../source/split/VitessSourceSplit.java | 159 +++++++ .../source/split/VitessTableSchemaState.java | 250 ++++++++++ .../source/VitessSourceFactoryTest.java | 387 ++++++++++++++++ .../reader/TestVitessSourceReaderIT.java | 431 ++++++++++++++++++ .../vitess/source/reader/VitessContainer.java | 90 ++++ seatunnel-connectors-v2/connector-cdc/pom.xml | 1 + seatunnel-dist/pom.xml | 6 + 23 files changed, 3088 insertions(+), 4 deletions(-) create mode 100644 docs/en/connectors/changelog/connector-cdc-vitess.md create mode 100644 docs/en/connectors/source/Vitess-CDC.md create mode 100644 docs/zh/connectors/changelog/connector-cdc-vitess.md create mode 100644 docs/zh/connectors/source/Vitess-CDC.md create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/pom.xml create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/config/VitessSourceConfig.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/config/VitessSourceOptions.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSource.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSourceFactory.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/enumerator/VitessSourceEnumerator.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/enumerator/VitessSourceEnumeratorState.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/VitessSourceReader.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessSourceSplit.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessTableSchemaState.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSourceFactoryTest.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/TestVitessSourceReaderIT.java create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/VitessContainer.java diff --git a/config/plugin_config b/config/plugin_config index 5f560a4a4615..3e7846976d1a 100644 --- a/config/plugin_config +++ b/config/plugin_config @@ -30,6 +30,7 @@ connector-cdc-sqlserver connector-cdc-postgres connector-cdc-oracle connector-cdc-tidb +connector-cdc-vitess connector-clickhouse connector-datahub connector-databend @@ -102,4 +103,4 @@ connector-sensorsdata connector-hugegraph connector-lance connector-mqtt -connector-bigquery \ No newline at end of file +connector-bigquery diff --git a/docs/en/connectors/changelog/connector-cdc-vitess.md b/docs/en/connectors/changelog/connector-cdc-vitess.md new file mode 100644 index 000000000000..2c65ae671330 --- /dev/null +++ b/docs/en/connectors/changelog/connector-cdc-vitess.md @@ -0,0 +1,7 @@ +

Change Log + +| Change | Commit | Version | +| --- | --- | --- | +|[Feature][Connector-V2][CDC] Add Vitess CDC source connector|-|Next| + +
diff --git a/docs/en/connectors/source/Vitess-CDC.md b/docs/en/connectors/source/Vitess-CDC.md new file mode 100644 index 000000000000..de9573c3f21e --- /dev/null +++ b/docs/en/connectors/source/Vitess-CDC.md @@ -0,0 +1,148 @@ +import ChangeLog from '../changelog/connector-cdc-vitess.md'; + +# Vitess CDC + +> Vitess CDC source connector + +## Support Those Engines + +> SeaTunnel Zeta
+> Flink
+ +## Description + +The Vitess CDC connector captures change events from Vitess VTGate through the VStream gRPC API. +The first delivery keeps the connector intentionally narrow: + +- streaming only, no initial snapshot phase +- explicit schema metadata only, provided through `schema` or `tables_configs` +- optional `table-names` / `table-pattern` filters over those declared schemas +- checkpoint / restore based on serialized Vitess VGTID state +- rows emitted as SeaTunnel CDC rows for existing multi-table downstream paths + +If you need a reproducible bootstrap position, use `startup.mode = SPECIFIC` with a concrete +Vitess VGTID. `LATEST` is provided as a convenience startup mode aligned with existing Vitess CDC +backends, but its initial position is symbolic until the first CDC event materializes a concrete +offset. + +## Key features + +- [ ] [batch](../../introduction/concepts/connector-v2-features.md) +- [x] [stream](../../introduction/concepts/connector-v2-features.md) +- [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) +- [ ] [column projection](../../introduction/concepts/connector-v2-features.md) +- [ ] [parallelism](../../introduction/concepts/connector-v2-features.md) +- [ ] [support user-defined split](../../introduction/concepts/connector-v2-features.md) + +## Supported DataSource Info + +| Datasource | Supported versions | Driver | Url | Maven | +| --- | --- | --- | --- | --- | +| Vitess VTGate VStream | VTGate deployments compatible with Debezium Vitess 1.9.8.Final | gRPC client built into the connector | `hostname` + `port` | https://mvnrepository.com/artifact/io.debezium/debezium-connector-vitess/1.9.8.Final | + +## Using Dependency + +No JDBC driver is required for the connector runtime itself because CDC traffic is read through +VTGate gRPC. If you use JDBC for verification or downstream examples, add the MySQL JDBC driver +separately. + +## Source Options + +| Name | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| hostname | String | Yes | - | Hostname or IP address of the Vitess VTGate gRPC server. | +| port | Int | No | 15991 | Port of the Vitess VTGate gRPC server. | +| keyspace | String | Yes | - | Vitess keyspace captured by the connector. | +| schema | Config | Yes* | - | Single-table schema definition. The schema block must provide `table` plus either `columns` or `metadata_table_id`. | +| tables_configs | List\ | Yes* | - | Multi-table schema definitions. Each entry must contain a `schema` block with `table` plus either `columns` or `metadata_table_id`. | +| table-names | List | No** | - | Optional database-qualified tables to capture from the declared schema set, for example `commerce.orders`. | +| table-pattern | String | No** | - | Optional regular expression used to filter the declared schema set. | +| metalake_type | Enum | No | GRAVITINO | Metadata lake implementation used when a schema block resolves columns through `metadata_table_id`. | +| startup.mode | Enum | No | LATEST | Supported values are `latest` and `specific`. `specific` is the stable startup mode for reproducible restore. | +| startup.specific-offset.vgtid | String | No | - | Vitess VGTID used when `startup.mode = specific`. | +| tablet-type | Enum | No | MASTER | VTGate tablet type used by VStream. Supported values are `MASTER`, `REPLICA`, `RDONLY`. | +| shard | String | No | - | Optional shard restriction. Omit it to capture all shards in the keyspace. | +| stop-on-reshard | Boolean | No | false | Whether the connector should stop after resharding. | +| keepalive.interval.ms | Long | No | Long.MAX_VALUE | gRPC keepalive interval in milliseconds. | +| grpc.headers | String | No | - | Optional comma-separated gRPC headers in `key:value` format. | +| grpc.max-inbound-message-size | Int | No | 4194304 | Maximum inbound gRPC message size in bytes. | +| server-time-zone | String | No | UTC | Time zone used by SeaTunnel row deserialization. | +| format | Enum | No | DEFAULT | Optional output format. Supported values are `DEFAULT` and `COMPATIBLE_DEBEZIUM_JSON`. | +| debezium | Config | No | - | Pass-through Debezium properties for the Vitess connector backend. | + +\* Configure exactly one of `schema` and `tables_configs`. + +\** Configure at most one of `table-names` and `table-pattern`. When both are omitted, the connector captures every declared schema table. + +## Notes + +- The first delivery does not read an initial table snapshot. +- Dynamic discovery of newly added tables is out of scope. +- Schema evolution events are not emitted in this first delivery. +- Restore uses the checkpointed SeaTunnel table schema snapshot instead of re-parsing the latest config shape. +- A focused integration path is provided by `TestVitessSourceReaderIT`, which runs against + `vitess/vttestserver`. + +## Task Example + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + Vitess-CDC { + plugin_output = "vitess_cdc" + hostname = "127.0.0.1" + port = 15992 + keyspace = "test" + tables_configs = [ + { + schema = { + table = "test.products" + columns = [ + { name = "id", type = "int" } + { name = "name", type = "string" } + { name = "description", type = "string" } + { name = "weight", type = "float" } + ] + primaryKey = { + name = "pk_products" + columnNames = ["id"] + } + } + }, + { + schema = { + table = "test.customers" + columns = [ + { name = "id", type = "int" } + { name = "name", type = "string" } + ] + primaryKey = { + name = "pk_customers" + columnNames = ["id"] + } + } + } + ] + table-names = ["test.products", "test.customers"] + startup.mode = "specific" + startup.specific-offset.vgtid = "[{\"keyspace\":\"test\",\"shard\":\"-\",\"gtid\":\"MySQL56/uuid:1-200\"}]" + server-time-zone = "UTC" + } +} + +transform { +} + +sink { + Console {} +} +``` + +## Changelog + + diff --git a/docs/zh/connectors/changelog/connector-cdc-vitess.md b/docs/zh/connectors/changelog/connector-cdc-vitess.md new file mode 100644 index 000000000000..312cdfa756ea --- /dev/null +++ b/docs/zh/connectors/changelog/connector-cdc-vitess.md @@ -0,0 +1,7 @@ +
Change Log + +| 变更 | Commit | 版本 | +| --- | --- | --- | +|[Feature][Connector-V2][CDC] 新增 Vitess CDC Source 连接器|-|Next| + +
diff --git a/docs/zh/connectors/source/Vitess-CDC.md b/docs/zh/connectors/source/Vitess-CDC.md new file mode 100644 index 000000000000..67a3aa065f30 --- /dev/null +++ b/docs/zh/connectors/source/Vitess-CDC.md @@ -0,0 +1,145 @@ +import ChangeLog from '../changelog/connector-cdc-vitess.md'; + +# Vitess CDC + +> Vitess CDC Source 连接器 + +## 支持的引擎 + +> SeaTunnel Zeta
+> Flink
+ +## 描述 + +Vitess CDC 连接器通过 VTGate 的 VStream gRPC API 订阅变更事件。第一版交付范围刻意收窄，只覆盖一条可复现、可恢复的 CDC 路径： + +- 仅支持流式 CDC，不包含初始快照阶段 +- 必须通过 `schema` 或 `tables_configs` 显式提供表结构 +- `table-names` / `table-pattern` 只作为这些已声明表结构的可选过滤条件 +- 基于序列化后的 Vitess VGTID 接入 SeaTunnel checkpoint / restore +- 输出 SeaTunnel CDC 行，兼容现有多表下游链路 + +如果你需要可复现的启动位点，请使用 `startup.mode = SPECIFIC` 并提供明确的 +Vitess VGTID。`LATEST` 作为便捷启动模式保留，但在第一条 CDC 事件落地成具体 offset +之前，它的初始位置仍然是符号化的 `current`。 + +## 主要功能 + +- [ ] [批处理](../../introduction/concepts/connector-v2-features.md) +- [x] [流处理](../../introduction/concepts/connector-v2-features.md) +- [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) +- [ ] [column projection](../../introduction/concepts/connector-v2-features.md) +- [ ] [并行度](../../introduction/concepts/connector-v2-features.md) +- [ ] [支持用户定义的拆分](../../introduction/concepts/connector-v2-features.md) + +## 支持的数据源信息 + +| 数据源 | 支持版本 | 驱动 | 地址形式 | Maven | +| --- | --- | --- | --- | --- | +| Vitess VTGate VStream | 与 Debezium Vitess 1.9.8.Final 兼容的 VTGate 部署 | 连接器内置 gRPC 客户端 | `hostname` + `port` | https://mvnrepository.com/artifact/io.debezium/debezium-connector-vitess/1.9.8.Final | + +## 依赖说明 + +连接器运行时本身不依赖 JDBC 驱动，因为 CDC 数据通过 VTGate gRPC 获取。如果你需要用 +JDBC 做验证或示例下游，请额外准备 MySQL JDBC 驱动。 + +## 源选项 + +| 名称 | 类型 | 必填 | 默认值 | 描述 | +| --- | --- | --- | --- | --- | +| hostname | String | 是 | - | Vitess VTGate gRPC 服务地址。 | +| port | Int | 否 | 15991 | Vitess VTGate gRPC 端口。 | +| keyspace | String | 是 | - | 当前连接器采集的 Vitess keyspace。 | +| schema | Config | 是* | - | 单表 schema 定义。schema 中必须提供 `table`，并且至少提供 `columns` 或 `metadata_table_id` 之一。 | +| tables_configs | List\ | 是* | - | 多表 schema 定义列表。每个元素都必须包含一个 `schema` 块，且其中必须提供 `table`，并至少提供 `columns` 或 `metadata_table_id` 之一。 | +| table-names | List | 否** | - | 从已声明 schema 集合里筛选要采集的表，表名必须带数据库前缀，例如 `commerce.orders`。 | +| table-pattern | String | 否** | - | 用于筛选已声明 schema 集合的表名正则，表名必须带数据库前缀。 | +| metalake_type | Enum | 否 | GRAVITINO | 当 schema 通过 `metadata_table_id` 从元数据中心解析列定义时使用的 metadata lake 实现。 | +| startup.mode | Enum | 否 | LATEST | 仅支持 `latest` 和 `specific`。其中 `specific` 是可复现恢复的稳定启动模式。 | +| startup.specific-offset.vgtid | String | 否 | - | 当 `startup.mode = specific` 时使用的 Vitess VGTID。 | +| tablet-type | Enum | 否 | MASTER | VStream 使用的 tablet 类型，支持 `MASTER`、`REPLICA`、`RDONLY`。 | +| shard | String | 否 | - | 可选 shard 限定。不配置时会采集 keyspace 的全部 shard。 | +| stop-on-reshard | Boolean | 否 | false | reshard 后是否停止当前采集。 | +| keepalive.interval.ms | Long | 否 | Long.MAX_VALUE | gRPC keepalive 间隔，单位毫秒。 | +| grpc.headers | String | 否 | - | 可选 gRPC headers，格式为 `key:value,key2:value2`。 | +| grpc.max-inbound-message-size | Int | 否 | 4194304 | 允许接收的最大 gRPC 消息大小，单位字节。 | +| server-time-zone | String | 否 | UTC | SeaTunnel 行反序列化使用的时区。 | +| format | Enum | 否 | DEFAULT | 输出格式，支持 `DEFAULT` 和 `COMPATIBLE_DEBEZIUM_JSON`。 | +| debezium | Config | 否 | - | 透传给 Debezium Vitess 后端的附加参数。 | + +\* `schema` 和 `tables_configs` 二选一，必须提供其中之一。 + +\** `table-names` 和 `table-pattern` 至多配置一个；如果都不配，则采集所有已声明 schema 的表。 + +## 说明 + +- 第一版不支持初始快照读取。 +- 第一版不支持运行时动态发现新表。 +- 第一版不发送 schema evolution 事件。 +- restore 时会优先使用 checkpoint 中保存的 SeaTunnel 表结构快照，而不是重新按最新配置重建字段列表。 +- 仓库内提供了 `TestVitessSourceReaderIT` 作为真实可跑的集成验证路径，底层使用 + `vitess/vttestserver`。 + +## 任务示例 + +```hocon +env { + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 +} + +source { + Vitess-CDC { + plugin_output = "vitess_cdc" + hostname = "127.0.0.1" + port = 15992 + keyspace = "test" + tables_configs = [ + { + schema = { + table = "test.products" + columns = [ + { name = "id", type = "int" } + { name = "name", type = "string" } + { name = "description", type = "string" } + { name = "weight", type = "float" } + ] + primaryKey = { + name = "pk_products" + columnNames = ["id"] + } + } + }, + { + schema = { + table = "test.customers" + columns = [ + { name = "id", type = "int" } + { name = "name", type = "string" } + ] + primaryKey = { + name = "pk_customers" + columnNames = ["id"] + } + } + } + ] + table-names = ["test.products", "test.customers"] + startup.mode = "specific" + startup.specific-offset.vgtid = "[{\"keyspace\":\"test\",\"shard\":\"-\",\"gtid\":\"MySQL56/uuid:1-200\"}]" + server-time-zone = "UTC" + } +} + +transform { +} + +sink { + Console {} +} +``` + +## 变更日志 + + diff --git a/plugin-mapping.properties b/plugin-mapping.properties index 5e0a4460ceb8..838c9fa4d4f3 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -115,6 +115,7 @@ seatunnel.sink.Maxcompute = connector-maxcompute seatunnel.source.MySQL-CDC = connector-cdc-mysql seatunnel.source.MongoDB-CDC = connector-cdc-mongodb seatunnel.source.TiDB-CDC = connector-cdc-tidb +seatunnel.source.Vitess-CDC = connector-cdc-vitess seatunnel.sink.S3Redshift = connector-s3-redshift seatunnel.source.Web3j = connector-web3j seatunnel.source.TDengine = connector-tdengine diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/utils/SourceRecordUtils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/utils/SourceRecordUtils.java index 7237d99dbc19..3d9f2a25f0dd 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/utils/SourceRecordUtils.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/utils/SourceRecordUtils.java @@ -38,7 +38,6 @@ import java.util.Arrays; import java.util.List; -import static io.debezium.connector.AbstractSourceInfo.DATABASE_NAME_KEY; import static io.debezium.connector.AbstractSourceInfo.SCHEMA_NAME_KEY; import static io.debezium.connector.AbstractSourceInfo.TABLE_NAME_KEY; @@ -123,7 +122,7 @@ public static boolean isHeartbeatRecord(SourceRecord record) { public static TableId getTableId(SourceRecord dataRecord) { Struct value = (Struct) dataRecord.value(); Struct source = value.getStruct(Envelope.FieldName.SOURCE); - String dbName = source.getString(DATABASE_NAME_KEY); + String dbName = resolveDatabaseName(source); // Oracle need schemaName String schemaName = getSchemaName(source); String tableName = source.getString(TABLE_NAME_KEY); @@ -214,7 +213,7 @@ private static BigDecimal toBigDecimal(Object numericObj) { public static TablePath getTablePath(SourceRecord record) { Struct messageStruct = (Struct) record.value(); Struct sourceStruct = messageStruct.getStruct(Envelope.FieldName.SOURCE); - String databaseName = sourceStruct.getString(AbstractSourceInfo.DATABASE_NAME_KEY); + String databaseName = resolveDatabaseName(sourceStruct); String tableName = sourceStruct.getString(AbstractSourceInfo.TABLE_NAME_KEY); String schemaName = null; if (sourceStruct.schema().field(AbstractSourceInfo.SCHEMA_NAME_KEY) != null) { @@ -223,6 +222,21 @@ public static TablePath getTablePath(SourceRecord record) { return TablePath.of(databaseName, schemaName, tableName); } + /** + * Resolves the logical database name from Debezium source metadata. + * + *

Vitess writes an empty string into the generic database field and stores the real logical + * database in {@code keyspace}, so blank values must also trigger the fallback. + */ + private static String resolveDatabaseName(Struct sourceStruct) { + String databaseName = sourceStruct.getString(AbstractSourceInfo.DATABASE_NAME_KEY); + if ((databaseName == null || databaseName.isEmpty()) + && sourceStruct.schema().field("keyspace") != null) { + return sourceStruct.getString("keyspace"); + } + return databaseName; + } + public static String getDdl(SourceRecord record) { Struct schemaChangeStruct = (Struct) record.value(); return schemaChangeStruct.getString(HistoryRecord.Fields.DDL_STATEMENTS); diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/utils/SourceRecordUtilsTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/utils/SourceRecordUtilsTest.java index 3b1fceb900f5..d481f4998164 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/utils/SourceRecordUtilsTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/utils/SourceRecordUtilsTest.java @@ -17,6 +17,8 @@ package org.apache.seatunnel.connectors.cdc.base.utils; +import org.apache.seatunnel.api.table.catalog.TablePath; + import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; @@ -25,7 +27,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import io.debezium.connector.AbstractSourceInfo; import io.debezium.data.Envelope; +import io.debezium.relational.TableId; import java.util.Collections; @@ -293,4 +297,45 @@ public void testGetGtid_nullForNonMysqlConnector() { SourceRecord record = buildRecord(valueSchema, value); Assertions.assertNull(SourceRecordUtils.getGtid(record)); } + + /** + * Vitess exposes the logical database as keyspace, so table identity must still resolve even + * when Debezium leaves database_name empty or blank. + */ + @Test + public void testGetTablePathUsesKeyspaceWhenDatabaseNameIsMissing() { + assertVitessTableIdentity(null); + assertVitessTableIdentity(""); + } + + /** + * Builds a minimal Vitess-like source record and verifies both SeaTunnel and Debezium table + * identifiers fall back to keyspace when the generic database field is absent. + */ + private static void assertVitessTableIdentity(String databaseName) { + Schema sourceSchema = + SchemaBuilder.struct() + .field( + AbstractSourceInfo.DATABASE_NAME_KEY, + SchemaBuilder.string().optional().build()) + .field("keyspace", SchemaBuilder.string().build()) + .field(AbstractSourceInfo.TABLE_NAME_KEY, SchemaBuilder.string().build()) + .build(); + Struct sourceStruct = + new Struct(sourceSchema) + .put(AbstractSourceInfo.DATABASE_NAME_KEY, databaseName) + .put("keyspace", "inventory") + .put(AbstractSourceInfo.TABLE_NAME_KEY, "products"); + Schema valueSchema = + SchemaBuilder.struct().field(Envelope.FieldName.SOURCE, sourceSchema).build(); + Struct valueStruct = new Struct(valueSchema).put(Envelope.FieldName.SOURCE, sourceStruct); + SourceRecord record = + new SourceRecord(null, null, "vitess", null, null, valueSchema, valueStruct); + + TablePath tablePath = SourceRecordUtils.getTablePath(record); + TableId tableId = SourceRecordUtils.getTableId(record); + + Assertions.assertEquals(TablePath.of("inventory", null, "products"), tablePath); + Assertions.assertEquals(new TableId("inventory", null, "products"), tableId); + } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/pom.xml b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/pom.xml new file mode 100644 index 000000000000..29bf8763e548 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/pom.xml @@ -0,0 +1,101 @@ + + + + 4.0.0 + + + org.apache.seatunnel + connector-cdc + ${revision} + + + connector-cdc-vitess + SeaTunnel : Connectors V2 : CDC : Vitess + + + + + org.apache.seatunnel + connector-cdc-base + ${project.version} + compile + + + org.apache.seatunnel + connector-jdbc + ${project.version} + pom + import + + + io.debezium + debezium-connector-vitess + ${debezium.version} + compile + + + io.debezium + debezium-core + + + io.debezium + debezium-api + + + + + org.testcontainers + mysql + ${testcontainer.version} + test + + + + + + + org.apache.seatunnel + connector-cdc-base + provided + + + + org.apache.seatunnel + connector-jdbc + ${project.version} + + + + io.debezium + debezium-connector-vitess + + + + org.testcontainers + mysql + + + mysql + mysql-connector-java + test + + + diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/config/VitessSourceConfig.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/config/VitessSourceConfig.java new file mode 100644 index 000000000000..adc63890c2e1 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/config/VitessSourceConfig.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; +import org.apache.seatunnel.connectors.cdc.debezium.DebeziumDeserializationSchema; +import org.apache.seatunnel.connectors.cdc.debezium.DeserializeFormat; +import org.apache.seatunnel.connectors.cdc.debezium.row.DebeziumJsonDeserializeSchema; +import org.apache.seatunnel.connectors.cdc.debezium.row.SeaTunnelRowDebeziumDeserializeSchema; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessSourceSplit; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessTableSchemaState; + +import io.debezium.connector.vitess.SourceInfo; +import io.debezium.connector.vitess.Vgtid; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Immutable runtime configuration for the Vitess CDC source. + * + *

The connector keeps startup semantics here instead of leaking Debezium-specific options into + * the source reader. + */ +public final class VitessSourceConfig { + + /** Captured tables must stay deterministic so downstream table identity remains stable. */ + private final List catalogTables; + + /** SeaTunnel-owned connector options. */ + private final ReadonlyConfig options; + + /** Startup mode is limited on purpose for the first delivery. */ + private final StartupMode startupMode; + + /** Stable VGTID used when startup.mode=specific. */ + private final String specificStartupVgtid; + + private VitessSourceConfig( + ReadonlyConfig options, + List catalogTables, + StartupMode startupMode, + String specificStartupVgtid) { + this.options = options; + this.catalogTables = Collections.unmodifiableList(new ArrayList<>(catalogTables)); + this.startupMode = startupMode; + this.specificStartupVgtid = specificStartupVgtid; + } + + /** + * Builds the validated connector configuration. + * + *

Vitess does not expose database/schema names the same way as MySQL-based connectors, so + * table paths must already be deterministic before the source starts. + */ + public static VitessSourceConfig of(ReadonlyConfig options, List catalogTables) { + if (catalogTables == null || catalogTables.isEmpty()) { + throw new IllegalArgumentException( + "Vitess CDC requires resolved catalog tables for deterministic table identity."); + } + + String keyspace = options.get(VitessSourceOptions.KEYSPACE); + for (CatalogTable catalogTable : catalogTables) { + String databaseName = catalogTable.getTablePath().getDatabaseName(); + if (databaseName == null) { + throw new IllegalArgumentException( + String.format( + "Vitess CDC requires database-qualified table paths, but table '%s' does not define a database name.", + catalogTable.getTablePath())); + } + if (!keyspace.equals(databaseName)) { + throw new IllegalArgumentException( + String.format( + "Vitess CDC captures one keyspace per source. Table '%s' does not belong to keyspace '%s'.", + catalogTable.getTablePath(), keyspace)); + } + if (catalogTable.getTablePath().getSchemaName() != null) { + throw new IllegalArgumentException( + String.format( + "Vitess CDC does not support schema-qualified table paths. Table '%s' contains an unexpected schema component.", + catalogTable.getTablePath())); + } + } + + StartupMode startupMode = options.get(VitessSourceOptions.STARTUP_MODE); + String specificVgtid = + options.getOptional(VitessSourceOptions.STARTUP_SPECIFIC_OFFSET_VGTID).orElse(null); + if (startupMode == StartupMode.SPECIFIC) { + if (specificVgtid == null) { + throw new IllegalArgumentException( + "startup.specific-offset.vgtid is required when startup.mode=specific."); + } + // Parse eagerly so configuration failures surface before the source thread starts. + Vgtid.of(specificVgtid); + } + + return new VitessSourceConfig(options, catalogTables, startupMode, specificVgtid); + } + + /** Returns the user-facing connector name used by plugin registration. */ + public String getPluginName() { + return "Vitess-CDC"; + } + + /** Returns the resolved output tables. */ + public List getCatalogTables() { + return catalogTables; + } + + /** Creates the initial split so checkpoint state already contains the startup intent. */ + public VitessSourceSplit createInitialSplit() { + return new VitessSourceSplit( + VitessSourceSplit.SPLIT_ID, + createStartupOffset(), + VitessTableSchemaState.serializeCatalogTables(catalogTables), + catalogTables); + } + + /** Creates the Debezium deserializer selected by the SeaTunnel format option. */ + @SuppressWarnings("unchecked") + public DebeziumDeserializationSchema createDeserializer() { + if (DeserializeFormat.COMPATIBLE_DEBEZIUM_JSON.equals(options.get(SourceOptions.FORMAT))) { + return new DebeziumJsonDeserializeSchema(getDebeziumPropertiesAsMap()); + } + return SeaTunnelRowDebeziumDeserializeSchema.builder() + .setTables(catalogTables) + .setServerTimeZone(ZoneId.of(options.get(VitessSourceOptions.SERVER_TIME_ZONE))) + .build(); + } + + /** + * Builds Debezium properties while keeping SeaTunnel-owned semantics authoritative. + * + *

Pass-through Debezium properties are applied first so connector-owned options can safely + * overwrite conflicting low-level keys. + */ + public Properties toDebeziumProperties() { + Properties properties = new Properties(); + properties.putAll(getDebeziumPropertiesAsMap()); + + String logicalName = buildLogicalName(); + properties.setProperty("connector.class", "io.debezium.connector.vitess.VitessConnector"); + properties.setProperty("name", logicalName); + properties.setProperty("tasks.max", "1"); + properties.setProperty("database.server.name", logicalName); + properties.setProperty("database.hostname", options.get(VitessSourceOptions.HOSTNAME)); + properties.setProperty( + "database.port", String.valueOf(options.get(VitessSourceOptions.PORT))); + properties.setProperty("vitess.keyspace", options.get(VitessSourceOptions.KEYSPACE)); + properties.setProperty( + "vitess.tablet.type", options.get(VitessSourceOptions.TABLET_TYPE).name()); + properties.setProperty( + "vitess.stop_on_reshard", + String.valueOf(options.get(VitessSourceOptions.STOP_ON_RESHARD))); + properties.setProperty( + "vitess.keepalive.interval.ms", + String.valueOf(options.get(VitessSourceOptions.KEEPALIVE_INTERVAL_MS))); + properties.setProperty( + "vitess.grpc.max_inbound_message_size", + String.valueOf(options.get(VitessSourceOptions.GRPC_MAX_INBOUND_MESSAGE_SIZE))); + properties.setProperty("plugin.name", "decoderbufs"); + properties.setProperty("include.schema.changes", "false"); + + options.getOptional(VitessSourceOptions.USERNAME) + .ifPresent( + username -> { + properties.setProperty("database.user", username); + properties.setProperty("vitess.database.user", username); + }); + options.getOptional(VitessSourceOptions.PASSWORD) + .ifPresent( + password -> { + properties.setProperty("database.password", password); + properties.setProperty("vitess.database.password", password); + }); + options.getOptional(VitessSourceOptions.SHARD) + .ifPresent(shard -> properties.setProperty("vitess.shard", shard)); + options.getOptional(VitessSourceOptions.GRPC_HEADERS) + .ifPresent(headers -> properties.setProperty("vitess.grpc.headers", headers)); + + properties.setProperty( + "table.include.list", + catalogTables.stream() + .map(catalogTable -> catalogTable.getTablePath().toString()) + .collect(Collectors.joining(","))); + + // LATEST intentionally uses Vitess' symbolic current position for convenience startup. + // SPECIFIC uses an explicit VGTID and is the reproducible startup path. + if (startupMode == StartupMode.SPECIFIC) { + properties.setProperty("vitess.gtid", specificStartupVgtid); + } else { + properties.setProperty("vitess.gtid", "current"); + } + + return properties; + } + + /** Returns the initial offset stored in the split before the first CDC row is emitted. */ + public Map createStartupOffset() { + if (startupMode != StartupMode.SPECIFIC) { + return null; + } + Map offset = new HashMap<>(); + offset.put(SourceInfo.VGTID_KEY, specificStartupVgtid); + return offset; + } + + private Map getDebeziumPropertiesAsMap() { + return options.getOptional(SourceOptions.DEBEZIUM_PROPERTIES) + .orElse(Collections.emptyMap()); + } + + private String buildLogicalName() { + String keyspace = options.get(VitessSourceOptions.KEYSPACE); + String sanitized = Pattern.compile("[^A-Za-z0-9_]").matcher(keyspace).replaceAll("_"); + return "seatunnel_vitess_" + sanitized; + } + + @Override + public String toString() { + return "VitessSourceConfig{" + + "catalogTables=" + + catalogTables.stream() + .map(catalogTable -> catalogTable.getTablePath().toString()) + .collect(Collectors.toList()) + + ", startupMode=" + + startupMode + + ", specificStartupVgtid='" + + specificStartupVgtid + + '\'' + + '}'; + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/config/VitessSourceOptions.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/config/VitessSourceOptions.java new file mode 100644 index 000000000000..e8f8fb5f978f --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/config/VitessSourceOptions.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; + +import io.debezium.connector.vitess.connection.VitessTabletType; + +import java.util.Arrays; + +/** Source options owned by the Vitess CDC connector. */ +public final class VitessSourceOptions { + + /** VTGate hostname used by the VStream gRPC client. */ + public static final Option HOSTNAME = + Options.key("hostname") + .stringType() + .noDefaultValue() + .withDescription("Hostname or IP address of the Vitess VTGate gRPC server."); + + /** VTGate VStream gRPC port. */ + public static final Option PORT = + Options.key("port") + .intType() + .defaultValue(15991) + .withDescription("Port of the Vitess VTGate gRPC server."); + + /** Optional VTGate username when the cluster enables authentication. */ + public static final Option USERNAME = + Options.key("username") + .stringType() + .noDefaultValue() + .withDescription("Username used by the Vitess VTGate gRPC connection."); + + /** Optional VTGate password when the cluster enables authentication. */ + public static final Option PASSWORD = + Options.key("password") + .stringType() + .noDefaultValue() + .withDescription("Password used by the Vitess VTGate gRPC connection."); + + /** Single keyspace captured by one connector instance. */ + public static final Option KEYSPACE = + Options.key("keyspace") + .stringType() + .noDefaultValue() + .withDescription("Vitess keyspace to capture."); + + /** + * Optional shard restriction for deployments that intentionally bind one connector to one + * shard. + */ + public static final Option SHARD = + Options.key("shard") + .stringType() + .noDefaultValue() + .withDescription( + "Optional shard restriction. When omitted, the connector captures all shards in the configured keyspace."); + + /** + * Startup mode is intentionally narrow so the first delivery keeps startup semantics explicit. + */ + public static final Option STARTUP_MODE = + Options.key(SourceOptions.STARTUP_MODE_KEY) + .singleChoice( + StartupMode.class, + Arrays.asList(StartupMode.LATEST, StartupMode.SPECIFIC)) + .defaultValue(StartupMode.LATEST) + .withDescription( + "Startup mode for Vitess CDC. Supported values are " + + "\"latest\" and \"specific\". " + + "\"specific\" is the stable startup path for reproducible restore semantics."); + + /** Stable startup VGTID used when startup.mode=specific. */ + public static final Option STARTUP_SPECIFIC_OFFSET_VGTID = + Options.key("startup.specific-offset.vgtid") + .stringType() + .noDefaultValue() + .withDescription("Vitess VGTID used when startup.mode is set to specific."); + + /** The tablet type used by Vitess streaming. */ + public static final Option TABLET_TYPE = + Options.key("tablet-type") + .enumType(VitessTabletType.class) + .defaultValue(VitessTabletType.MASTER) + .withDescription( + "Vitess tablet type used by VStream. Supported values are MASTER, REPLICA and RDONLY."); + + /** Whether VStream should stop after resharding. */ + public static final Option STOP_ON_RESHARD = + Options.key("stop-on-reshard") + .booleanType() + .defaultValue(false) + .withDescription("Whether the connector should stop after Vitess resharding."); + + /** Optional gRPC keepalive interval. */ + public static final Option KEEPALIVE_INTERVAL_MS = + Options.key("keepalive.interval.ms") + .longType() + .defaultValue(Long.MAX_VALUE) + .withDescription( + "VStream gRPC keepalive interval in milliseconds. Long.MAX_VALUE disables keepalive."); + + /** Optional raw gRPC headers passed through to VTGate. */ + public static final Option GRPC_HEADERS = + Options.key("grpc.headers") + .stringType() + .noDefaultValue() + .withDescription("Optional comma-separated gRPC headers in key:value format."); + + /** Maximum inbound gRPC message size. */ + public static final Option GRPC_MAX_INBOUND_MESSAGE_SIZE = + Options.key("grpc.max-inbound-message-size") + .intType() + .defaultValue(4_194_304) + .withDescription("Maximum inbound VStream gRPC message size in bytes."); + + /** Time zone used by SeaTunnel row deserialization for temporal normalization. */ + public static final Option SERVER_TIME_ZONE = + Options.key("server-time-zone") + .stringType() + .defaultValue("UTC") + .withDescription("Time zone used by SeaTunnel row deserialization."); + + private VitessSourceOptions() {} +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSource.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSource.java new file mode 100644 index 000000000000..f8902148c631 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSource.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source; + +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config.VitessSourceConfig; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.enumerator.VitessSourceEnumerator; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.enumerator.VitessSourceEnumeratorState; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.reader.VitessSourceReader; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessSourceSplit; + +import java.util.Collections; +import java.util.List; + +/** SeaTunnel source implementation for Vitess CDC. */ +public class VitessSource + implements SeaTunnelSource { + + public static final String IDENTIFIER = "Vitess-CDC"; + + /** Immutable connector configuration used by reader and enumerator. */ + private final VitessSourceConfig sourceConfig; + + public VitessSource(VitessSourceConfig sourceConfig) { + this.sourceConfig = sourceConfig; + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.UNBOUNDED; + } + + @Override + public String getPluginName() { + return IDENTIFIER; + } + + @Override + public List getProducedCatalogTables() { + return sourceConfig.getCatalogTables(); + } + + @Override + public SourceReader createReader( + SourceReader.Context readerContext) { + return new VitessSourceReader(readerContext, sourceConfig); + } + + @Override + public SourceSplitEnumerator createEnumerator( + SourceSplitEnumerator.Context enumeratorContext) { + return new VitessSourceEnumerator( + enumeratorContext, + new VitessSourceEnumeratorState( + Collections.singletonList(sourceConfig.createInitialSplit()))); + } + + @Override + public SourceSplitEnumerator restoreEnumerator( + SourceSplitEnumerator.Context enumeratorContext, + VitessSourceEnumeratorState checkpointState) { + return new VitessSourceEnumerator(enumeratorContext, checkpointState); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSourceFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSourceFactory.java new file mode 100644 index 000000000000..3528cfb8140f --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSourceFactory.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.options.ConnectorCommonOptions; +import org.apache.seatunnel.api.options.table.ColumnOptions; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.connector.TableSource; +import org.apache.seatunnel.api.table.factory.ChangeStreamTableSourceFactory; +import org.apache.seatunnel.api.table.factory.ChangeStreamTableSourceState; +import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; +import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config.VitessSourceConfig; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config.VitessSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessSourceSplit; + +import com.google.auto.service.AutoService; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** Factory for the Vitess CDC source connector. */ +@AutoService(Factory.class) +public class VitessSourceFactory implements ChangeStreamTableSourceFactory { + + @Override + public String factoryIdentifier() { + return VitessSource.IDENTIFIER; + } + + @Override + public OptionRule optionRule() { + return OptionRule.builder() + .required(VitessSourceOptions.HOSTNAME, VitessSourceOptions.KEYSPACE) + .exclusive(ConnectorCommonOptions.SCHEMA, ConnectorCommonOptions.TABLE_CONFIGS) + .optional( + ConnectorCommonOptions.TABLE_NAMES, + ConnectorCommonOptions.TABLE_PATTERN, + ConnectorCommonOptions.METALAKE_TYPE, + VitessSourceOptions.PORT, + VitessSourceOptions.USERNAME, + VitessSourceOptions.PASSWORD, + VitessSourceOptions.SHARD, + VitessSourceOptions.STARTUP_MODE, + VitessSourceOptions.TABLET_TYPE, + VitessSourceOptions.STOP_ON_RESHARD, + VitessSourceOptions.KEEPALIVE_INTERVAL_MS, + VitessSourceOptions.GRPC_HEADERS, + VitessSourceOptions.GRPC_MAX_INBOUND_MESSAGE_SIZE, + VitessSourceOptions.SERVER_TIME_ZONE, + SourceOptions.FORMAT, + SourceOptions.DEBEZIUM_PROPERTIES) + .conditional( + VitessSourceOptions.STARTUP_MODE, + StartupMode.SPECIFIC, + VitessSourceOptions.STARTUP_SPECIFIC_OFFSET_VGTID) + .build(); + } + + @Override + public Class getSourceClass() { + return VitessSource.class; + } + + @Override + public + TableSource createSource(TableSourceFactoryContext context) { + return createVitessTableSource(context, Collections.emptyList()); + } + + @Override + public + TableSource restoreSource( + TableSourceFactoryContext context, + ChangeStreamTableSourceState state) { + return createVitessTableSource(context, extractCheckpointTables(state)); + } + + @SuppressWarnings("unchecked") + private + TableSource createVitessTableSource( + TableSourceFactoryContext context, List checkpointTables) { + return () -> { + List catalogTables = + checkpointTables.isEmpty() ? resolveCatalogTables(context) : checkpointTables; + VitessSourceConfig sourceConfig = + VitessSourceConfig.of(context.getOptions(), catalogTables); + return (SeaTunnelSource) new VitessSource(sourceConfig); + }; + } + + /** + * Vitess does not provide a SeaTunnel catalog implementation yet, so the first delivery keeps + * schema discovery deterministic by requiring explicit schema metadata from the user. + */ + private List resolveCatalogTables(TableSourceFactoryContext context) { + validateSchemaMetadataContract(context); + validateSelectionOptions(context); + return filterDeclaredTables(context, discoverTableSchemas(context)); + } + + private void validateSchemaMetadataContract(TableSourceFactoryContext context) { + Optional> singleSchema = + context.getOptions().getOptional(ConnectorCommonOptions.SCHEMA); + Optional>> tableConfigs = + context.getOptions().getOptional(ConnectorCommonOptions.TABLE_CONFIGS); + if (singleSchema.isPresent()) { + validateRootSchemaDefinition(singleSchema.get(), "schema"); + return; + } + if (tableConfigs.isPresent()) { + List> configs = tableConfigs.get(); + if (configs.isEmpty()) { + throw new IllegalArgumentException("tables_configs can not be empty."); + } + for (int index = 0; index < configs.size(); index++) { + validateSchemaDefinition(configs.get(index), "tables_configs[" + index + "]"); + } + return; + } + throw new IllegalArgumentException( + "Vitess CDC requires explicit schema metadata through either 'schema' or 'tables_configs'."); + } + + @SuppressWarnings("unchecked") + private void validateRootSchemaDefinition(Map schema, String optionName) { + if (!schema.containsKey(ColumnOptions.COLUMNS.key()) + && !schema.containsKey(ColumnOptions.METADATA_TABLE_ID.key())) { + throw new IllegalArgumentException( + String.format( + "Vitess CDC requires explicit columns or metadata_table_id in '%s' so table schemas stay deterministic.", + optionName)); + } + } + + @SuppressWarnings("unchecked") + private void validateSchemaDefinition(Map config, String optionName) { + Object schemaValue = config.get(ConnectorCommonOptions.SCHEMA.key()); + if (!(schemaValue instanceof Map)) { + throw new IllegalArgumentException( + String.format( + "Vitess CDC requires explicit schema metadata. '%s' must contain a schema block.", + optionName)); + } + validateRootSchemaDefinition((Map) schemaValue, optionName + ".schema"); + } + + private void validateSelectionOptions(TableSourceFactoryContext context) { + if (context.getOptions().getOptional(ConnectorCommonOptions.TABLE_NAMES).isPresent() + && context.getOptions() + .getOptional(ConnectorCommonOptions.TABLE_PATTERN) + .isPresent()) { + throw new IllegalArgumentException( + "Vitess CDC accepts either table-names or table-pattern, but not both."); + } + } + + private List filterDeclaredTables( + TableSourceFactoryContext context, List declaredTables) { + Optional> explicitTableNames = + context.getOptions().getOptional(ConnectorCommonOptions.TABLE_NAMES); + if (explicitTableNames.isPresent()) { + List selectedTables = new ArrayList<>(explicitTableNames.get().size()); + for (String tableName : explicitTableNames.get()) { + CatalogTable matchedTable = + declaredTables.stream() + .filter(table -> table.getTablePath().toString().equals(tableName)) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + String.format( + "Vitess CDC did not find schema metadata for declared table '%s'. Add it to schema/tables_configs first.", + tableName))); + selectedTables.add(matchedTable); + } + return selectedTables; + } + + Optional tablePattern = + context.getOptions().getOptional(ConnectorCommonOptions.TABLE_PATTERN); + if (tablePattern.isPresent()) { + Pattern regex = Pattern.compile(tablePattern.get()); + List selectedTables = + declaredTables.stream() + .filter( + table -> + regex.matcher(table.getTablePath().toString()) + .matches()) + .collect(Collectors.toList()); + if (selectedTables.isEmpty()) { + throw new IllegalArgumentException( + String.format( + "Vitess CDC table-pattern '%s' did not match any declared schema table.", + tablePattern.get())); + } + return selectedTables; + } + + return declaredTables; + } + + private + List extractCheckpointTables( + ChangeStreamTableSourceState state) { + if (state == null || state.getSplits() == null) { + return Collections.emptyList(); + } + for (List splitGroup : state.getSplits()) { + if (splitGroup == null) { + continue; + } + for (SplitT split : splitGroup) { + if (!(split instanceof VitessSourceSplit)) { + continue; + } + List checkpointTables = + ((VitessSourceSplit) split).getCheckpointTables(); + if (checkpointTables != null && !checkpointTables.isEmpty()) { + return checkpointTables; + } + } + } + return Collections.emptyList(); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/enumerator/VitessSourceEnumerator.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/enumerator/VitessSourceEnumerator.java new file mode 100644 index 000000000000..9f50e728c360 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/enumerator/VitessSourceEnumerator.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.enumerator; + +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessSourceSplit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.OptionalInt; +import java.util.Set; + +/** + * Enumerator for the first Vitess CDC delivery. + * + *

The connector intentionally owns one long-running streaming split. This keeps the first + * delivery narrow while still integrating with SeaTunnel checkpoint and restore semantics. + */ +public class VitessSourceEnumerator + implements SourceSplitEnumerator { + + /** Pending split list doubles as the checkpoint state. */ + private final List pendingSplits; + + private final Context context; + + public VitessSourceEnumerator( + Context context, VitessSourceEnumeratorState restoredState) { + this.context = context; + if (restoredState == null) { + this.pendingSplits = new ArrayList<>(); + } else { + this.pendingSplits = restoredState.getPendingSplits(); + } + } + + @Override + public void open() {} + + @Override + public void run() { + assignPendingSplits(context.registeredReaders()); + } + + @Override + public void close() throws IOException {} + + @Override + public void addSplitsBack(List splits, int subtaskId) { + if (splits == null || splits.isEmpty()) { + return; + } + pendingSplits.addAll(copySplits(splits)); + assignPendingSplits(Collections.singleton(subtaskId)); + } + + @Override + public int currentUnassignedSplitSize() { + return pendingSplits.size(); + } + + @Override + public void handleSplitRequest(int subtaskId) { + assignPendingSplits(Collections.singleton(subtaskId)); + } + + @Override + public void registerReader(int subtaskId) { + assignPendingSplits(Collections.singleton(subtaskId)); + } + + @Override + public VitessSourceEnumeratorState snapshotState(long checkpointId) { + return new VitessSourceEnumeratorState(pendingSplits); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) {} + + private void assignPendingSplits(Set readers) { + if (pendingSplits.isEmpty() || readers == null || readers.isEmpty()) { + return; + } + OptionalInt readerId = readers.stream().mapToInt(Integer::intValue).min(); + if (!readerId.isPresent()) { + return; + } + VitessSourceSplit split = pendingSplits.remove(0); + context.assignSplit(readerId.getAsInt(), split); + for (Integer registeredReader : readers) { + context.signalNoMoreSplits(registeredReader); + } + } + + private static List copySplits(List splits) { + List copies = new ArrayList<>(splits.size()); + for (VitessSourceSplit split : splits) { + copies.add(split.copy()); + } + return copies; + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/enumerator/VitessSourceEnumeratorState.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/enumerator/VitessSourceEnumeratorState.java new file mode 100644 index 000000000000..b88db4cbb84d --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/enumerator/VitessSourceEnumeratorState.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.enumerator; + +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessSourceSplit; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Checkpoint state for the single-split Vitess enumerator. */ +public class VitessSourceEnumeratorState implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * Pending splits are enough because the first delivery intentionally owns one streaming split. + */ + private final List pendingSplits; + + public VitessSourceEnumeratorState(List pendingSplits) { + List copies = new ArrayList<>(); + if (pendingSplits != null) { + for (VitessSourceSplit pendingSplit : pendingSplits) { + copies.add(pendingSplit.copy()); + } + } + this.pendingSplits = Collections.unmodifiableList(copies); + } + + public List getPendingSplits() { + List copies = new ArrayList<>(pendingSplits.size()); + for (VitessSourceSplit pendingSplit : pendingSplits) { + copies.add(pendingSplit.copy()); + } + return copies; + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/VitessSourceReader.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/VitessSourceReader.java new file mode 100644 index 000000000000..7a16db35325e --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/VitessSourceReader.java @@ -0,0 +1,379 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.reader; + +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.cdc.debezium.DebeziumDeserializationSchema; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config.VitessSourceConfig; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessSourceSplit; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessTableSchemaState; + +import org.apache.kafka.connect.source.SourceRecord; + +import io.debezium.config.Configuration; +import io.debezium.connector.base.ChangeEventQueue; +import io.debezium.connector.vitess.Filters; +import io.debezium.connector.vitess.VitessChangeEventSourceFactory; +import io.debezium.connector.vitess.VitessConnectorConfig; +import io.debezium.connector.vitess.VitessDatabaseSchema; +import io.debezium.connector.vitess.VitessErrorHandler; +import io.debezium.connector.vitess.VitessEventMetadataProvider; +import io.debezium.connector.vitess.VitessOffsetContext; +import io.debezium.connector.vitess.VitessPartition; +import io.debezium.connector.vitess.VitessTaskContext; +import io.debezium.connector.vitess.VitessTopicSelector; +import io.debezium.connector.vitess.connection.VitessReplicationConnection; +import io.debezium.pipeline.ChangeEventSourceCoordinator; +import io.debezium.pipeline.DataChangeEvent; +import io.debezium.pipeline.EventDispatcher; +import io.debezium.pipeline.metrics.DefaultChangeEventSourceMetricsFactory; +import io.debezium.pipeline.spi.Offsets; +import io.debezium.relational.Table; +import io.debezium.relational.TableId; +import io.debezium.relational.Tables; +import io.debezium.schema.TopicSelector; +import io.debezium.util.Clock; +import io.debezium.util.LoggingContext; +import io.debezium.util.SchemaNameAdjuster; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * Source reader backed by Debezium Vitess streaming. + * + *

The reader owns one Debezium streaming runtime and persists the latest source offset directly + * in the SeaTunnel split so checkpoint / restore can resume from the last emitted Vitess position. + */ +public class VitessSourceReader implements SourceReader { + + /** Reader context is also used to request the single split. */ + private final Context context; + + /** Immutable connector configuration shared by runtime and checkpoint code. */ + private final VitessSourceConfig sourceConfig; + + /** + * Debezium deserializer is owned by the source config so startup semantics stay centralized. + */ + private final DebeziumDeserializationSchema deserializer; + + /** Reader state is tiny but still guarded because snapshotState can race with pollNext. */ + private final Object stateLock = new Object(); + + private final List sourceSplits = new ArrayList<>(1); + + private VitessStreamingRuntime runtime; + + public VitessSourceReader(Context context, VitessSourceConfig sourceConfig) { + this.context = context; + this.sourceConfig = sourceConfig; + this.deserializer = sourceConfig.createDeserializer(); + } + + @Override + public void open() { + context.sendSplitRequest(); + } + + /** + * Polls Debezium records and updates the split offset inside the same critical section that + * emits SeaTunnel rows. + * + *

This keeps checkpoint state aligned with the last fully emitted source record. + */ + @Override + public void pollNext(Collector output) throws Exception { + VitessStreamingRuntime currentRuntime = ensureRuntime(); + if (currentRuntime == null) { + return; + } + + List records = currentRuntime.poll(); + if (records.isEmpty()) { + return; + } + + synchronized (output.getCheckpointLock()) { + synchronized (stateLock) { + VitessSourceSplit split = sourceSplits.get(0); + for (SourceRecord record : records) { + deserializer.deserialize(record, output); + split.setOffset(record.sourceOffset()); + } + } + } + } + + @Override + public List snapshotState(long checkpointId) { + synchronized (stateLock) { + if (runtime != null && !sourceSplits.isEmpty()) { + Map runtimeTableSchemas = runtime.snapshotTableSchemas(); + if (runtimeTableSchemas != null && !runtimeTableSchemas.isEmpty()) { + sourceSplits.get(0).setTableSchemas(runtimeTableSchemas); + } + } + return sourceSplits.stream().map(VitessSourceSplit::copy).collect(Collectors.toList()); + } + } + + @Override + public void addSplits(List splits) { + synchronized (stateLock) { + if (!sourceSplits.isEmpty() && splits != null && !splits.isEmpty()) { + throw new IllegalStateException( + "Vitess CDC reader only supports one active streaming split."); + } + if (splits != null) { + for (VitessSourceSplit split : splits) { + sourceSplits.add(split.copy()); + } + } + if (runtime == null && !sourceSplits.isEmpty()) { + // Start Debezium as soon as the split arrives so latest startup does not miss + // changes produced before the framework issues the first pollNext call. + runtime = new VitessStreamingRuntime(sourceConfig, sourceSplits.get(0)); + } + } + } + + @Override + public void handleNoMoreSplits() {} + + @Override + public void notifyCheckpointComplete(long checkpointId) {} + + @Override + public void close() throws IOException { + VitessStreamingRuntime currentRuntime; + synchronized (stateLock) { + currentRuntime = runtime; + runtime = null; + } + if (currentRuntime != null) { + currentRuntime.close(); + } + } + + private VitessStreamingRuntime ensureRuntime() { + synchronized (stateLock) { + if (sourceSplits.isEmpty()) { + return null; + } + if (runtime == null) { + runtime = new VitessStreamingRuntime(sourceConfig, sourceSplits.get(0)); + } + return runtime; + } + } + + /** Small Debezium runtime wrapper so the SeaTunnel reader can stay checkpoint-focused. */ + static final class VitessStreamingRuntime implements AutoCloseable { + + private final ChangeEventQueue queue; + private final VitessDatabaseSchema schema; + private final EventDispatcher dispatcher; + private final VitessReplicationConnection replicationConnection; + private final ChangeEventSourceCoordinator + coordinator; + + /** + * Creates and starts the Debezium runtime for one SeaTunnel split using the same + * coordinator lifecycle as Debezium's official Vitess connector task. + */ + VitessStreamingRuntime(VitessSourceConfig sourceConfig, VitessSourceSplit split) { + Properties properties = sourceConfig.toDebeziumProperties(); + Configuration configuration = Configuration.from(properties); + VitessConnectorConfig connectorConfig = new VitessConnectorConfig(configuration); + TopicSelector topicSelector = + VitessTopicSelector.defaultSelector(connectorConfig); + SchemaNameAdjuster schemaNameAdjuster = + connectorConfig.schemaNameAdjustmentMode().createAdjuster(); + this.schema = + new VitessDatabaseSchema(connectorConfig, schemaNameAdjuster, topicSelector); + restoreTableSchemas(split.getTableSchemas()); + this.queue = + new ChangeEventQueue.Builder() + .pollInterval(connectorConfig.getPollInterval()) + .maxBatchSize(connectorConfig.getMaxBatchSize()) + .maxQueueSize(connectorConfig.getMaxQueueSize()) + .loggingContextSupplier( + () -> + LoggingContext.forConnector( + connectorConfig.getConnectorName(), + connectorConfig.getLogicalName(), + "seatunnel-vitess-cdc-reader")) + .build(); + + VitessErrorHandler errorHandler = new VitessErrorHandler(connectorConfig, queue); + this.dispatcher = + new EventDispatcher<>( + connectorConfig, + topicSelector, + schema, + queue, + createTableFilter(connectorConfig), + DataChangeEvent::new, + new VitessEventMetadataProvider(), + schemaNameAdjuster); + this.replicationConnection = new VitessReplicationConnection(connectorConfig, schema); + VitessTaskContext taskContext = new VitessTaskContext(connectorConfig, schema); + VitessPartition partition = new VitessPartition(connectorConfig.getLogicalName()); + VitessOffsetContext previousOffset = + split.getOffset() == null + ? null + : new VitessOffsetContext.Loader(connectorConfig) + .load(split.getOffset()); + this.coordinator = + new ChangeEventSourceCoordinator<>( + Offsets.of(partition, previousOffset), + errorHandler, + io.debezium.connector.vitess.VitessConnector.class, + connectorConfig, + new VitessChangeEventSourceFactory( + connectorConfig, + errorHandler, + dispatcher, + Clock.system(), + schema, + replicationConnection), + new DefaultChangeEventSourceMetricsFactory<>(), + dispatcher, + schema); + this.coordinator.start(taskContext, queue, new VitessEventMetadataProvider()); + } + + /** + * Debezium already batches SourceRecords for us, so the reader only converts queue items. + */ + List poll() throws InterruptedException { + return queue.poll().stream() + .map(DataChangeEvent::getRecord) + .collect(Collectors.toList()); + } + + /** + * Captures the current Debezium table definitions so restore can resume from backlog rows + * without waiting for VTGate to resend FIELD metadata. + */ + Map snapshotTableSchemas() { + Map tableSchemas = new HashMap<>(); + for (TableId tableId : schema.tableIds()) { + Table table = schema.tableFor(tableId); + if (table != null) { + tableSchemas.put( + tableId.toDoubleQuotedString(), + VitessTableSchemaState.serialize(table)); + } + } + return tableSchemas.isEmpty() ? null : tableSchemas; + } + + @Override + public void close() throws IOException { + IOException closeException = null; + + try { + coordinator.stop(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + IOException interruptedException = + new IOException( + "Interrupted while stopping Vitess change-event coordinator", e); + if (closeException == null) { + closeException = interruptedException; + } else { + closeException.addSuppressed(interruptedException); + } + } + + try { + replicationConnection.close(); + } catch (Exception e) { + IOException connectionException = + new IOException("Failed to close Vitess replication connection", e); + if (closeException == null) { + closeException = connectionException; + } else { + closeException.addSuppressed(connectionException); + } + } + + try { + schema.close(); + } catch (Exception e) { + IOException schemaException = + new IOException("Failed to close Vitess database schema", e); + if (closeException == null) { + closeException = schemaException; + } else { + closeException.addSuppressed(schemaException); + } + } + + if (closeException != null) { + throw closeException; + } + } + + /** + * Debezium 1.9's Vitess connector keeps the table filter in its dedicated Filters helper + * instead of wiring RelationalDatabaseConnectorConfig#getTableFilters(). + */ + private static Tables.TableFilter createTableFilter(VitessConnectorConfig connectorConfig) { + return new AccessibleVitessFilters(connectorConfig).exposeTableFilter(); + } + + private void restoreTableSchemas(Map tableSchemas) { + if (tableSchemas == null || tableSchemas.isEmpty()) { + return; + } + tableSchemas.values().forEach(this::restoreTableSchema); + } + + private void restoreTableSchema(byte[] tableSchemaBytes) { + Table restoredTable = VitessTableSchemaState.deserialize(tableSchemaBytes); + if (restoredTable != null) { + schema.applySchemaChangesForTable(restoredTable); + } + } + } + + /** Small bridge that exposes Debezium's protected Vitess table filter. */ + static final class AccessibleVitessFilters extends Filters { + + AccessibleVitessFilters(VitessConnectorConfig connectorConfig) { + super(connectorConfig); + } + + /** + * Returns the Debezium-owned table filter so SeaTunnel stays aligned with upstream logic. + */ + private Tables.TableFilter exposeTableFilter() { + return super.tableFilter(); + } + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessSourceSplit.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessSourceSplit.java new file mode 100644 index 000000000000..0eca8d54b4d0 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessSourceSplit.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split; + +import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.api.table.catalog.CatalogTable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Single streaming split used by the Vitess connector. + * + *

The split persists both the last committed Debezium source offset and the latest Debezium + * table definitions needed to decode backlog rows after checkpoint restore. + */ +public class VitessSourceSplit implements SourceSplit { + + public static final String SPLIT_ID = "vitess-stream-split"; + + private static final long serialVersionUID = 1L; + + /** Stable split id because the connector currently owns one streaming split per source. */ + private final String splitId; + + /** Debezium source offset stored in SeaTunnel checkpoints. */ + private Map offset; + + /** + * Serialized Debezium table definitions captured from FIELD events. + * + *

Vitess does not guarantee that field metadata is replayed after every resume position, so + * restore must keep the latest known schema together with the VGTID. + */ + private Map tableSchemas; + + /** + * SeaTunnel-side table schemas used to rebuild row converters during checkpoint restore. + * + *

The first Vitess delivery keeps schema evolution out of scope, so this stays aligned with + * the declared table schemas and must survive restart unchanged. + */ + private List checkpointTables; + + public VitessSourceSplit(String splitId, Map offset) { + this(splitId, offset, null, null); + } + + public VitessSourceSplit( + String splitId, Map offset, Map tableSchemas) { + this(splitId, offset, tableSchemas, null); + } + + public VitessSourceSplit( + String splitId, + Map offset, + Map tableSchemas, + List checkpointTables) { + this.splitId = splitId; + this.offset = copyOffset(offset); + this.tableSchemas = copyTableSchemas(tableSchemas); + this.checkpointTables = copyCheckpointTables(checkpointTables); + } + + @Override + public String splitId() { + return splitId; + } + + /** + * Returns a defensive copy so checkpoint callers cannot mutate in-memory state accidentally. + */ + public Map getOffset() { + return copyOffset(offset); + } + + /** Updates the in-memory checkpoint position after a record has been emitted successfully. */ + public void setOffset(Map offset) { + this.offset = copyOffset(offset); + } + + /** Returns the latest serialized Debezium table schemas captured for this split. */ + public Map getTableSchemas() { + return copyTableSchemas(tableSchemas); + } + + /** Updates the schema snapshot so restore can decode rows before new FIELD events arrive. */ + public void setTableSchemas(Map tableSchemas) { + this.tableSchemas = copyTableSchemas(tableSchemas); + } + + /** Returns the SeaTunnel schema snapshot used to rebuild the deserializer on restore. */ + public List getCheckpointTables() { + return copyCheckpointTables(checkpointTables); + } + + /** Creates a deep-enough copy for checkpoint serialization. */ + public VitessSourceSplit copy() { + return new VitessSourceSplit(splitId, offset, tableSchemas, checkpointTables); + } + + @Override + public String toString() { + return "VitessSourceSplit{" + + "splitId='" + + splitId + + '\'' + + ", offset=" + + offset + + ", tableSchemas=" + + (tableSchemas == null ? 0 : tableSchemas.size()) + + ", checkpointTables=" + + (checkpointTables == null ? 0 : checkpointTables.size()) + + '}'; + } + + private static Map copyOffset(Map offset) { + if (offset == null || offset.isEmpty()) { + return null; + } + return new HashMap<>(offset); + } + + private static Map copyTableSchemas(Map tableSchemas) { + if (tableSchemas == null || tableSchemas.isEmpty()) { + return null; + } + Map copiedSchemas = new HashMap<>(tableSchemas.size()); + tableSchemas.forEach( + (tableId, schemaBytes) -> + copiedSchemas.put( + tableId, schemaBytes == null ? null : schemaBytes.clone())); + return copiedSchemas; + } + + private static List copyCheckpointTables(List checkpointTables) { + if (checkpointTables == null || checkpointTables.isEmpty()) { + return null; + } + return new ArrayList<>(checkpointTables); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessTableSchemaState.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessTableSchemaState.java new file mode 100644 index 000000000000..f4b86264663e --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/split/VitessTableSchemaState.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split; + +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.type.SqlType; +import org.apache.seatunnel.connectors.cdc.debezium.ConnectTableChangeSerializer; + +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.json.JsonConverter; + +import io.debezium.relational.Table; +import io.debezium.relational.TableId; +import io.debezium.relational.history.TableChanges; + +import java.sql.Types; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Helper for persisting Debezium table definitions in Vitess split state. + * + *

Vitess may resume from a VGTID without replaying FIELD metadata immediately, so the connector + * keeps a checkpoint-safe copy of the latest known Debezium table schema. + */ +public final class VitessTableSchemaState { + + private static final String SERIALIZATION_TOPIC = "vitess-table-schema-state"; + + private static final ConnectTableChangeSerializer TABLE_CHANGE_SERIALIZER = + new ConnectTableChangeSerializer(); + + private static final JsonConverter JSON_CONVERTER = new JsonConverter(); + + static { + JSON_CONVERTER.configure(Collections.singletonMap("schemas.enable", true), false); + } + + private VitessTableSchemaState() {} + + /** Serializes a Debezium table definition into checkpoint-safe bytes. */ + public static byte[] serialize(Table table) { + TableChanges tableChanges = new TableChanges().create(table); + Struct tableChangeStruct = TABLE_CHANGE_SERIALIZER.serialize(tableChanges).get(0); + return JSON_CONVERTER.fromConnectData( + SERIALIZATION_TOPIC, tableChangeStruct.schema(), tableChangeStruct); + } + + /** Restores one Debezium table definition from checkpoint bytes. */ + public static Table deserialize(byte[] tableSchemaBytes) { + if (tableSchemaBytes == null || tableSchemaBytes.length == 0) { + return null; + } + Struct tableChangeStruct = + (Struct) + JSON_CONVERTER.toConnectData(SERIALIZATION_TOPIC, tableSchemaBytes).value(); + TableChanges tableChanges = + TABLE_CHANGE_SERIALIZER.deserialize( + Collections.singletonList(tableChangeStruct), false); + return tableChanges.iterator().next().getTable(); + } + + /** + * Builds the initial schema snapshot from user-declared catalog tables. + * + *

This bootstrap path keeps startup.mode=specific reproducible even before VTGate re-sends + * FIELD metadata for the captured tables. + */ + public static Map serializeCatalogTables(List catalogTables) { + if (catalogTables == null || catalogTables.isEmpty()) { + return null; + } + Map tableSchemas = new HashMap<>(catalogTables.size()); + for (CatalogTable catalogTable : catalogTables) { + Table table = fromCatalogTable(catalogTable); + tableSchemas.put(table.id().toDoubleQuotedString(), serialize(table)); + } + return tableSchemas; + } + + private static Table fromCatalogTable(CatalogTable catalogTable) { + TableId tableId = + new TableId( + null, + catalogTable.getTablePath().getDatabaseName(), + catalogTable.getTablePath().getTableName()); + List columns = + new java.util.ArrayList<>(catalogTable.getTableSchema().getColumns().size()); + int position = 1; + for (org.apache.seatunnel.api.table.catalog.Column catalogColumn : + catalogTable.getTableSchema().getColumns()) { + if (!catalogColumn.isPhysical()) { + continue; + } + columns.add(toDebeziumColumn(catalogColumn, position++)); + } + + io.debezium.relational.TableEditor tableEditor = Table.editor().tableId(tableId); + tableEditor.setColumns(columns); + PrimaryKey primaryKey = catalogTable.getTableSchema().getPrimaryKey(); + if (primaryKey != null) { + tableEditor.setPrimaryKeyNames(primaryKey.getColumnNames()); + } + return tableEditor.create(); + } + + /** + * Converts a SeaTunnel catalog column into a Debezium relational column definition. + * + *

The bootstrap schema is only used until real FIELD metadata arrives, so generic SQL type + * mappings are sufficient when the catalog column does not provide a database-specific + * sourceType. + */ + private static io.debezium.relational.Column toDebeziumColumn( + org.apache.seatunnel.api.table.catalog.Column catalogColumn, int position) { + io.debezium.relational.ColumnEditor columnEditor = + io.debezium.relational.Column.editor() + .name(catalogColumn.getName()) + .jdbcType(resolveJdbcType(catalogColumn)) + .position(position) + .optional(catalogColumn.isNullable()) + .comment(catalogColumn.getComment()); + + String typeExpression = normalizeTypeExpression(catalogColumn.getSourceType()); + String typeName = + typeExpression == null + ? defaultTypeName(catalogColumn.getDataType().getSqlType()) + : baseTypeName(typeExpression); + if (typeExpression == null) { + columnEditor.type(typeName); + } else { + columnEditor.type(typeName, typeExpression); + } + + if (catalogColumn.getColumnLength() != null) { + columnEditor.length(Math.toIntExact(catalogColumn.getColumnLength())); + } + if (catalogColumn.getScale() != null) { + columnEditor.scale(catalogColumn.getScale()); + } + return columnEditor.create(); + } + + private static int resolveJdbcType(org.apache.seatunnel.api.table.catalog.Column column) { + switch (column.getDataType().getSqlType()) { + case STRING: + return Types.VARCHAR; + case BOOLEAN: + return Types.BOOLEAN; + case TINYINT: + return Types.TINYINT; + case SMALLINT: + return Types.SMALLINT; + case INT: + return Types.INTEGER; + case BIGINT: + return Types.BIGINT; + case FLOAT: + return Types.FLOAT; + case DOUBLE: + return Types.DOUBLE; + case DECIMAL: + return Types.DECIMAL; + case BYTES: + return Types.BINARY; + case DATE: + return Types.DATE; + case TIME: + return Types.TIME; + case TIMESTAMP: + case TIMESTAMP_TZ: + return Types.TIMESTAMP; + default: + throw new IllegalArgumentException( + String.format( + "Vitess CDC bootstrap schema does not support catalog SQL type '%s' for column '%s'.", + column.getDataType().getSqlType(), column.getName())); + } + } + + private static String defaultTypeName(SqlType sqlType) { + switch (sqlType) { + case STRING: + return "VARCHAR"; + case BOOLEAN: + return "BOOLEAN"; + case TINYINT: + return "TINYINT"; + case SMALLINT: + return "SMALLINT"; + case INT: + return "INT"; + case BIGINT: + return "BIGINT"; + case FLOAT: + return "FLOAT"; + case DOUBLE: + return "DOUBLE"; + case DECIMAL: + return "DECIMAL"; + case BYTES: + return "BLOB"; + case DATE: + return "DATE"; + case TIME: + return "TIME"; + case TIMESTAMP: + case TIMESTAMP_TZ: + return "TIMESTAMP"; + default: + throw new IllegalArgumentException( + "Unsupported Vitess catalog SQL type: " + sqlType); + } + } + + private static String normalizeTypeExpression(String sourceType) { + if (sourceType == null) { + return null; + } + String trimmed = sourceType.trim(); + return trimmed.isEmpty() ? null : trimmed.toUpperCase(Locale.ROOT); + } + + private static String baseTypeName(String typeExpression) { + int parenthesisIndex = typeExpression.indexOf('('); + if (parenthesisIndex < 0) { + return typeExpression; + } + return typeExpression.substring(0, parenthesisIndex).trim(); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSourceFactoryTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSourceFactoryTest.java new file mode 100644 index 000000000000..b1603127da06 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/VitessSourceFactoryTest.java @@ -0,0 +1,387 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.connector.TableSource; +import org.apache.seatunnel.api.table.factory.ChangeStreamTableSourceState; +import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config.VitessSourceConfig; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config.VitessSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessSourceSplit; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Covers factory-path validation, schema selection, and restore semantics for Vitess CDC. */ +class VitessSourceFactoryTest { + + private final VitessSourceFactory factory = new VitessSourceFactory(); + + /** Specific startup must fail fast when the connector does not receive a reproducible VGTID. */ + @Test + void testSpecificStartupRequiresVgtid() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + VitessSourceConfig.of( + createConfig( + StartupMode.SPECIFIC, + null, + null, + singleTableConfigs("test.products")), + testTable())); + + Assertions.assertTrue( + exception.getMessage().contains("startup.specific-offset.vgtid is required")); + } + + /** Specific startup must persist both the offset and the checkpoint table schema snapshot. */ + @Test + void testSpecificStartupPersistsInitialCheckpointState() { + String startupVgtid = "[{\"keyspace\":\"test\",\"shard\":\"-\",\"gtid\":\"MySQL56/1-10\"}]"; + VitessSourceConfig sourceConfig = + VitessSourceConfig.of( + createConfig( + StartupMode.SPECIFIC, + startupVgtid, + null, + singleTableConfigs("test.products")), + testTable()); + + VitessSourceSplit split = sourceConfig.createInitialSplit(); + + Assertions.assertEquals(startupVgtid, split.getOffset().get("vgtid")); + Assertions.assertNotNull(split.getTableSchemas()); + Assertions.assertNotNull(split.getCheckpointTables()); + Assertions.assertEquals(1, split.getCheckpointTables().size()); + } + + /** + * The factory must reject configs that declare tables without deterministic schema metadata. + */ + @Test + void testFactoryRequiresExplicitSchemaMetadata() { + Map options = baseOptions(StartupMode.LATEST, null); + options.put("table-names", Collections.singletonList("test.products")); + + TableSourceFactoryContext context = + new TableSourceFactoryContext( + ReadonlyConfig.fromMap(options), getClass().getClassLoader()); + + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> factory.createSource(context).createSource()); + + Assertions.assertTrue(exception.getMessage().contains("requires explicit schema metadata")); + } + + /** The factory must create sources from declared table schemas instead of catalog discovery. */ + @Test + void testFactoryCreatesSourceFromDeclaredTableConfigs() { + TableSourceFactoryContext context = + createContext( + StartupMode.LATEST, + null, + null, + multiTableConfigs("test.products", "test.customers")); + + VitessSource source = createVitessSource(factory.createSource(context)); + + Assertions.assertEquals(2, source.getProducedCatalogTables().size()); + Assertions.assertEquals( + Arrays.asList("test.products", "test.customers"), + Arrays.asList( + source.getProducedCatalogTables().get(0).getTablePath().toString(), + source.getProducedCatalogTables().get(1).getTablePath().toString())); + } + + /** + * Root-level schema must work for the single-table path instead of requiring tables_configs. + */ + @Test + void testFactoryCreatesSourceFromRootSchema() { + TableSourceFactoryContext context = + createContext( + StartupMode.LATEST, + Collections.singletonList("test.products"), + null, + singleTableConfigs("test.products")); + + VitessSource source = createVitessSource(factory.createSource(context)); + + Assertions.assertEquals(1, source.getProducedCatalogTables().size()); + Assertions.assertEquals( + "test.products", + source.getProducedCatalogTables().get(0).getTablePath().toString()); + Assertions.assertEquals( + 3, source.getProducedCatalogTables().get(0).getTableSchema().getColumns().size()); + } + + /** table-pattern must filter the declared schema list deterministically before startup. */ + @Test + void testFactoryFiltersDeclaredTablesByPattern() { + Map options = baseOptions(StartupMode.LATEST, null); + options.put("table-pattern", "test\\.prod.*"); + options.put("tables_configs", multiTableConfigs("test.products", "test.customers")); + + TableSourceFactoryContext context = + new TableSourceFactoryContext( + ReadonlyConfig.fromMap(options), getClass().getClassLoader()); + + VitessSource source = createVitessSource(factory.createSource(context)); + + Assertions.assertEquals(1, source.getProducedCatalogTables().size()); + Assertions.assertEquals( + "test.products", + source.getProducedCatalogTables().get(0).getTablePath().toString()); + } + + /** + * Restore must reuse the checkpoint table snapshot so the SeaTunnel row converter stays aligned + * with the split state instead of silently downgrading to the latest config. + */ + @Test + void testRestoreSourceUsesCheckpointTables() { + List checkpointTables = + Collections.singletonList( + createTable( + "test", + "products", + Arrays.asList( + physicalColumn("id", BasicType.INT_TYPE), + physicalColumn("name", BasicType.STRING_TYPE), + physicalColumn("description", BasicType.STRING_TYPE)))); + VitessSourceSplit split = + VitessSourceConfig.of( + createConfig( + StartupMode.SPECIFIC, + "[{\"keyspace\":\"test\",\"shard\":\"-\",\"gtid\":\"MySQL56/1-10\"}]", + Collections.singletonList("test.products"), + singleTableConfigs("test.products")), + checkpointTables) + .createInitialSplit(); + ChangeStreamTableSourceState state = + new ChangeStreamTableSourceState<>( + null, Collections.singletonList(Collections.singletonList(split))); + + TableSourceFactoryContext context = + createContext( + StartupMode.LATEST, + Collections.singletonList("test.products"), + null, + singleTableConfigsWithoutDescription("test.products")); + + VitessSource restoredSource = createVitessSource(factory.restoreSource(context, state)); + + Assertions.assertEquals( + 3, + restoredSource + .getProducedCatalogTables() + .get(0) + .getTableSchema() + .getColumns() + .size()); + Assertions.assertEquals( + "description", + restoredSource + .getProducedCatalogTables() + .get(0) + .getTableSchema() + .getColumns() + .get(2) + .getName()); + } + + /** + * Table resolution must stay inside one configured keyspace so runtime table identity is + * stable. + */ + @Test + void testConfiguredKeyspaceRejectsForeignTable() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + VitessSourceConfig.of( + createConfig( + StartupMode.LATEST, + null, + Collections.singletonList("test.products"), + singleTableConfigs("test.products")), + foreignTable())); + + Assertions.assertTrue( + exception.getMessage().contains("does not belong to keyspace 'test'")); + } + + private VitessSource createVitessSource(TableSource tableSource) { + return (VitessSource) tableSource.createSource(); + } + + private TableSourceFactoryContext createContext( + StartupMode startupMode, + List tableNames, + String tablePattern, + List> tableConfigs) { + return new TableSourceFactoryContext( + createConfig(startupMode, null, tableNames, tableConfigs, tablePattern), + getClass().getClassLoader()); + } + + private ReadonlyConfig createConfig( + StartupMode startupMode, + String specificVgtid, + List tableNames, + List> tableConfigs) { + return createConfig(startupMode, specificVgtid, tableNames, tableConfigs, null); + } + + private ReadonlyConfig createConfig( + StartupMode startupMode, + String specificVgtid, + List tableNames, + List> tableConfigs, + String tablePattern) { + Map options = baseOptions(startupMode, specificVgtid); + if (tableNames != null) { + options.put("table-names", tableNames); + } + if (tablePattern != null) { + options.put("table-pattern", tablePattern); + } + if (tableConfigs.size() == 1) { + options.put("schema", tableConfigs.get(0).get("schema")); + } else { + options.put("tables_configs", tableConfigs); + } + return ReadonlyConfig.fromMap(options); + } + + private Map baseOptions(StartupMode startupMode, String specificVgtid) { + Map options = new HashMap<>(); + options.put(VitessSourceOptions.HOSTNAME.key(), "127.0.0.1"); + options.put(VitessSourceOptions.KEYSPACE.key(), "test"); + options.put(VitessSourceOptions.STARTUP_MODE.key(), startupMode.name()); + if (specificVgtid != null) { + options.put(VitessSourceOptions.STARTUP_SPECIFIC_OFFSET_VGTID.key(), specificVgtid); + } + return options; + } + + private List> singleTableConfigs(String tableName) { + return Collections.singletonList(tableConfig(tableName, true)); + } + + private List> singleTableConfigsWithoutDescription(String tableName) { + return Collections.singletonList(tableConfig(tableName, false)); + } + + private List> multiTableConfigs(String... tableNames) { + List> configs = new ArrayList<>(tableNames.length); + for (String tableName : tableNames) { + configs.add(tableConfig(tableName, tableName.endsWith("products"))); + } + return configs; + } + + private Map tableConfig(String tableName, boolean withDescription) { + List> columns = new ArrayList<>(); + columns.add(column("id", "int")); + columns.add(column("name", "string")); + if (withDescription) { + columns.add(column("description", "string")); + } + + Map primaryKey = new HashMap<>(); + primaryKey.put("name", "pk_" + tableName.substring(tableName.indexOf('.') + 1)); + primaryKey.put("columnNames", Collections.singletonList("id")); + + Map schema = new HashMap<>(); + schema.put("table", tableName); + schema.put("columns", columns); + schema.put("primaryKey", primaryKey); + + Map tableConfig = new HashMap<>(); + tableConfig.put("schema", schema); + return tableConfig; + } + + private Map column(String name, String type) { + Map column = new HashMap<>(); + column.put("name", name); + column.put("type", type); + return column; + } + + private static List testTable() { + return Collections.singletonList( + createTable( + "test", + "products", + Arrays.asList( + physicalColumn("id", BasicType.INT_TYPE), + physicalColumn("name", BasicType.STRING_TYPE), + physicalColumn("description", BasicType.STRING_TYPE)))); + } + + private static List foreignTable() { + return Collections.singletonList( + createTable( + "other", + "products", + Collections.singletonList(physicalColumn("id", BasicType.INT_TYPE)))); + } + + private static CatalogTable createTable( + String keyspace, String tableName, List columns) { + TableSchema.Builder schemaBuilder = + TableSchema.builder() + .primaryKey( + PrimaryKey.of("pk_" + tableName, Collections.singletonList("id"))); + columns.forEach(schemaBuilder::column); + return CatalogTable.of( + TableIdentifier.of(keyspace, TablePath.of(keyspace, tableName)), + schemaBuilder.build(), + Collections.emptyMap(), + Collections.emptyList(), + null); + } + + private static PhysicalColumn physicalColumn(String name, BasicType dataType) { + return PhysicalColumn.builder().name(name).dataType(dataType).build(); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/TestVitessSourceReaderIT.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/TestVitessSourceReaderIT.java new file mode 100644 index 000000000000..a083a64c385a --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/TestVitessSourceReaderIT.java @@ -0,0 +1,431 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.reader; + +import org.apache.seatunnel.api.common.metrics.MetricsContext; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.event.EventListener; +import org.apache.seatunnel.api.options.ConnectorCommonOptions; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceEvent; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.RowKind; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config.VitessSourceConfig; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.config.VitessSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.split.VitessSourceSplit; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.lifecycle.Startables; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Stream; + +/** Integration tests that exercise stable startup, table identity and checkpoint restore. */ +class TestVitessSourceReaderIT { + + private static final VitessContainer VITESS_CONTAINER = + new VitessContainer() + .withKeyspace("test") + .withExposedPorts(VitessContainer.MYSQL_PORT, VitessContainer.GRPC_PORT); + + @BeforeAll + static void startContainer() { + Startables.deepStart(Stream.of(VITESS_CONTAINER)).join(); + } + + @AfterAll + static void stopContainer() { + VITESS_CONTAINER.stop(); + } + + @BeforeEach + void initializeTables() throws Exception { + executeStatements( + "USE test", + "DROP TABLE IF EXISTS customers", + "DROP TABLE IF EXISTS products", + "CREATE TABLE products (" + + "id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY," + + "name VARCHAR(255) NOT NULL," + + "description VARCHAR(512)," + + "weight FLOAT)", + "ALTER TABLE products AUTO_INCREMENT = 101", + "CREATE TABLE customers (" + + "id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY," + + "name VARCHAR(255) NOT NULL)", + "ALTER TABLE customers AUTO_INCREMENT = 201"); + } + + @Test + void testCheckpointRestorePreservesOffsetsAndTableIdentity() throws Exception { + ReadonlyConfig config = + createConfig( + StartupMode.LATEST, null, Arrays.asList("test.products", "test.customers")); + List catalogTables = Arrays.asList(productTable(), customerTable()); + + VitessSourceConfig sourceConfig = VitessSourceConfig.of(config, catalogTables); + VitessSourceReader reader = + new VitessSourceReader(new TestingSourceReaderContext(), sourceConfig); + reader.open(); + reader.addSplits(Collections.singletonList(sourceConfig.createInitialSplit())); + waitForVitessStreamStartup(); + + TestingCollector collector = new TestingCollector(); + emitUntilRowObserved( + reader, + collector, + "INSERT INTO products VALUES (default, 'scooter', 'Small 2-wheel scooter', 3.14)", + row -> + "test.products".equals(row.getTableId()) + && "scooter".equals(row.getField(1))); + emitUntilRowObserved( + reader, + collector, + "INSERT INTO customers VALUES (default, 'alice')", + row -> + "test.customers".equals(row.getTableId()) + && "alice".equals(row.getField(1))); + + Assertions.assertTrue( + collector.rows.stream() + .anyMatch( + row -> + "test.products".equals(row.getTableId()) + && RowKind.INSERT.equals(row.getRowKind()) + && "scooter".equals(row.getField(1)))); + Assertions.assertTrue( + collector.rows.stream() + .anyMatch( + row -> + "test.customers".equals(row.getTableId()) + && "alice".equals(row.getField(1)))); + + List checkpointSplits = reader.snapshotState(1L); + Assertions.assertEquals(1, checkpointSplits.size()); + Assertions.assertNotNull(checkpointSplits.get(0).getOffset()); + Assertions.assertTrue(checkpointSplits.get(0).getOffset().containsKey("vgtid")); + Assertions.assertNotNull(checkpointSplits.get(0).getTableSchemas()); + Assertions.assertEquals(2, checkpointSplits.get(0).getTableSchemas().size()); + reader.close(); + + executeStatements( + "INSERT INTO products VALUES (default, 'car battery', '12V car battery', 8.1)"); + + VitessSourceReader restoredReader = + new VitessSourceReader(new TestingSourceReaderContext(), sourceConfig); + restoredReader.open(); + restoredReader.addSplits(checkpointSplits); + waitForVitessStreamStartup(); + + TestingCollector restoredCollector = new TestingCollector(); + waitUntilRowMatches( + restoredReader, + restoredCollector, + row -> + "test.products".equals(row.getTableId()) + && "car battery".equals(row.getField(1))); + + Assertions.assertTrue( + restoredCollector.rows.stream() + .anyMatch( + row -> + "test.products".equals(row.getTableId()) + && "car battery".equals(row.getField(1)))); + restoredReader.close(); + } + + @Test + void testSpecificStartupUsesCapturedVgtid() throws Exception { + ReadonlyConfig latestConfig = + createConfig(StartupMode.LATEST, null, Collections.singletonList("test.products")); + VitessSourceConfig latestSourceConfig = + VitessSourceConfig.of(latestConfig, Collections.singletonList(productTable())); + VitessSourceReader bootstrapReader = + new VitessSourceReader(new TestingSourceReaderContext(), latestSourceConfig); + bootstrapReader.open(); + bootstrapReader.addSplits( + Collections.singletonList(latestSourceConfig.createInitialSplit())); + waitForVitessStreamStartup(); + + TestingCollector bootstrapCollector = new TestingCollector(); + emitUntilRowObserved( + bootstrapReader, + bootstrapCollector, + "INSERT INTO products VALUES (default, 'scooter', 'Small 2-wheel scooter', 3.14)", + row -> + "test.products".equals(row.getTableId()) + && "scooter".equals(row.getField(1))); + List bootstrapCheckpointSplits = bootstrapReader.snapshotState(1L); + String capturedVgtid = (String) bootstrapCheckpointSplits.get(0).getOffset().get("vgtid"); + Assertions.assertNotNull(bootstrapCheckpointSplits.get(0).getTableSchemas()); + bootstrapReader.close(); + + executeStatements( + "INSERT INTO products VALUES (default, 'hammer', '16oz carpenters hammer', 1.0)"); + + ReadonlyConfig specificConfig = + createConfig( + StartupMode.SPECIFIC, + capturedVgtid, + Collections.singletonList("test.products")); + VitessSourceConfig specificSourceConfig = + VitessSourceConfig.of(specificConfig, Collections.singletonList(productTable())); + VitessSourceReader specificReader = + new VitessSourceReader(new TestingSourceReaderContext(), specificSourceConfig); + specificReader.open(); + VitessSourceSplit specificSplit = specificSourceConfig.createInitialSplit(); + Assertions.assertNotNull(specificSplit.getTableSchemas()); + specificReader.addSplits(Collections.singletonList(specificSplit)); + waitForVitessStreamStartup(); + + TestingCollector specificCollector = new TestingCollector(); + waitUntilRowMatches( + specificReader, + specificCollector, + row -> + "test.products".equals(row.getTableId()) + && "hammer".equals(row.getField(1))); + + Assertions.assertTrue( + specificCollector.rows.stream() + .anyMatch( + row -> + "test.products".equals(row.getTableId()) + && "hammer".equals(row.getField(1)))); + specificReader.close(); + } + + private static CatalogTable productTable() { + TableSchema tableSchema = + TableSchema.builder() + .primaryKey(PrimaryKey.of("pk_products", Collections.singletonList("id"))) + .column( + PhysicalColumn.builder() + .name("id") + .dataType(BasicType.INT_TYPE) + .build()) + .column( + PhysicalColumn.builder() + .name("name") + .dataType(BasicType.STRING_TYPE) + .build()) + .column( + PhysicalColumn.builder() + .name("description") + .dataType(BasicType.STRING_TYPE) + .build()) + .column( + PhysicalColumn.builder() + .name("weight") + .dataType(BasicType.FLOAT_TYPE) + .build()) + .build(); + return CatalogTable.of( + TableIdentifier.of("test", TablePath.of("test", "products")), + tableSchema, + Collections.emptyMap(), + Collections.emptyList(), + null); + } + + private static CatalogTable customerTable() { + TableSchema tableSchema = + TableSchema.builder() + .primaryKey(PrimaryKey.of("pk_customers", Collections.singletonList("id"))) + .column( + PhysicalColumn.builder() + .name("id") + .dataType(BasicType.INT_TYPE) + .build()) + .column( + PhysicalColumn.builder() + .name("name") + .dataType(BasicType.STRING_TYPE) + .build()) + .build(); + return CatalogTable.of( + TableIdentifier.of("test", TablePath.of("test", "customers")), + tableSchema, + Collections.emptyMap(), + Collections.emptyList(), + null); + } + + private static ReadonlyConfig createConfig( + StartupMode startupMode, String specificVgtid, List tableNames) { + Map options = new HashMap<>(); + options.put(VitessSourceOptions.HOSTNAME.key(), VITESS_CONTAINER.getHost()); + options.put(VitessSourceOptions.PORT.key(), VITESS_CONTAINER.getGrpcPort()); + options.put(VitessSourceOptions.KEYSPACE.key(), VITESS_CONTAINER.getKeyspace()); + options.put(VitessSourceOptions.STARTUP_MODE.key(), startupMode.name()); + options.put(ConnectorCommonOptions.TABLE_NAMES.key(), tableNames); + options.put(VitessSourceOptions.SERVER_TIME_ZONE.key(), "UTC"); + options.put( + SourceOptions.DEBEZIUM_PROPERTIES.key(), + Collections.singletonMap("poll.interval.ms", "100")); + if (specificVgtid != null) { + options.put(VitessSourceOptions.STARTUP_SPECIFIC_OFFSET_VGTID.key(), specificVgtid); + } + return ReadonlyConfig.fromMap(options); + } + + private static void executeStatements(String... sqlStatements) throws Exception { + try (Connection connection = DriverManager.getConnection(VITESS_CONTAINER.getJdbcUrl()); + Statement statement = connection.createStatement()) { + for (String sqlStatement : sqlStatements) { + statement.execute(sqlStatement); + } + } + } + + private static void emitUntilRowObserved( + VitessSourceReader reader, + TestingCollector collector, + String sqlStatement, + Predicate predicate) + throws Exception { + long deadline = System.currentTimeMillis() + 30_000L; + while (System.currentTimeMillis() < deadline) { + executeStatements(sqlStatement); + if (waitUntilRowMatches(reader, collector, predicate, 5_000L)) { + return; + } + } + Assertions.fail( + "Timed out waiting for Vitess CDC rows after repeated inserts. " + + "collectorRows=" + + collectorDebug(collector)); + } + + private static void waitUntilRowMatches( + VitessSourceReader reader, + TestingCollector collector, + Predicate predicate) + throws Exception { + Assertions.assertTrue( + waitUntilRowMatches(reader, collector, predicate, 30_000L), + "Timed out waiting for Vitess CDC rows."); + } + + private static boolean waitUntilRowMatches( + VitessSourceReader reader, + TestingCollector collector, + Predicate predicate, + long timeoutMillis) + throws Exception { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + reader.pollNext(collector); + if (collector.rows.stream().anyMatch(predicate)) { + return true; + } + } + return false; + } + + /** + * Flink CDC waits until the Debezium task reports a fully started state before producing test + * changes. SeaTunnel's source-reader abstraction does not expose that signal, so the IT keeps + * the same startup budget with a fixed warm-up window. + */ + private static void waitForVitessStreamStartup() throws InterruptedException { + Thread.sleep(10_000L); + } + + private static String collectorDebug(TestingCollector collector) { + if (collector.rows.isEmpty()) { + return "[]"; + } + int fromIndex = Math.max(0, collector.rows.size() - 5); + return collector.rows.subList(fromIndex, collector.rows.size()).toString(); + } + + /** Minimal source reader context used by the integration tests. */ + static final class TestingSourceReaderContext implements SourceReader.Context { + + @Override + public int getIndexOfSubtask() { + return 0; + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.UNBOUNDED; + } + + @Override + public void signalNoMoreElement() {} + + @Override + public void sendSplitRequest() {} + + @Override + public void sendSourceEventToEnumerator(SourceEvent sourceEvent) {} + + @Override + public MetricsContext getMetricsContext() { + return null; + } + + @Override + public EventListener getEventListener() { + return null; + } + } + + /** Collector used by the integration tests to capture emitted SeaTunnel CDC rows. */ + static final class TestingCollector implements Collector { + + private final Object checkpointLock = new Object(); + private final List rows = new ArrayList<>(); + + @Override + public void collect(SeaTunnelRow record) { + rows.add(record.copy()); + } + + @Override + public Object getCheckpointLock() { + return checkpointLock; + } + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/VitessContainer.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/VitessContainer.java new file mode 100644 index 000000000000..488e70bed68b --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-vitess/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/vitess/source/reader/VitessContainer.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.vitess.source.reader; + +import org.testcontainers.containers.JdbcDatabaseContainer; + +/** Minimal Testcontainers wrapper around Vitess vttestserver. */ +public class VitessContainer extends JdbcDatabaseContainer { + + public static final String IMAGE = "vitess/vttestserver"; + public static final String DEFAULT_TAG = "v17.0.2-mysql80"; + public static final Integer VITESS_PORT = 15991; + public static final Integer GRPC_PORT = VITESS_PORT + 1; + public static final Integer MYSQL_PORT = VITESS_PORT + 3; + + private String keyspace = "test"; + + public VitessContainer() { + this(DEFAULT_TAG); + } + + public VitessContainer(String tag) { + super(IMAGE + ":" + tag); + } + + @Override + protected void configure() { + addEnv("PORT", VITESS_PORT.toString()); + addEnv("KEYSPACES", keyspace); + addEnv("NUM_SHARDS", "1"); + addEnv("MYSQL_BIND_HOST", "0.0.0.0"); + } + + @Override + public String getDriverClassName() { + return "com.mysql.cj.jdbc.Driver"; + } + + @Override + public String getJdbcUrl() { + return "jdbc:mysql://" + getHost() + ":" + getMysqlPort() + "/" + keyspace; + } + + @Override + public String getUsername() { + return ""; + } + + @Override + public String getPassword() { + return ""; + } + + public Integer getGrpcPort() { + return getMappedPort(GRPC_PORT); + } + + public Integer getMysqlPort() { + return getMappedPort(MYSQL_PORT); + } + + public String getKeyspace() { + return keyspace; + } + + @Override + protected String getTestQueryString() { + return "SELECT 1"; + } + + public VitessContainer withKeyspace(String keyspace) { + this.keyspace = keyspace; + return this; + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/pom.xml b/seatunnel-connectors-v2/connector-cdc/pom.xml index 4d867d0625e6..652201859e21 100644 --- a/seatunnel-connectors-v2/connector-cdc/pom.xml +++ b/seatunnel-connectors-v2/connector-cdc/pom.xml @@ -38,6 +38,7 @@ connector-cdc-oracle connector-cdc-opengauss connector-cdc-tidb + connector-cdc-vitess diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index 92d25d8aa86f..afe4ca33f732 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -580,6 +580,12 @@ ${project.version} provided + + org.apache.seatunnel + connector-cdc-vitess + ${project.version} + provided + org.apache.seatunnel connector-cdc-postgres From cb846af92f1b8d73204b98823d948c539584e5b7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 28 Jun 2026 22:35:30 +0800 Subject: [PATCH 074/375] [Test][E2E] Stabilize MySQL CDC schema change waits (#11204) --- .../cdc/mysql/MysqlCDCWithSchemaChangeIT.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithSchemaChangeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithSchemaChangeIT.java index d2150a0103fb..c8af5ef20cdf 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithSchemaChangeIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithSchemaChangeIT.java @@ -66,6 +66,15 @@ disabledReason = "Currently SPARK do not support cdc. In addition, currently only the zeta engine supports schema evolution for pr https://github.com/apache/seatunnel/pull/5125.") public class MysqlCDCWithSchemaChangeIT extends TestSuiteBase implements TestResource { + /** + * The zeta schema-evolution path applies DDL and follow-up CDC records more slowly than a local + * MySQL/MySQL comparison, especially on loaded CI runners. + */ + private static final long SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS = 180_000L; + + private static final long STRUCTURE_AND_DATA_ASSERT_TIMEOUT_MILLIS = 300_000L; + private static final int MAX_TIMESTAMP_DRIFT_SECONDS = 60; + private static final String MYSQL_DATABASE = "shop"; private static final String SOURCE_TABLE = "products"; private static final String SINK_TABLE = "mysql_cdc_e2e_sink_table_with_schema_change"; @@ -231,7 +240,7 @@ public void testMysqlCdcSchemaChangeEventTypeFilter(TestContainer container) { }); // initial snapshot synced - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( @@ -248,7 +257,7 @@ public void testMysqlCdcSchemaChangeEventTypeFilter(TestContainer container) { // add.column is NOT excluded shopDatabase.setTemplateName("add_columns_filter").createAndInitialize(); - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertTrue( @@ -260,7 +269,7 @@ public void testMysqlCdcSchemaChangeEventTypeFilter(TestContainer container) { shopDatabase.setTemplateName("drop_columns_filter").createAndInitialize(); // regression: the concurrent data changes must still reach the sink (job did not crash) - await().atMost(60000, TimeUnit.MILLISECONDS) + await().atMost(STRUCTURE_AND_DATA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( @@ -320,7 +329,7 @@ private boolean columnExists(String database, String table, String column) { } private void assertSchemaEvolution(String database, String sourceTable, String sinkTable) { - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( @@ -329,13 +338,13 @@ private void assertSchemaEvolution(String database, String sourceTable, String s // case1 add columns with cdc data at same time shopDatabase.setTemplateName("add_columns").createAndInitialize(); - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( query(String.format(DESC, database, sourceTable)), query(String.format(DESC, database, sinkTable)))); - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> { Assertions.assertIterableEquals( @@ -367,8 +376,10 @@ private void assertSchemaEvolution(String database, String sourceTable, String s while (resultSet.next()) { int timeDiff = resultSet.getInt("time_diff"); Assertions.assertTrue( - timeDiff <= 3, - "Time difference exceeds 3 seconds: " + timeDiff <= MAX_TIMESTAMP_DRIFT_SECONDS, + "Time difference exceeds " + + MAX_TIMESTAMP_DRIFT_SECONDS + + " seconds: " + timeDiff + " seconds"); } @@ -393,7 +404,7 @@ private void assertCaseByDdlName( private void assertSchemaEvolutionForAddColumns( String database, String sourceTable, String sinkTable) { - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( @@ -402,13 +413,13 @@ private void assertSchemaEvolutionForAddColumns( // case1 add columns with cdc data at same time shopDatabase.setTemplateName("add_columns").createAndInitialize(); - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( query(String.format(DESC, database, sourceTable)), query(String.format(DESC, database, sinkTable)))); - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(SCHEMA_EVOLUTION_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> { Assertions.assertIterableEquals( @@ -440,8 +451,10 @@ private void assertSchemaEvolutionForAddColumns( while (resultSet.next()) { int timeDiff = resultSet.getInt("time_diff"); Assertions.assertTrue( - timeDiff <= 3, - "Time difference exceeds 3 seconds: " + timeDiff <= MAX_TIMESTAMP_DRIFT_SECONDS, + "Time difference exceeds " + + MAX_TIMESTAMP_DRIFT_SECONDS + + " seconds: " + timeDiff + " seconds"); } @@ -451,13 +464,13 @@ private void assertSchemaEvolutionForAddColumns( private void assertTableStructureAndData( String database, String sourceTable, String sinkTable) { - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(STRUCTURE_AND_DATA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( query(String.format(DESC, database, sourceTable)), query(String.format(DESC, database, sinkTable)))); - await().atMost(30000, TimeUnit.MILLISECONDS) + await().atMost(STRUCTURE_AND_DATA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( From 3bca70125848d3f254363c3c08f02397a1d88e9c Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 28 Jun 2026 22:51:03 +0800 Subject: [PATCH 075/375] [Test][E2E] Stabilize Flink MySQL schema evolution assertions (#11194) --- .../MysqlCDCWithFlinkSchemaChangeIT.java | 122 +++++++++++++----- 1 file changed, 88 insertions(+), 34 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithFlinkSchemaChangeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithFlinkSchemaChangeIT.java index 37fea77fe210..3ce40de7833d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithFlinkSchemaChangeIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCWithFlinkSchemaChangeIT.java @@ -48,9 +48,11 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.awaitility.Awaitility.await; @@ -72,10 +74,9 @@ public class MysqlCDCWithFlinkSchemaChangeIT extends TestSuiteBase implements Te private static final String MYSQL_USER_NAME = "mysqluser"; private static final String MYSQL_USER_PASSWORD = "mysqlpw"; - private static final String QUERY = "select * from %s.%s"; private static final String DESC = "desc %s.%s"; private static final String PROJECTION_QUERY = - "select id,name,description,weight,add_column1,add_column2,add_column3 from %s.%s;"; + "select id,name,description,weight,add_column1,add_column2,add_column3 from %s.%s order by id;"; private static final MySqlContainer MYSQL_CONTAINER = createMySqlContainer(MySqlVersion.V8_0); @@ -171,28 +172,21 @@ private void assertSchemaEvolution(String database, String sourceTable, String s await().atMost(180000, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - query(String.format(QUERY, database, sourceTable)), - query(String.format(QUERY, database, sinkTable)))); + assertTableDataEqualsBySourceColumnOrder( + database, sourceTable, sinkTable, null)); // case1 add columns with cdc data at same time shopDatabase.setTemplateName("add_columns").createAndInitialize(); await().atMost(180000, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - query(String.format(DESC, database, sourceTable)), - query(String.format(DESC, database, sinkTable)))); + assertSchemaDescriptionEqualsIgnoringColumnOrder( + database, sourceTable, sinkTable)); await().atMost(180000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { - Assertions.assertIterableEquals( - query( - String.format(QUERY, database, sourceTable) - + " where id >= 128"), - query( - String.format(QUERY, database, sinkTable) - + " where id >= 128")); + assertTableDataEqualsBySourceColumnOrder( + database, sourceTable, sinkTable, "id >= 128"); Assertions.assertIterableEquals( query(String.format(PROJECTION_QUERY, database, sourceTable)), @@ -239,33 +233,37 @@ private void assertCaseByDdlName( assertTableStructureAndData(database, sourceTable, sinkTable); } + /** + * Flink JDBC schema evolution can materialize columns in a different physical order even when + * the effective schema matches, so normalize DESCRIBE output by column name before asserting. + */ + private void assertSchemaDescriptionEqualsIgnoringColumnOrder( + String database, String sourceTable, String sinkTable) { + Assertions.assertIterableEquals( + normalizeDescRows(query(String.format(DESC, database, sourceTable))), + normalizeDescRows(query(String.format(DESC, database, sinkTable)))); + } + private void assertSchemaEvolutionForAddColumns( String database, String sourceTable, String sinkTable) { await().atMost(180000, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - query(String.format(QUERY, database, sourceTable)), - query(String.format(QUERY, database, sinkTable)))); + assertTableDataEqualsBySourceColumnOrder( + database, sourceTable, sinkTable, null)); // case1 add columns with cdc data at same time shopDatabase.setTemplateName("add_columns").createAndInitialize(); await().atMost(180000, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - query(String.format(DESC, database, sourceTable)), - query(String.format(DESC, database, sinkTable)))); + assertSchemaDescriptionEqualsIgnoringColumnOrder( + database, sourceTable, sinkTable)); await().atMost(180000, TimeUnit.MILLISECONDS) .untilAsserted( () -> { - Assertions.assertIterableEquals( - query( - String.format(QUERY, database, sourceTable) - + " where id >= 128"), - query( - String.format(QUERY, database, sinkTable) - + " where id >= 128")); + assertTableDataEqualsBySourceColumnOrder( + database, sourceTable, sinkTable, "id >= 128"); Assertions.assertIterableEquals( query(String.format(PROJECTION_QUERY, database, sourceTable)), @@ -302,15 +300,13 @@ private void assertTableStructureAndData( await().atMost(300000, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - query(String.format(DESC, database, sourceTable)), - query(String.format(DESC, database, sinkTable)))); + assertSchemaDescriptionEqualsIgnoringColumnOrder( + database, sourceTable, sinkTable)); await().atMost(300000, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - query(String.format(QUERY, database, sourceTable)), - query(String.format(QUERY, database, sinkTable)))); + assertTableDataEqualsBySourceColumnOrder( + database, sourceTable, sinkTable, null)); } private Connection getJdbcConnection() throws SQLException { @@ -320,6 +316,64 @@ private Connection getJdbcConnection() throws SQLException { MYSQL_CONTAINER.getPassword()); } + /** + * Read both tables using the source column order so data assertions stay stable when the sink + * keeps equivalent columns but stores them in a different physical position. + */ + private void assertTableDataEqualsBySourceColumnOrder( + String database, String sourceTable, String sinkTable, String whereClause) { + List sourceColumns = getColumnNames(database, sourceTable); + Assertions.assertIterableEquals( + query( + buildOrderedProjectionQuery( + database, sourceTable, sourceColumns, whereClause)), + query( + buildOrderedProjectionQuery( + database, sinkTable, sourceColumns, whereClause))); + } + + /** + * Returns source column names from DESCRIBE so later projections follow the semantic schema. + */ + private List getColumnNames(String database, String table) { + List columnNames = new ArrayList<>(); + for (List row : query(String.format(DESC, database, table))) { + columnNames.add(String.valueOf(row.get(0))); + } + return columnNames; + } + + /** Builds an explicit projection to avoid relying on engine-specific physical column order. */ + private String buildOrderedProjectionQuery( + String database, String table, List columns, String whereClause) { + StringBuilder queryBuilder = + new StringBuilder("select ") + .append( + columns.stream() + .map(this::quoteIdentifier) + .collect(Collectors.joining(","))) + .append(" from ") + .append(quoteIdentifier(database)) + .append(".") + .append(quoteIdentifier(table)); + if (whereClause != null && !whereClause.isEmpty()) { + queryBuilder.append(" where ").append(whereClause); + } + return queryBuilder.append(" order by id").toString(); + } + + /** Quotes identifiers because schema-change cases rename and reposition columns dynamically. */ + private String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + /** Sorts schema rows by column name so the assertion ignores physical column placement only. */ + private List> normalizeDescRows(List> descRows) { + List> normalizedRows = new ArrayList<>(descRows); + normalizedRows.sort(Comparator.comparing(row -> String.valueOf(row.get(0)))); + return normalizedRows; + } + @BeforeAll @Override public void startUp() { From a5940a7d762d02805f6d65c54c7cda114cd0ab38 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 29 Jun 2026 21:20:13 +0800 Subject: [PATCH 076/375] [Docs] Fix Jira API token guide link (#11183) --- docs/en/connectors/source/Jira.md | 2 +- docs/zh/connectors/source/Jira.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/connectors/source/Jira.md b/docs/en/connectors/source/Jira.md index 78fa6dd5653b..39c2282a6243 100644 --- a/docs/en/connectors/source/Jira.md +++ b/docs/en/connectors/source/Jira.md @@ -50,7 +50,7 @@ Jira Email Jira API Token -https://id.atlassian.com/manage-profile/security/api-tokens +https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/ ### method [String] diff --git a/docs/zh/connectors/source/Jira.md b/docs/zh/connectors/source/Jira.md index 8b374a9c96cf..f84eacfa4369 100644 --- a/docs/zh/connectors/source/Jira.md +++ b/docs/zh/connectors/source/Jira.md @@ -50,7 +50,7 @@ Jira 邮件 Jira API 接口 -https://id.atlassian.com/manage-profile/security/api-tokens +https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/ ### method [String] From 872077f6480ade460efca5501a3847fa2703e681 Mon Sep 17 00:00:00 2001 From: Doyong Kwon <42794359+doyong365@users.noreply.github.com> Date: Mon, 29 Jun 2026 23:02:28 +0900 Subject: [PATCH 077/375] [SEATUNNEL-10685] prevent timestamp_ntz from being saved as timestamp_ltz (#10724) Co-authored-by: Chanhae Oh Co-authored-by: yzeng1618 --- .gitignore | 6 + .../concepts/incompatible-changes.md | 17 ++ .../concepts/incompatible-changes.md | 17 ++ ...lRowDebeziumDeserializationConverters.java | 127 ++++++++ ...DebeziumDeserializationConvertersTest.java | 56 ++++ .../source/parser/OracleDdlParserTest.java | 2 +- .../converter/DefaultDataConverter.java | 46 +++ .../datatype/AbstractDorisTypeConverter.java | 7 + .../doris/datatype/DorisTypeConverterV1.java | 3 + .../doris/datatype/DorisTypeConverterV2.java | 3 + .../serialize/SeaTunnelRowSerializer.java | 4 +- .../sink/convert/AvroSchemaConverter.java | 5 +- .../sink/convert/RowDataToAvroConverters.java | 13 + .../iceberg/data/DefaultDeserializer.java | 4 +- .../iceberg/data/IcebergTypeMapper.java | 10 + .../iceberg/data/IcebergTypeMapperTest.java | 75 +++++ .../snowflake/SnowflakeDataTypeConvertor.java | 9 +- .../converter/AbstractJdbcRowConverter.java | 21 +- .../dialect/dm/DmdbTypeConverter.java | 24 +- .../dialect/duckdb/DuckDBTypeConverter.java | 8 +- .../inceptor/InceptorJdbcRowConverter.java | 8 +- .../kingbase/KingbaseJdbcRowConverter.java | 14 +- .../kingbase/KingbaseTypeConverter.java | 15 +- .../dialect/mysql/MySqlTypeConverter.java | 31 +- .../OceanBaseMySqlTypeConverter.java | 27 +- .../OceanBaseMysqlJdbcRowConverter.java | 12 +- .../oracle/OracleJdbcRowConverter.java | 6 + .../dialect/oracle/OracleTypeConverter.java | 48 ++- .../psql/PostgresJdbcRowConverter.java | 27 +- .../redshift/RedshiftTypeConverter.java | 19 +- .../snowflake/SnowflakeTypeConverter.java | 29 +- .../sqlserver/SqlServerTypeConverter.java | 27 +- .../sqlserver/SqlserverJdbcRowConverter.java | 18 ++ .../dialect/xugu/XuguJdbcRowConverter.java | 42 +++ .../dialect/xugu/XuguTypeConverter.java | 35 ++- .../jdbc/utils/JdbcFieldTypeUtils.java | 86 +++++- .../OracleCreateTableSqlBuilderTest.java | 11 +- .../dialect/dm/DmdbTypeConverterTest.java | 6 +- .../duckdb/DuckDBTypeConverterTest.java | 18 +- .../kingbase/KingbaseTypeConverterTest.java | 15 + .../dialect/mysql/MySqlTypeConverterTest.java | 38 ++- .../OceanBaseMySqlTypeConverterTest.java | 93 ++++++ .../oracle/OracleTypeConverterTest.java | 38 ++- .../redshift/RedshiftTypeConverterTest.java | 34 ++- .../sqlserver/SqlServerTypeConverterTest.java | 38 ++- .../dialect/xugu/XuguTypeConverterTest.java | 9 +- .../jdbc/utils/JdbcFieldTypeUtilsTest.java | 3 + .../seatunnel/paimon/utils/RowConverter.java | 33 +++ .../paimon/utils/RowTypeConverter.java | 26 +- .../datatypes/StarRocksTypeConverter.java | 13 + .../serialize/StarRocksBaseSerializer.java | 4 + .../catalog/StarRocksTypeConverterTest.java | 17 ++ .../StarRocksJsonSerializerTest.java | 22 ++ .../connector-iceberg-e2e/pom.xml | 13 + .../iceberg/JdbcToIcebergTimestampIT.java | 275 ++++++++++++++++++ .../resources/iceberg/iceberg_source.conf | 2 +- .../iceberg/mysql_iceberg_to_assert.conf | 60 ++++ .../mysql_jdbc_to_iceberg_timestamp.conf | 45 +++ .../iceberg/pg_iceberg_to_assert.conf | 60 ++++ .../iceberg/pg_jdbc_to_iceberg_timestamp.conf | 45 +++ .../resources/iceberg/iceberg_source.conf | 2 +- .../resources/iceberg/iceberg_source.conf | 2 +- .../seatunnel/jdbc/AbstractJdbcIT.java | 32 ++ .../jdbc/AbstractSchemaChangeBaseIT.java | 17 ++ .../seatunnel/jdbc/JdbcMysqlTimestampIT.java | 211 ++++++++++++++ .../seatunnel/jdbc/JdbcOracleIT.java | 11 + .../jdbc_mysql_datetime_to_assert.conf | 60 ++++ ...dbc_mysql_timestamp_non_utc_to_assert.conf | 70 +++++ .../jdbc_mysql_timestamp_to_assert.conf | 59 ++++ .../jdbc/JdbcPostgresTimestampIT.java | 188 ++++++++++++ .../jdbc_pg_timestamp_to_assert.conf | 60 ++++ .../jdbc_pg_timestamptz_to_assert.conf | 59 ++++ .../starrocks/StarRocksSchemaChangeIT.java | 25 +- .../format/csv/CsvDeserializationSchema.java | 23 ++ .../format/csv/CsvSerializationSchema.java | 27 +- .../format/csv/CsvTextFormatSchemaTest.java | 50 ++++ .../format/json/JsonSerializationSchema.java | 8 + .../format/json/RowToJsonConverters.java | 21 ++ .../text/TextDeserializationSchema.java | 21 ++ .../format/text/TextSerializationSchema.java | 28 +- .../format/text/TextFormatSchemaTest.java | 59 ++++ .../transform/sql/zeta/ZetaSQLFilter.java | 26 ++ .../sql/zeta/functions/CastFunction.java | 8 + .../sql/zeta/functions/DateTimeFunction.java | 57 +++- .../sql/zeta/functions/SystemFunction.java | 39 +++ .../zeta/functions/SystemFunctionTest.java | 37 +++ .../spark/utils/OffsetDateTimeUtils.java | 31 +- .../spark/utils/OffsetDateTimeUtilsTest.java | 69 +++++ 88 files changed, 3009 insertions(+), 110 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlTypeConverterTest.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/JdbcToIcebergTimestampIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/mysql_iceberg_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/mysql_jdbc_to_iceberg_timestamp.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/pg_iceberg_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/pg_jdbc_to_iceberg_timestamp.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlTimestampIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_datetime_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_timestamp_non_utc_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_timestamp_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresTimestampIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_pg_timestamp_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_pg_timestamptz_to_assert.conf create mode 100644 seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/test/java/org/apache/seatunnel/translation/spark/utils/OffsetDateTimeUtilsTest.java diff --git a/.gitignore b/.gitignore index bf83032a9c98..9ea48e4b1dcd 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,12 @@ logs.zip *.iml .idea/* +# Eclipse / M2E files +.classpath +.factorypath +.project +.settings/ + .DS_Store metastore_db/ diff --git a/docs/en/introduction/concepts/incompatible-changes.md b/docs/en/introduction/concepts/incompatible-changes.md index c0fe5c18240f..8903c03ca949 100644 --- a/docs/en/introduction/concepts/incompatible-changes.md +++ b/docs/en/introduction/concepts/incompatible-changes.md @@ -5,6 +5,23 @@ You need to check this document before you upgrade to related version. ## dev +### JDBC Connector + +- **Breaking Change: Mapping of timezone-aware timestamp columns to `TIMESTAMP_TZ` type** + - **Affected component**: `seatunnel-connectors-v2/connector-jdbc`, `seatunnel-connectors-v2/connector-iceberg`, `seatunnel-connectors-v2/connector-cdc-base`, `seatunnel-connectors-v2/connector-cdc-tidb`, `seatunnel-connectors-v2/connector-starrocks`, `seatunnel-connectors-v2/connector-hudi`, `seatunnel-connectors-v2/connector-snowflake` (via JDBC dialect) + - **Description**: Previously, JDBC sources mapped both timezone-naive (e.g., MySQL `DATETIME`) and timezone-aware (e.g., MySQL `TIMESTAMP`) timestamp columns to SeaTunnel's internal `TIMESTAMP` type. Now, timezone-aware columns like MySQL `TIMESTAMP`, PostgreSQL `timestamptz`, Oracle `TIMESTAMP WITH LOCAL TIME ZONE`, SQL Server `datetimeoffset`, Snowflake `TIMESTAMP_LTZ/TZ`, and others are explicitly mapped to `TIMESTAMP_TZ`. This ensures that timezone semantics are accurately preserved when writing to formats like Iceberg, where `TIMESTAMP` is saved as `timestamp` (without timezone) and `TIMESTAMP_TZ` is saved as `timestamptz` (with timezone). + - **Impact**: If your downstream Sink relies on receiving `TIMESTAMP` types and does not support `TIMESTAMP_TZ` natively, you may encounter type mismatch errors. For Iceberg users, this means columns previously written as `timestamp` (without timezone) may now be written as `timestamptz` (with timezone) and change the table schema. You may need to cast the column in sql transform or update your sink configurations. (#10685) + - **Connector-specific behavior changes**: + - **Snowflake**: `TIMESTAMP_LTZ` and `TIMESTAMP_TZ` columns are now mapped to `OFFSET_DATE_TIME_TYPE` (`TIMESTAMP_TZ`) instead of `LOCAL_DATE_TIME_TYPE`. This affects both Source and Sink paths for Snowflake. + - **StarRocks**: `TIMESTAMP_TZ` values written to StarRocks Sink are stored as `DATETIME` (wall-clock only, timezone offset is dropped) due to StarRocks not having a native timezone-aware datetime type. + - **Hudi**: `TIMESTAMP_TZ` is now mapped to Avro `timestampMillis` (UTC epoch). Existing Hudi tables written with the old schema may need to be re-created if schema evolution is not supported. + - **CDC (Debezium-based, TiDB)**: CDC connectors now correctly handle `TIMESTAMP_TZ` type in the Debezium deserialization layer. Previously, `TIMESTAMP_TZ` was unsupported and would throw `UnsupportedOperationException`. Users who were previously unable to use timezone-aware columns in CDC pipelines can now do so. + - **Iceberg (existing tables)**: Before this PR, SeaTunnel's `TIMESTAMP` type was incorrectly written to Iceberg as `timestamp` with timezone (`withZone()`). After this PR, `TIMESTAMP` is written as `timestamp` without timezone (`withoutZone()`), and Iceberg `withZone()` columns are read back as `TIMESTAMP_TZ`. **Upgrade impact**: If you have existing Iceberg tables where timestamp columns were created by an older SeaTunnel version, those columns are stored as `withZone()`. After upgrading, SeaTunnel will read them as `TIMESTAMP_TZ` instead of `TIMESTAMP`. Downstream sinks or transforms that expected `TIMESTAMP` may encounter type mismatch errors. **Migration**: Re-create the affected Iceberg table with the new schema, or use a SQL Transform to cast `TIMESTAMP_TZ` back to `TIMESTAMP` in your pipeline configuration. + - **TIMESTAMP_TZ downgrade contract**: SeaTunnel applies a two-tier serialization contract for `TIMESTAMP_TZ` depending on what the sink format can represent: + - **DB column-typed sinks without native timezone support (Doris, StarRocks, Xugu)**: The timezone offset is dropped and the wall-clock value (local datetime) is stored. For example, `2024-01-01T03:00:00+09:00` is stored as `2024-01-01 03:00:00`. This is a lossy operation — the original UTC instant cannot be recovered from the stored value alone. + - **String/text-based sinks (Text file, Kafka, Pulsar, RocketMQ, RabbitMQ, Redis, etc.)**: The full ISO 8601 offset is preserved (e.g., `"2024-01-01T03:00:00+09:00"`). These formats can represent timezone offsets as strings, so no information is lost. If you need wall-clock behavior for a string sink, use a SQL Transform to cast `TIMESTAMP_TZ` to `TIMESTAMP` before writing. + - **Xugu TIMESTAMP_TZ (lossy)**: Xugu `TIMESTAMP WITH TIME ZONE` columns are exposed as `TIMESTAMP_TZ` at the type layer, but the actual write path drops the timezone offset and stores only the wall-clock value due to a Xugu JDBC driver batch limitation (bug [E19138]). A warning is logged on the first write. + ### API Changes - **Breaking Change: Engine REST table metrics key format** diff --git a/docs/zh/introduction/concepts/incompatible-changes.md b/docs/zh/introduction/concepts/incompatible-changes.md index 8c046178f267..fe261ff28736 100644 --- a/docs/zh/introduction/concepts/incompatible-changes.md +++ b/docs/zh/introduction/concepts/incompatible-changes.md @@ -4,6 +4,23 @@ ## dev +### JDBC Connector + +- **破坏性变更：带时区的时间戳列映射为 `TIMESTAMP_TZ` 类型** + - **影响范围**：`seatunnel-connectors-v2/connector-jdbc`、`seatunnel-connectors-v2/connector-iceberg`、`seatunnel-connectors-v2/connector-cdc-base`、`seatunnel-connectors-v2/connector-cdc-tidb`、`seatunnel-connectors-v2/connector-starrocks`、`seatunnel-connectors-v2/connector-hudi`、`seatunnel-connectors-v2/connector-snowflake`（通过 JDBC 方言） + - **变更说明**：以前，JDBC Source 将无时区（如 MySQL `DATETIME`）和带时区（如 MySQL `TIMESTAMP`）的时间戳列都映射为 SeaTunnel 内部的 `TIMESTAMP` 类型。现在，带时区的列（如 MySQL `TIMESTAMP`、PostgreSQL `timestamptz`、Oracle `TIMESTAMP WITH LOCAL TIME ZONE`、SQL Server `datetimeoffset`、Snowflake `TIMESTAMP_LTZ/TZ` 等）被显式映射为 `TIMESTAMP_TZ`。这确保了在写入 Iceberg 等格式时，时区语义得到准确保留（在 Iceberg 中 `TIMESTAMP` 存为无时区的 `timestamp`，`TIMESTAMP_TZ` 存为带时区的 `timestamptz`）。 + - **影响**：如果您的下游 Sink 依赖接收 `TIMESTAMP` 类型且不支持 `TIMESTAMP_TZ`，您可能会遇到类型不匹配错误。对于 Iceberg 用户，这意味着以前作为 `timestamp`（无时区）写入的列现在可能会作为 `timestamptz`（带时区）写入，从而改变表结构。您可能需要在 SQL Transform 中转换该列或更新您的 Sink 配置。(#10685) + - **各连接器具体行为变更**： + - **Snowflake**：`TIMESTAMP_LTZ` 和 `TIMESTAMP_TZ` 列现在映射为 `OFFSET_DATE_TIME_TYPE`（`TIMESTAMP_TZ`），而不是原来的 `LOCAL_DATE_TIME_TYPE`。这同时影响 Snowflake 的 Source 和 Sink 路径。 + - **StarRocks**：写入 StarRocks Sink 的 `TIMESTAMP_TZ` 值以 `DATETIME`（仅保留时钟时间，时区偏移量丢失）形式存储，这是由于 StarRocks 不支持原生带时区的日期时间类型。 + - **Hudi**：`TIMESTAMP_TZ` 现在映射为 Avro `timestampMillis`（UTC 纪元时间）。如果 Hudi 表不支持 Schema Evolution，以旧 Schema 写入的现有表可能需要重新创建。 + - **CDC（基于 Debezium，TiDB）**：CDC 连接器现在可以正确处理 Debezium 反序列化层中的 `TIMESTAMP_TZ` 类型。以前，`TIMESTAMP_TZ` 不受支持，会抛出 `UnsupportedOperationException`。现在，在 CDC 管道中使用带时区列的用户可以正常使用。 + - **Iceberg（已有表）**：在本 PR 之前，SeaTunnel 的 `TIMESTAMP` 类型错误地以带时区（`withZone()`）的形式写入 Iceberg。本 PR 之后，`TIMESTAMP` 写为不带时区（`withoutZone()`），而 Iceberg `withZone()` 列读取时返回 `TIMESTAMP_TZ`。**升级影响**：如果您的 Iceberg 表是由旧版 SeaTunnel 创建的，其时间戳列以 `withZone()` 形式存储。升级后，SeaTunnel 会将其读取为 `TIMESTAMP_TZ` 而非 `TIMESTAMP`，下游 Sink 或 Transform 若期望 `TIMESTAMP` 类型可能遇到类型不匹配错误。**迁移方案**：重新创建受影响的 Iceberg 表，或在管道配置中使用 SQL Transform 将 `TIMESTAMP_TZ` 转换回 `TIMESTAMP`。 + - **TIMESTAMP_TZ 写入约定**：SeaTunnel 根据 Sink 格式的表达能力，对 `TIMESTAMP_TZ` 采用两级序列化约定： + - **不支持原生时区类型的 DB 列类型 Sink（Doris、StarRocks、Xugu）**：丢弃时区偏移，保留时钟时间（wall-clock）。例如，`2024-01-01T03:00:00+09:00` 将存储为 `2024-01-01 03:00:00`。这是有损操作——仅凭存储值无法还原原始 UTC 时刻。 + - **基于字符串/文本的 Sink（Text 文件、Kafka、Pulsar、RocketMQ、RabbitMQ、Redis 等）**：保留完整的 ISO 8601 偏移（例如 `"2024-01-01T03:00:00+09:00"`）。这些格式可以用字符串表示时区偏移，不会丢失信息。如果需要在这类 Sink 中使用 wall-clock 行为，请在写入前通过 SQL Transform 将 `TIMESTAMP_TZ` 转换为 `TIMESTAMP`。 + - **Xugu TIMESTAMP_TZ（有损写入）**：Xugu `TIMESTAMP WITH TIME ZONE` 列在类型层面暴露为 `TIMESTAMP_TZ`，但由于 Xugu JDBC 驱动批量执行缺陷（[E19138]），实际写入时会丢弃时区偏移，仅存储时钟时间。首次写入时会输出 WARN 日志。 + ### API 变更 - **破坏性变更：Engine REST 表级指标 key 格式变化** diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializationConverters.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializationConverters.java index b5982736d50b..3e1acd45d066 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializationConverters.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializationConverters.java @@ -165,6 +165,8 @@ public Object convert(Object dbzObj, Schema schema) throws Exception { return convertToTime(); case TIMESTAMP: return convertToTimestamp(serverTimeZone); + case TIMESTAMP_TZ: + return convertToTimestampTz(serverTimeZone); case FLOAT: return wrapNumericConverter(convertToFloat()); case DOUBLE: @@ -392,6 +394,131 @@ public Object convert(Object dbzObj, Schema schema) { }; } + /** + * Flexible fallback formatter that covers non-ISO variants emitted by Debezium: + * + *
    + *
  • Space separator instead of 'T' (MySQL in certain schema-history modes): {@code + * 2024-01-01 12:00:00+08:00} + *
  • Hour-only offset (PostgreSQL short form): {@code 2024-01-01T12:00:00+08} + *
  • Both: {@code 2024-01-01 12:00:00+08} + *
+ */ + private static final java.time.format.DateTimeFormatter FLEXIBLE_OFFSET_FORMATTER = + new java.time.format.DateTimeFormatterBuilder() + .parseLenient() + .append(java.time.format.DateTimeFormatter.ISO_LOCAL_DATE) + .optionalStart() + .appendLiteral('T') + .optionalEnd() + .optionalStart() + .appendLiteral(' ') + .optionalEnd() + .append(java.time.format.DateTimeFormatter.ISO_LOCAL_TIME) + .appendPattern("[XXX][XX][X]") + .toFormatter(); + + private static DebeziumDeserializationConverter convertToTimestampTz(ZoneId serverTimeZone) { + return new DebeziumDeserializationConverter() { + private static final long serialVersionUID = 1L; + + @Override + public Object convert(Object dbzObj, Schema schema) { + if (dbzObj instanceof String) { + return parseOffsetDateTimeFromString((String) dbzObj, serverTimeZone); + } + java.time.LocalDateTime localDateTime = + TemporalConversions.toLocalDateTime(dbzObj, serverTimeZone); + return localDateTime + .atZone(serverTimeZone) + .toOffsetDateTime() + .withOffsetSameInstant(java.time.ZoneOffset.UTC); + } + }; + } + + /** + * Parses a Debezium-emitted timestamp string into an {@link java.time.OffsetDateTime} using a + * fallback chain that tolerates the variety of formats Debezium may produce: + * + *
    + *
  1. {@link java.time.OffsetDateTime#parse} — strict ISO-8601 with numeric offset, e.g. + * {@code 2024-01-01T12:00:00+08:00} + *
  2. {@link java.time.ZonedDateTime#parse} — IANA zone-region id, e.g. {@code + * 2024-01-01T12:00:00 Asia/Shanghai} or {@code 2024-01-01 12:00:00 Asia/Shanghai}. The + * last space before an alphabetic token is treated as the zone-id boundary; any remaining + * space in the datetime portion is replaced with 'T'. + *
  3. {@link #FLEXIBLE_OFFSET_FORMATTER} — space date/time separator or short offset, e.g. + * {@code 2024-01-01 12:00:00+08:00} or {@code 2024-01-01T12:00:00+08} + *
  4. {@link Instant#parse} — UTC epoch literal, e.g. {@code 2024-01-01T12:00:00Z} + *
+ * + * If all attempts fail, an {@link IllegalArgumentException} is thrown with the raw value + * included so that the failing CDC task carries enough context for diagnosis. + */ + @VisibleForTesting + static java.time.OffsetDateTime parseOffsetDateTimeFromString( + String str, ZoneId serverTimeZone) { + // 1. Strict ISO-8601 with numeric offset: 2024-01-01T12:00:00+08:00 / Z + try { + return java.time.OffsetDateTime.parse(str) + .withOffsetSameInstant(java.time.ZoneOffset.UTC); + } catch (java.time.format.DateTimeParseException ignored) { + // fall through + } + + // 2. IANA zone-region id: "2024-01-01T12:00:00 Asia/Shanghai" + // or space-date/time variant: "2024-01-01 12:00:00 Asia/Shanghai" + // Use lastIndexOf to isolate the zone id from the datetime part so that + // any space between the date and time is not corrupted by a global replace. + try { + int zoneStart = str.lastIndexOf(' '); + if (zoneStart > 0 + && zoneStart + 1 < str.length() + && Character.isLetter(str.charAt(zoneStart + 1))) { + // Replace any space between date and time in the datetime portion with 'T'. + String dateTimePart = str.substring(0, zoneStart).trim().replace(' ', 'T'); + String zonePart = str.substring(zoneStart + 1); + String normalized = dateTimePart + "[" + zonePart + "]"; + java.time.format.DateTimeFormatter zoneRegionFmt = + new java.time.format.DateTimeFormatterBuilder() + .append(java.time.format.DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .appendLiteral('[') + .parseCaseSensitive() + .appendZoneRegionId() + .appendLiteral(']') + .toFormatter(); + return java.time.ZonedDateTime.parse(normalized, zoneRegionFmt) + .toOffsetDateTime() + .withOffsetSameInstant(java.time.ZoneOffset.UTC); + } + } catch (Exception ignored) { + // fall through + } + + // 3. Space separator or short offset: 2024-01-01 12:00:00+08:00 / +08 + try { + return java.time.OffsetDateTime.parse(str, FLEXIBLE_OFFSET_FORMATTER) + .withOffsetSameInstant(java.time.ZoneOffset.UTC); + } catch (java.time.format.DateTimeParseException ignored) { + // fall through + } + + // 4. UTC epoch literal: 2024-01-01T12:00:00Z + try { + return Instant.parse(str).atOffset(java.time.ZoneOffset.UTC); + } catch (java.time.format.DateTimeParseException ignored) { + // fall through + } + + throw new IllegalArgumentException( + "Unable to parse OffsetDateTime from CDC TIMESTAMP_TZ value: '" + + str + + "'. Supported formats: ISO-8601 with numeric offset, IANA zone-region" + + " id, space-separated date/time, short-form hour-only offset, UTC" + + " epoch literal."); + } + private static DebeziumDeserializationConverter convertToTimestamp(ZoneId serverTimeZone) { return new DebeziumDeserializationConverter() { private static final long serialVersionUID = 1L; diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializationConvertersTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializationConvertersTest.java index c17243b2bda6..e243b8fe17da 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializationConvertersTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/debezium/row/SeaTunnelRowDebeziumDeserializationConvertersTest.java @@ -37,7 +37,10 @@ import io.debezium.data.geometry.Geography; import io.debezium.data.geometry.Geometry; +import java.time.Instant; +import java.time.OffsetDateTime; import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -206,4 +209,57 @@ void testGeographyStringConversion() throws Exception { Assertions.assertTrue(fieldValue instanceof String); Assertions.assertEquals("0102FF", fieldValue); } + + /** + * Verifies the fallback chain in {@code parseOffsetDateTimeFromString} against the four + * Debezium timestamp formats called out in the review: + * + *
    + *
  1. Standard ISO-8601 with numeric offset (baseline) + *
  2. Oracle TIMESTAMP WITH LOCAL TIME ZONE — IANA zone-region id + *
  3. MySQL TIMESTAMP in certain schema-history modes — space date/time separator + *
  4. PostgreSQL timestamptz — short-form hour-only offset + *
+ * + * All four variants must parse to the same UTC instant. + */ + @Test + void testParseOffsetDateTimeFromStringFallbackChain() { + ZoneId serverTz = ZoneId.of("Asia/Shanghai"); + // Expected UTC instant: 2024-01-01 04:00:00Z (2024-01-01 12:00:00+08:00) + OffsetDateTime expected = Instant.parse("2024-01-01T04:00:00Z").atOffset(ZoneOffset.UTC); + + // 1. ISO-8601 with numeric offset — handled by OffsetDateTime.parse + OffsetDateTime r1 = + SeaTunnelRowDebeziumDeserializationConverters.parseOffsetDateTimeFromString( + "2024-01-01T12:00:00+08:00", serverTz); + Assertions.assertEquals(expected, r1, "ISO-8601 numeric offset failed"); + + // 2. IANA zone-region id — Oracle TIMESTAMP WITH LOCAL TIME ZONE + OffsetDateTime r2 = + SeaTunnelRowDebeziumDeserializationConverters.parseOffsetDateTimeFromString( + "2024-01-01T12:00:00 Asia/Shanghai", serverTz); + Assertions.assertEquals(expected, r2, "IANA zone-region id failed"); + + // 3. Space separator — MySQL in certain schema-history modes + OffsetDateTime r3 = + SeaTunnelRowDebeziumDeserializationConverters.parseOffsetDateTimeFromString( + "2024-01-01 12:00:00+08:00", serverTz); + Assertions.assertEquals(expected, r3, "Space date/time separator failed"); + + // 4. Short-form hour-only offset — PostgreSQL timestamptz + OffsetDateTime r4 = + SeaTunnelRowDebeziumDeserializationConverters.parseOffsetDateTimeFromString( + "2024-01-01T12:00:00+08", serverTz); + Assertions.assertEquals(expected, r4, "Short-form hour-only offset failed"); + } + + @Test + void testParseOffsetDateTimeFromStringThrowsOnUnknownFormat() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + SeaTunnelRowDebeziumDeserializationConverters.parseOffsetDateTimeFromString( + "not-a-timestamp", ZoneId.systemDefault())); + } } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/parser/OracleDdlParserTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/parser/OracleDdlParserTest.java index 850b8add8446..5bcf98938b07 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/parser/OracleDdlParserTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oracle/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/parser/OracleDdlParserTest.java @@ -221,7 +221,7 @@ public void testParseDDLForAddColumn() { addEvent3.get(12), "col13".toUpperCase(), "timestamp with time zone(6)", - "TIMESTAMP", + "TIMESTAMP_TZ", null, 6, false, diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-tidb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/tidb/source/converter/DefaultDataConverter.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-tidb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/tidb/source/converter/DefaultDataConverter.java index f17c6c4a341f..f71437a903a3 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-tidb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/tidb/source/converter/DefaultDataConverter.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-tidb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/tidb/source/converter/DefaultDataConverter.java @@ -37,6 +37,8 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; @Slf4j public class DefaultDataConverter implements DataConverter { @@ -101,6 +103,11 @@ public SeaTunnelRow convert(Object[] values, TiTableInfo tableInfo, SeaTunnelRow case TIMESTAMP: fields[fieldIndex] = convertToTimestamp(value, dataType); break; + case TIMESTAMP_TZ: + // TiDB TIMESTAMP is stored as UTC (LTZ). + // Convert to OffsetDateTime with UTC offset to preserve timezone semantics. + fields[fieldIndex] = convertToOffsetDateTime(value, dataType); + break; case BYTES: fields[fieldIndex] = convertToBinary(value); break; @@ -259,6 +266,45 @@ private static Object convertToTime(Object value) { return TemporalConversions.toLocalTime(value); } + private static Object convertToOffsetDateTime( + Object value, org.tikv.common.types.DataType dataType) { + if (value instanceof Timestamp) { + // TiDB TIMESTAMP is stored in UTC; convert to OffsetDateTime with UTC zone. + Instant instant = ((Timestamp) value).toInstant(); + return OffsetDateTime.ofInstant(instant, ZoneOffset.UTC); + } + if (value instanceof Long) { + // TiDB may emit TIMESTAMP as epoch milliseconds in some snapshot modes. + return OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) value), ZoneOffset.UTC); + } + if (value instanceof LocalDateTime) { + // LocalDateTime without explicit zone — treat as UTC wall-clock value. + return ((LocalDateTime) value).atOffset(ZoneOffset.UTC); + } + if (value instanceof String) { + // String representation from TiDB CDC — attempt ISO-8601 parse. + try { + return OffsetDateTime.parse((String) value); + } catch (java.time.format.DateTimeParseException e) { + throw new IllegalArgumentException( + "Unable to convert TIMESTAMP_TZ from String value: '" + + value + + "' for TiDB dataType: " + + dataType, + e); + } + } + // Unknown type — fail fast with enough context for diagnosis instead of silently + // returning the raw value which would cause a ClassCastException downstream. + throw new IllegalArgumentException( + "Unsupported value type for TIMESTAMP_TZ conversion: " + + value.getClass().getName() + + ", value='" + + value + + "', TiDB dataType=" + + dataType); + } + private static Object convertToTimestamp( Object value, org.tikv.common.types.DataType dataType) { switch (dataType.getType()) { diff --git a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/AbstractDorisTypeConverter.java b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/AbstractDorisTypeConverter.java index df057b38c430..20ab1a517bd8 100644 --- a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/AbstractDorisTypeConverter.java +++ b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/AbstractDorisTypeConverter.java @@ -353,6 +353,12 @@ protected BasicTypeDefine sampleReconvert( builder.columnType(String.format("%s(%s)", DORIS_VARCHAR, 8)); builder.dataType(DORIS_VARCHAR); break; + case TIMESTAMP_TZ: + // Doris has no timezone-aware datetime type; store as DATETIME (wall-clock value) + builder.columnType(String.format("%s(%s)", DORIS_DATETIME, MAX_DATETIME_SCALE)); + builder.dataType(DORIS_DATETIME); + builder.scale(MAX_DATETIME_SCALE); + break; case ARRAY: SeaTunnelDataType dataType = column.getDataType(); SeaTunnelDataType elementType = null; @@ -426,6 +432,7 @@ private void reconvertBuildArrayInternal( builder.dataType(DORIS_DATEV2_ARRAY); break; case TIMESTAMP: + case TIMESTAMP_TZ: builder.columnType(DORIS_DATETIMEV2_ARRAY); builder.dataType(DORIS_DATETIMEV2_ARRAY); break; diff --git a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/DorisTypeConverterV1.java b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/DorisTypeConverterV1.java index a3c4684f7da1..d74362676aaf 100644 --- a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/DorisTypeConverterV1.java +++ b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/DorisTypeConverterV1.java @@ -103,6 +103,9 @@ public BasicTypeDefine reconvert(Column column) { builder.dataType(DORIS_DATEV2); break; case TIMESTAMP: + case TIMESTAMP_TZ: + // Doris has no timezone-aware datetime type; TIMESTAMP_TZ is stored as DATETIMEV2 + // (wall-clock value, timezone offset is lost). if (column.getScale() != null && column.getScale() > 0 && column.getScale() <= MAX_DATETIME_SCALE) { diff --git a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/DorisTypeConverterV2.java b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/DorisTypeConverterV2.java index feef2c3956f8..84c82465ad56 100644 --- a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/DorisTypeConverterV2.java +++ b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/datatype/DorisTypeConverterV2.java @@ -227,6 +227,9 @@ public BasicTypeDefine reconvert(Column column) { builder.dataType(DORIS_DATE); break; case TIMESTAMP: + case TIMESTAMP_TZ: + // Doris has no timezone-aware datetime type; TIMESTAMP_TZ is stored as DATETIME + // (wall-clock value, timezone offset is lost). if (column.getScale() != null && column.getScale() >= 0 && column.getScale() <= MAX_DATETIME_SCALE) { diff --git a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/serialize/SeaTunnelRowSerializer.java b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/serialize/SeaTunnelRowSerializer.java index 023724ada09f..e0f8559e3d0b 100644 --- a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/serialize/SeaTunnelRowSerializer.java +++ b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/serialize/SeaTunnelRowSerializer.java @@ -88,16 +88,18 @@ public SeaTunnelRowSerializer( if (JSON.equals(type)) { JsonSerializationSchema jsonSerializationSchema = - new JsonSerializationSchema(this.seaTunnelRowType); + new JsonSerializationSchema(this.seaTunnelRowType, true); ObjectMapper mapper = jsonSerializationSchema.getMapper(); mapper.configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true); this.serialize = jsonSerializationSchema; } else { + // Doris DATETIME has no native timezone support: serialize TIMESTAMP_TZ as wall-clock. this.serialize = TextSerializationSchema.builder() .seaTunnelRowType(this.seaTunnelRowType) .delimiter(fieldDelimiter) .nullValue(NULL_VALUE) + .wallClockTimestampTz(true) .build(); } } diff --git a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/AvroSchemaConverter.java b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/AvroSchemaConverter.java index 2f9ac4f2a751..f47a10366d1c 100644 --- a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/AvroSchemaConverter.java +++ b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/AvroSchemaConverter.java @@ -94,7 +94,10 @@ private static Schema convertToSchema( Schema binary = SchemaBuilder.builder().bytesType(); return nullableSchema(binary); case TIMESTAMP: - // use long to represents Timestamp + case TIMESTAMP_TZ: + // use long to represents Timestamp / Timestamp with timezone + // TIMESTAMP_TZ (OffsetDateTime/LTZ) is stored as timestampMillis (UTC epoch) + // same as TIMESTAMP, as Avro/Hudi does not have a native timezone-aware type LogicalType avroLogicalType; avroLogicalType = LogicalTypes.timestampMillis(); Schema timestamp = avroLogicalType.addToSchema(SchemaBuilder.builder().longType()); diff --git a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/RowDataToAvroConverters.java b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/RowDataToAvroConverters.java index 5c0636266934..930fcf39ed6a 100644 --- a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/RowDataToAvroConverters.java +++ b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/RowDataToAvroConverters.java @@ -162,6 +162,19 @@ public Object convert(Schema schema, Object object) { } }; break; + case TIMESTAMP_TZ: + converter = + new RowDataToAvroConverter() { + private static final long serialVersionUID = 1L; + + @Override + public Object convert(Schema schema, Object object) { + return ((java.time.OffsetDateTime) object) + .toInstant() + .toEpochMilli(); + } + }; + break; case DECIMAL: converter = new RowDataToAvroConverter() { diff --git a/seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/DefaultDeserializer.java b/seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/DefaultDeserializer.java index 4243c794656f..2899c4acb4e1 100644 --- a/seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/DefaultDeserializer.java +++ b/seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/DefaultDeserializer.java @@ -90,8 +90,10 @@ private Object convert( case TIMESTAMP: Types.TimestampType timestampType = (Types.TimestampType) icebergType; if (timestampType.shouldAdjustToUTC()) { - return OffsetDateTime.class.cast(icebergValue).toLocalDateTime(); + // withZone() → LTZ → return OffsetDateTime to preserve timezone info + return OffsetDateTime.class.cast(icebergValue); } + // withoutZone() → NTZ → return LocalDateTime as-is return LocalDateTime.class.cast(icebergValue); case STRING: return String.class.cast(icebergValue); diff --git a/seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapper.java b/seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapper.java index 20b8ba3adcaf..6bf9f90367a4 100644 --- a/seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapper.java +++ b/seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapper.java @@ -55,6 +55,12 @@ public static SeaTunnelDataType mapping(String field, @NonNull Type icebergTy case TIME: return LocalTimeType.LOCAL_TIME_TYPE; case TIMESTAMP: + Types.TimestampType timestampType = (Types.TimestampType) icebergType; + if (timestampType.shouldAdjustToUTC()) { + // withZone() → LTZ → TIMESTAMP_TZ + return LocalTimeType.OFFSET_DATE_TIME_TYPE; + } + // withoutZone() → NTZ → TIMESTAMP return LocalTimeType.LOCAL_DATE_TIME_TYPE; case STRING: return BasicType.STRING_TYPE; @@ -173,6 +179,10 @@ public static Type toIcebergType(SeaTunnelDataType dataType, AtomicInteger nextI case TIME: return Types.TimeType.get(); case TIMESTAMP: + // NTZ → Iceberg withoutZone() + return Types.TimestampType.withoutZone(); + case TIMESTAMP_TZ: + // LTZ → Iceberg withZone() return Types.TimestampType.withZone(); case STRING: default: diff --git a/seatunnel-connectors-v2/connector-iceberg/src/test/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapperTest.java b/seatunnel-connectors-v2/connector-iceberg/src/test/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapperTest.java index baf9b9441fc7..5761af5a8790 100644 --- a/seatunnel-connectors-v2/connector-iceberg/src/test/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapperTest.java +++ b/seatunnel-connectors-v2/connector-iceberg/src/test/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapperTest.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.LocalTimeType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.iceberg.types.Type; @@ -80,4 +81,78 @@ void throwsExceptionWhenSinkTypeIsInvalid() { IcebergTypeMapper.toIcebergType(column, new AtomicInteger(1)); }); } + + @Test + void timestampNtzMapsToWithoutZone() { + // TIMESTAMP (NTZ) → Iceberg withoutZone() + Column column = mock(Column.class); + when(column.getSinkType()).thenReturn(null); + when(column.getDataType()) + .thenReturn((SeaTunnelDataType) LocalTimeType.LOCAL_DATE_TIME_TYPE); + + Type result = IcebergTypeMapper.toIcebergType(column, new AtomicInteger(1)); + assertEquals(Types.TimestampType.withoutZone(), result); + } + + @Test + void timestampTzMapsToWithZone() { + // TIMESTAMP_TZ (LTZ) → Iceberg withZone() + Column column = mock(Column.class); + when(column.getSinkType()).thenReturn(null); + when(column.getDataType()) + .thenReturn((SeaTunnelDataType) LocalTimeType.OFFSET_DATE_TIME_TYPE); + + Type result = IcebergTypeMapper.toIcebergType(column, new AtomicInteger(1)); + assertEquals(Types.TimestampType.withZone(), result); + } + + @Test + void icebergWithoutZoneMapsToLocalDateTimeType() { + // Iceberg withoutZone() → SeaTunnel LOCAL_DATE_TIME_TYPE (TIMESTAMP / NTZ) + SeaTunnelDataType result = + IcebergTypeMapper.mapping("ts_ntz", Types.TimestampType.withoutZone()); + assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, result); + } + + @Test + void icebergWithZoneMapsToOffsetDateTimeType() { + // Iceberg withZone() → SeaTunnel OFFSET_DATE_TIME_TYPE (TIMESTAMP_TZ / LTZ) + SeaTunnelDataType result = + IcebergTypeMapper.mapping("ts_ltz", Types.TimestampType.withZone()); + assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, result); + } + + /** + * Upgrade-path regression: old SeaTunnel (before SEATUNNEL-10685) incorrectly wrote SeaTunnel + * TIMESTAMP columns to Iceberg as {@code withZone()}. After the fix, reading such a legacy + * column back must return {@code TIMESTAMP_TZ} (OFFSET_DATE_TIME_TYPE), not TIMESTAMP + * (LOCAL_DATE_TIME_TYPE). + * + *

This is a documented breaking change — see incompatible-changes.md. + */ + @Test + void upgradePath_legacyWithZoneColumnIsReadAsTimestampTz() { + SeaTunnelDataType result = + IcebergTypeMapper.mapping("legacy_ts_col", Types.TimestampType.withZone()); + assertEquals( + LocalTimeType.OFFSET_DATE_TIME_TYPE, + result, + "Existing Iceberg withZone() column (written by old SeaTunnel as TIMESTAMP) " + + "must be read as TIMESTAMP_TZ after upgrade"); + } + + /** + * Upgrade-path regression: new SeaTunnel writes SeaTunnel TIMESTAMP columns to Iceberg as + * {@code withoutZone()} (NTZ). Reading them back must return TIMESTAMP (LOCAL_DATE_TIME_TYPE). + */ + @Test + void upgradePath_newTimestampWrittenAsWithoutZoneIsReadAsTimestamp() { + SeaTunnelDataType result = + IcebergTypeMapper.mapping("new_ts_col", Types.TimestampType.withoutZone()); + assertEquals( + LocalTimeType.LOCAL_DATE_TIME_TYPE, + result, + "New Iceberg withoutZone() column (written by new SeaTunnel as TIMESTAMP) " + + "must be read as TIMESTAMP (NTZ)"); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/snowflake/SnowflakeDataTypeConvertor.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/snowflake/SnowflakeDataTypeConvertor.java index 52dd03cef0df..dd8e89886f38 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/snowflake/SnowflakeDataTypeConvertor.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/snowflake/SnowflakeDataTypeConvertor.java @@ -143,10 +143,11 @@ public SeaTunnelDataType toSeaTunnelType( return LocalTimeType.LOCAL_TIME_TYPE; case SNOWFLAKE_DATE_TIME: case SNOWFLAKE_TIMESTAMP: - case SNOWFLAKE_TIMESTAMP_LTZ: case SNOWFLAKE_TIMESTAMP_NTZ: - case SNOWFLAKE_TIMESTAMP_TZ: return LocalTimeType.LOCAL_DATE_TIME_TYPE; + case SNOWFLAKE_TIMESTAMP_LTZ: + case SNOWFLAKE_TIMESTAMP_TZ: + return LocalTimeType.OFFSET_DATE_TIME_TYPE; default: throw CommonError.convertToSeaTunnelTypeError( DatabaseIdentifier.SNOWFLAKE, connectorDataType, field); @@ -186,7 +187,9 @@ public String toConnectorType( case TIME: return SNOWFLAKE_TIME; case TIMESTAMP: - return SNOWFLAKE_TIMESTAMP; + return SNOWFLAKE_TIMESTAMP_NTZ; + case TIMESTAMP_TZ: + return SNOWFLAKE_TIMESTAMP_TZ; default: throw CommonError.convertToSeaTunnelTypeError( DatabaseIdentifier.SNOWFLAKE, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/converter/AbstractJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/converter/AbstractJdbcRowConverter.java index 882288524cd1..c2a3fb04abba 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/converter/AbstractJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/converter/AbstractJdbcRowConverter.java @@ -114,11 +114,9 @@ public SeaTunnelRow toInternal(ResultSet rs, TableSchema tableSchema) throws SQL fields[fieldIndex] = readTime(rs, resultSetIndex); break; case TIMESTAMP: - Timestamp sqlTimestamp = JdbcFieldTypeUtils.getTimestamp(rs, resultSetIndex); - fields[fieldIndex] = - Optional.ofNullable(sqlTimestamp) - .map(e -> e.toLocalDateTime()) - .orElse(null); + // Use getLocalDateTime() which avoids JVM-default-timezone influence. + // See JdbcFieldTypeUtils.getLocalDateTime() for full strategy details. + fields[fieldIndex] = JdbcFieldTypeUtils.getLocalDateTime(rs, resultSetIndex); break; case TIMESTAMP_TZ: OffsetDateTime offsetDateTime = @@ -311,12 +309,19 @@ protected void setValueToStatementByDataType( case TIMESTAMP_TZ: OffsetDateTime offsetDateTime = (OffsetDateTime) value; try { - // Try to use setObject first for better timezone support + // Try to use setObject first for better timezone support. + // Modern Oracle JDBC (12.2+) and most other drivers accept OffsetDateTime + // directly and preserve the offset accurately. statement.setObject(statementIndex, offsetDateTime); } catch (SQLException e) { - // Fallback to setTimestamp if setObject is not supported + // Fallback for older drivers that do not support OffsetDateTime via setObject. + // Pass an explicit UTC Calendar so the driver does not apply the JVM default + // timezone when interpreting the Timestamp, which would silently corrupt the + // stored instant in any non-UTC environment. statement.setTimestamp( - statementIndex, Timestamp.from(offsetDateTime.toInstant())); + statementIndex, + Timestamp.from(offsetDateTime.toInstant()), + java.util.Calendar.getInstance(java.util.TimeZone.getTimeZone("UTC"))); } break; case BYTES: diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbTypeConverter.java index 1ed2c3a8b8d5..8ea3de9b6f45 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbTypeConverter.java @@ -311,7 +311,7 @@ public Column convert(BasicTypeDefine typeDefine) { builder.sourceType( String.format("DATETIME(%s) WITH TIME ZONE", typeDefine.getScale())); } - builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); builder.scale(typeDefine.getScale()); break; default: @@ -491,6 +491,28 @@ public BasicTypeDefine reconvert(Column column) { builder.columnType(DM_TIMESTAMP); } break; + case TIMESTAMP_TZ: + builder.dataType(DM_DATETIME_WITH_TIME_ZONE); + if (column.getScale() != null && column.getScale() > 0) { + Integer timestampTzScale = column.getScale(); + if (timestampTzScale > MAX_TIMESTAMP_SCALE) { + timestampTzScale = MAX_TIMESTAMP_SCALE; + log.warn( + "The timestamp_tz column {} type datetime_tz({}) is out of range, " + + "which exceeds the maximum scale of {}, " + + "it will be converted to datetime_tz({})", + column.getName(), + column.getScale(), + MAX_TIMESTAMP_SCALE, + timestampTzScale); + } + builder.columnType( + String.format("DATETIME(%s) WITH TIME ZONE", timestampTzScale)); + builder.scale(timestampTzScale); + } else { + builder.columnType(DM_DATETIME_WITH_TIME_ZONE); + } + break; default: throw CommonError.convertToConnectorTypeError( DatabaseIdentifier.DAMENG, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/duckdb/DuckDBTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/duckdb/DuckDBTypeConverter.java index 6debaf90f3fa..88dddab00518 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/duckdb/DuckDBTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/duckdb/DuckDBTypeConverter.java @@ -164,9 +164,11 @@ public Column convert(BasicTypeDefine typeDefine) { builder.dataType(LocalTimeType.LOCAL_TIME_TYPE); break; case DUCKDB_TIMESTAMP: - case DUCKDB_TIMESTAMP_WITH_TZ: builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); break; + case DUCKDB_TIMESTAMP_WITH_TZ: + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); + break; case DUCKDB_INTERVAL: builder.dataType(BasicType.STRING_TYPE); builder.columnLength(50L); @@ -281,6 +283,10 @@ public BasicTypeDefine reconvert(Column column) { builder.columnType(DUCKDB_TIMESTAMP); builder.dataType(DUCKDB_TIMESTAMP); break; + case TIMESTAMP_TZ: + builder.columnType(DUCKDB_TIMESTAMP_WITH_TZ); + builder.dataType(DUCKDB_TIMESTAMP_WITH_TZ); + break; case BYTES: builder.columnType(DUCKDB_BLOB); builder.dataType(DUCKDB_BLOB); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/inceptor/InceptorJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/inceptor/InceptorJdbcRowConverter.java index 71180d472685..e9ec7a03017f 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/inceptor/InceptorJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/inceptor/InceptorJdbcRowConverter.java @@ -40,6 +40,8 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.OffsetDateTime; +import java.util.Calendar; +import java.util.TimeZone; public class InceptorJdbcRowConverter extends HiveJdbcRowConverter { @@ -107,8 +109,12 @@ public PreparedStatement toExternal( break; case TIMESTAMP_TZ: OffsetDateTime offsetDateTime = (OffsetDateTime) row.getField(fieldIndex); + // Inceptor (Hive-based) has no native timezone type; convert to UTC epoch + // and pass Calendar.UTC so the driver does not apply the session timezone. statement.setTimestamp( - statementIndex, Timestamp.from(offsetDateTime.toInstant())); + statementIndex, + Timestamp.from(offsetDateTime.toInstant()), + Calendar.getInstance(TimeZone.getTimeZone("UTC"))); break; case BYTES: statement.setBytes(statementIndex, (byte[]) row.getField(fieldIndex)); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseJdbcRowConverter.java index db5d23aae907..66a3e23c1480 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseJdbcRowConverter.java @@ -102,6 +102,9 @@ public SeaTunnelRow toInternal(ResultSet rs, TableSchema tableSchema) throws SQL .map(Timestamp::toLocalDateTime) .orElse(null); break; + case TIMESTAMP_TZ: + fields[fieldIndex] = JdbcFieldTypeUtils.getOffsetDateTime(rs, resultSetIndex); + break; case BYTES: fields[fieldIndex] = JdbcFieldTypeUtils.getBytes(rs, resultSetIndex); break; @@ -179,8 +182,15 @@ public PreparedStatement toExternal( break; case TIMESTAMP_TZ: OffsetDateTime offsetDateTime = (OffsetDateTime) row.getField(fieldIndex); - statement.setTimestamp( - statementIndex, Timestamp.from(offsetDateTime.toInstant())); + try { + statement.setObject(statementIndex, offsetDateTime); + } catch (SQLException e) { + statement.setTimestamp( + statementIndex, + Timestamp.from(offsetDateTime.toInstant()), + java.util.Calendar.getInstance( + java.util.TimeZone.getTimeZone("UTC"))); + } break; case BYTES: statement.setBytes(statementIndex, (byte[]) row.getField(fieldIndex)); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseTypeConverter.java index f46fccda931d..abf624fe8b86 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseTypeConverter.java @@ -79,7 +79,7 @@ public Column convert(BasicTypeDefine typeDefine) { case MySqlTypeConverter.MYSQL_YEAR_UNSIGNED: builder.dataType(BasicType.INT_TYPE); break; - // DATETIME not in PG (PG has TIMESTAMP) + // DATETIME not in PG (PG has TIMESTAMP) — NTZ case MySqlTypeConverter.MYSQL_DATETIME: builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); if (typeDefine.getScale() != null @@ -155,10 +155,14 @@ public Column convert(BasicTypeDefine typeDefine) { builder.columnLength((long) (1024 * 1024 * 1024)); } break; - // SQLServer compatibility - SQLServer specific types + // MySQL TIMESTAMP — LTZ (timezone-aware) + case MySqlTypeConverter.MYSQL_TIMESTAMP: + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); + builder.scale(typeDefine.getScale()); + break; + // SQLServer compatibility - NTZ types case SqlServerTypeConverter.SQLSERVER_DATETIME2: case SqlServerTypeConverter.SQLSERVER_SMALLDATETIME: - case SqlServerTypeConverter.SQLSERVER_DATETIMEOFFSET: builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); if (typeDefine.getScale() != null && typeDefine.getScale() > MAX_TIMESTAMP_SCALE) { @@ -175,6 +179,11 @@ public Column convert(BasicTypeDefine typeDefine) { builder.scale(typeDefine.getScale()); } break; + // SQLServer DATETIMEOFFSET — LTZ (timezone-aware) + case SqlServerTypeConverter.SQLSERVER_DATETIMEOFFSET: + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); + builder.scale(typeDefine.getScale()); + break; case KB_TINYINT: builder.dataType(BasicType.BYTE_TYPE); break; diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlTypeConverter.java index 4076a2303e6a..f1344cb58011 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlTypeConverter.java @@ -317,10 +317,14 @@ public Column convert(BasicTypeDefine typeDefine) { builder.scale(typeDefine.getScale()); break; case MYSQL_DATETIME: - case MYSQL_TIMESTAMP: builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); builder.scale(typeDefine.getScale()); break; + case MYSQL_TIMESTAMP: + // MySQL TIMESTAMP is LTZ (stored as UTC, displayed in session timezone) + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); + builder.scale(typeDefine.getScale()); + break; default: throw CommonError.convertToSeaTunnelTypeError( DatabaseIdentifier.MYSQL, mysqlDataType, typeDefine.getName()); @@ -537,6 +541,31 @@ public BasicTypeDefine reconvert(Column column) { builder.columnType(MYSQL_DATETIME); } break; + case TIMESTAMP_TZ: + // TIMESTAMP_TZ (LTZ) maps back to MySQL TIMESTAMP + builder.nativeType(MysqlType.TIMESTAMP); + builder.dataType(MYSQL_TIMESTAMP); + if (version.isAtOrBefore(MySqlVersion.V_5_5)) { + builder.columnType(MYSQL_TIMESTAMP); + } else if (column.getScale() != null && column.getScale() > 0) { + int timestampTzScale = column.getScale(); + if (timestampTzScale > MAX_TIMESTAMP_SCALE) { + timestampTzScale = MAX_TIMESTAMP_SCALE; + log.warn( + "The timestamp_tz column {} type timestamp({}) is out of range, " + + "which exceeds the maximum scale of {}, " + + "it will be converted to timestamp({})", + column.getName(), + column.getScale(), + MAX_TIMESTAMP_SCALE, + timestampTzScale); + } + builder.columnType(String.format("%s(%s)", MYSQL_TIMESTAMP, timestampTzScale)); + builder.scale(timestampTzScale); + } else { + builder.columnType(MYSQL_TIMESTAMP); + } + break; default: throw CommonError.convertToConnectorTypeError( DatabaseIdentifier.MYSQL, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlTypeConverter.java index e3463fc914d9..ca426551b4c1 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlTypeConverter.java @@ -291,10 +291,13 @@ public Column convert(BasicTypeDefine typeDefine) { builder.scale(typeDefine.getScale()); break; case MYSQL_DATETIME: - case MYSQL_TIMESTAMP: builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); builder.scale(typeDefine.getScale()); break; + case MYSQL_TIMESTAMP: + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); + builder.scale(typeDefine.getScale()); + break; case VECTOR_NAME: String columnType = typeDefine.getColumnType().toUpperCase(); if (columnType.startsWith("VECTOR(") && columnType.endsWith(")")) { @@ -518,6 +521,28 @@ public BasicTypeDefine reconvert(Column column) { builder.columnType(MYSQL_DATETIME); } break; + case TIMESTAMP_TZ: + builder.nativeType(OceanBaseMysqlType.TIMESTAMP); + builder.dataType(MYSQL_TIMESTAMP); + if (column.getScale() != null && column.getScale() > 0) { + int timestampTzScale = column.getScale(); + if (timestampTzScale > MAX_TIMESTAMP_SCALE) { + timestampTzScale = MAX_TIMESTAMP_SCALE; + log.warn( + "The timestamp_tz column {} type timestamp({}) is out of range, " + + "which exceeds the maximum scale of {}, " + + "it will be converted to timestamp({})", + column.getName(), + column.getScale(), + MAX_TIMESTAMP_SCALE, + timestampTzScale); + } + builder.columnType(String.format("%s(%s)", MYSQL_TIMESTAMP, timestampTzScale)); + builder.scale(timestampTzScale); + } else { + builder.columnType(MYSQL_TIMESTAMP); + } + break; case FLOAT_VECTOR: builder.nativeType(VECTOR_NAME); builder.columnType(String.format("%s(%s)", VECTOR_NAME, column.getScale())); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMysqlJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMysqlJdbcRowConverter.java index e51211e8df7f..ff114fcc2288 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMysqlJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMysqlJdbcRowConverter.java @@ -47,7 +47,9 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.OffsetDateTime; +import java.util.Calendar; import java.util.Optional; +import java.util.TimeZone; public class OceanBaseMysqlJdbcRowConverter extends AbstractJdbcRowConverter { @Override @@ -126,6 +128,10 @@ public SeaTunnelRow toInternal(ResultSet rs, TableSchema tableSchema) throws SQL .map(e -> e.toLocalDateTime()) .orElse(null); break; + case TIMESTAMP_TZ: + // OceanBase MySQL TIMESTAMP (LTZ) → read as OffsetDateTime + fields[fieldIndex] = JdbcFieldTypeUtils.getOffsetDateTime(rs, resultSetIndex); + break; case BYTES: fields[fieldIndex] = JdbcFieldTypeUtils.getBytes(rs, resultSetIndex); break; @@ -223,8 +229,12 @@ public PreparedStatement toExternal( break; case TIMESTAMP_TZ: OffsetDateTime offsetDateTime = (OffsetDateTime) row.getField(fieldIndex); + // OceanBase MySQL has no native timezone-aware type; convert to UTC epoch + // and pass Calendar.UTC so the driver does not apply the session timezone. statement.setTimestamp( - statementIndex, Timestamp.from(offsetDateTime.toInstant())); + statementIndex, + Timestamp.from(offsetDateTime.toInstant()), + Calendar.getInstance(TimeZone.getTimeZone("UTC"))); break; case BYTES: statement.setBytes(statementIndex, (byte[]) row.getField(fieldIndex)); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleJdbcRowConverter.java index 8d91e311083f..c0a3d8865335 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleJdbcRowConverter.java @@ -66,6 +66,12 @@ protected void setValueToStatementByDataType( } else { statement.setString(statementIndex, (String) value); } + } else if (seaTunnelDataType.getSqlType().equals(SqlType.TIMESTAMP_TZ)) { + // Delegate to AbstractJdbcRowConverter which uses setObject() first (preserving the + // offset for Oracle JDBC 12.2+) and falls back to setTimestamp() with an explicit UTC + // Calendar for older drivers, avoiding JVM-default-timezone corruption. + super.setValueToStatementByDataType( + value, statement, seaTunnelDataType, statementIndex, sourceType); } else { super.setValueToStatementByDataType( value, statement, seaTunnelDataType, statementIndex, sourceType); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleTypeConverter.java index 6d2b56992161..2b91fdd48449 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleTypeConverter.java @@ -240,9 +240,18 @@ public Column convert(BasicTypeDefine typeDefine) { builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); break; case ORACLE_TIMESTAMP: + // TIMESTAMP without timezone is NTZ + builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); + if (typeDefine.getScale() == null) { + builder.scale(TIMESTAMP_DEFAULT_SCALE); + } else { + builder.scale(typeDefine.getScale()); + } + break; case ORACLE_TIMESTAMP_WITH_TIME_ZONE: case ORACLE_TIMESTAMP_WITH_LOCAL_TIME_ZONE: - builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); + // TIMESTAMP WITH (LOCAL) TIME ZONE is LTZ + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); if (typeDefine.getScale() == null) { builder.scale(TIMESTAMP_DEFAULT_SCALE); } else { @@ -376,25 +385,48 @@ public BasicTypeDefine reconvert(Column column) { builder.dataType(ORACLE_DATE); break; case TIMESTAMP: + // NTZ: maps to ORACLE_TIMESTAMP (without timezone) if (column.getScale() == null || column.getScale() <= 0) { - builder.columnType(ORACLE_TIMESTAMP_WITH_LOCAL_TIME_ZONE); + builder.columnType(ORACLE_TIMESTAMP); } else { int timestampScale = column.getScale(); - if (column.getScale() > MAX_TIMESTAMP_SCALE) { - timestampScale = MAX_TIMESTAMP_SCALE; + if (timestampScale > MAX_TIMESTAMP_SCALE) { log.warn( "The timestamp column {} type timestamp({}) is out of range, " + "which exceeds the maximum scale of {}, " + "it will be converted to timestamp({})", column.getName(), - column.getScale(), + timestampScale, MAX_TIMESTAMP_SCALE, - timestampScale); + MAX_TIMESTAMP_SCALE); + timestampScale = MAX_TIMESTAMP_SCALE; } - builder.columnType( - String.format("TIMESTAMP(%s) WITH LOCAL TIME ZONE", timestampScale)); + builder.columnType(String.format("TIMESTAMP(%s)", timestampScale)); builder.scale(timestampScale); } + builder.dataType(ORACLE_TIMESTAMP); + break; + case TIMESTAMP_TZ: + // LTZ: maps to ORACLE_TIMESTAMP_WITH_LOCAL_TIME_ZONE + if (column.getScale() == null || column.getScale() <= 0) { + builder.columnType(ORACLE_TIMESTAMP_WITH_LOCAL_TIME_ZONE); + } else { + int timestampTzScale = column.getScale(); + if (timestampTzScale > MAX_TIMESTAMP_SCALE) { + log.warn( + "The timestamp_tz column {} type timestamp({}) is out of range, " + + "which exceeds the maximum scale of {}, " + + "it will be converted to timestamp({})", + column.getName(), + timestampTzScale, + MAX_TIMESTAMP_SCALE, + MAX_TIMESTAMP_SCALE); + timestampTzScale = MAX_TIMESTAMP_SCALE; + } + builder.columnType( + String.format("TIMESTAMP(%s) WITH LOCAL TIME ZONE", timestampTzScale)); + builder.scale(timestampTzScale); + } builder.dataType(ORACLE_TIMESTAMP_WITH_LOCAL_TIME_ZONE); break; default: diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverter.java index a19c4814f1ed..ffe1ffec9674 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresJdbcRowConverter.java @@ -150,11 +150,9 @@ public SeaTunnelRow toInternal(ResultSet rs, TableSchema tableSchema) throws SQL Optional.ofNullable(sqlTime).map(e -> e.toLocalTime()).orElse(null); break; case TIMESTAMP: - Timestamp sqlTimestamp = JdbcFieldTypeUtils.getTimestamp(rs, resultSetIndex); - fields[fieldIndex] = - Optional.ofNullable(sqlTimestamp) - .map(e -> e.toLocalDateTime()) - .orElse(null); + // Use getLocalDateTime() which avoids JVM-default-timezone influence. + // See JdbcFieldTypeUtils.getLocalDateTime() for full strategy details. + fields[fieldIndex] = JdbcFieldTypeUtils.getLocalDateTime(rs, resultSetIndex); break; case TIMESTAMP_TZ: // Enhanced PostgreSQL TIMESTAMP_TZ handling @@ -390,9 +388,6 @@ private OffsetDateTime getPostgresOffsetDateTime(ResultSet rs, int columnIndex) if (obj instanceof OffsetDateTime) { return (OffsetDateTime) obj; } - if (obj instanceof Timestamp) { - return ((Timestamp) obj).toInstant().atOffset(ZoneOffset.UTC); - } if (obj instanceof java.time.ZonedDateTime) { return ((java.time.ZonedDateTime) obj).toOffsetDateTime(); } @@ -400,6 +395,22 @@ private OffsetDateTime getPostgresOffsetDateTime(ResultSet rs, int columnIndex) return ((java.util.Date) obj).toInstant().atOffset(ZoneOffset.UTC); } + // Handle java.sql.Timestamp: avoid using toInstant() directly because the Timestamp + // was constructed with JVM-default-timezone semantics, which would shift the value. + // Instead, re-read as string and parse the timezone info explicitly. + if (obj instanceof Timestamp) { + String strVal = rs.getString(columnIndex); + if (strVal == null) { + return null; + } + try { + return JdbcFieldTypeUtils.parseOffsetDateTimeFromString(strVal); + } catch (Exception e) { + // Last resort: fall back to instant-based conversion + return ((Timestamp) obj).toInstant().atOffset(ZoneOffset.UTC); + } + } + // Remaining PostgreSQL-specific or driver types: fall back to string representation return parseTimestampFromObjectString(obj); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftTypeConverter.java index f48d0c77cb4d..f82291198cef 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftTypeConverter.java @@ -177,7 +177,7 @@ public Column convert(BasicTypeDefine typeDefine) { break; case REDSHIFT_TIMESTAMPTZ: builder.sourceType(REDSHIFT_TIMESTAMPTZ); - builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); builder.scale(MAX_TIMESTAMP_SCALE); break; default: @@ -373,6 +373,23 @@ public BasicTypeDefine reconvert(Column column) { builder.dataType(REDSHIFT_TIMESTAMP); builder.scale(timestampScale); break; + case TIMESTAMP_TZ: + Integer timestampTzScale = column.getScale(); + if (timestampTzScale != null && timestampTzScale > MAX_TIMESTAMP_SCALE) { + timestampTzScale = MAX_TIMESTAMP_SCALE; + log.warn( + "The timestamp_tz column {} type timestamptz({}) is out of range, " + + "which exceeds the maximum scale of {}, " + + "it will be converted to timestamptz({})", + column.getName(), + column.getScale(), + MAX_TIMESTAMP_SCALE, + timestampTzScale); + } + builder.columnType(REDSHIFT_TIMESTAMPTZ); + builder.dataType(REDSHIFT_TIMESTAMPTZ); + builder.scale(timestampTzScale); + break; case MAP: case ARRAY: case ROW: diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/snowflake/SnowflakeTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/snowflake/SnowflakeTypeConverter.java index c9e4adcadd29..ac0a74624771 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/snowflake/SnowflakeTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/snowflake/SnowflakeTypeConverter.java @@ -171,12 +171,15 @@ public Column convert(BasicTypeDefine typeDefine) { break; case SNOWFLAKE_DATE_TIME: case SNOWFLAKE_TIMESTAMP: - case SNOWFLAKE_TIMESTAMP_LTZ: case SNOWFLAKE_TIMESTAMP_NTZ: - case SNOWFLAKE_TIMESTAMP_TZ: builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); builder.scale(9); break; + case SNOWFLAKE_TIMESTAMP_LTZ: + case SNOWFLAKE_TIMESTAMP_TZ: + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); + builder.scale(9); + break; default: throw CommonError.convertToSeaTunnelTypeError( DatabaseIdentifier.SNOWFLAKE, dataType, typeDefine.getName()); @@ -306,7 +309,7 @@ public BasicTypeDefine reconvert(Column column) { builder.dataType(SNOWFLAKE_GEOMETRY); break; case TIME: - if (column.getScale() > 9) { + if (column.getScale() != null && column.getScale() > 9) { log.warn( "The timestamp column {} type time({}) is out of range, " + "which exceeds the maximum scale of {}, " @@ -320,7 +323,7 @@ public BasicTypeDefine reconvert(Column column) { builder.dataType(SNOWFLAKE_TIME); break; case TIMESTAMP: - if (column.getScale() > 9) { + if (column.getScale() != null && column.getScale() > 9) { log.warn( "The timestamp column {} type timestamp({}) is out of range, " + "which exceeds the maximum scale of {}, " @@ -330,8 +333,22 @@ public BasicTypeDefine reconvert(Column column) { 9, 9); } - builder.columnType(SNOWFLAKE_TIMESTAMP); - builder.dataType(SNOWFLAKE_TIMESTAMP); + builder.columnType(SNOWFLAKE_TIMESTAMP_NTZ); + builder.dataType(SNOWFLAKE_TIMESTAMP_NTZ); + break; + case TIMESTAMP_TZ: + if (column.getScale() != null && column.getScale() > 9) { + log.warn( + "The timestamp_tz column {} type timestamp_tz({}) is out of range, " + + "which exceeds the maximum scale of {}, " + + "it will be converted to timestamp_tz({})", + column.getName(), + column.getScale(), + 9, + 9); + } + builder.columnType(SNOWFLAKE_TIMESTAMP_TZ); + builder.dataType(SNOWFLAKE_TIMESTAMP_TZ); break; default: throw CommonError.convertToSeaTunnelTypeError( diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerTypeConverter.java index 59eb19cc4ad5..fd4e9d913408 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerTypeConverter.java @@ -290,9 +290,10 @@ public Column convert(BasicTypeDefine typeDefine) { builder.scale(typeDefine.getScale()); break; case SQLSERVER_DATETIMEOFFSET: + // DATETIMEOFFSET is LTZ (includes timezone offset) builder.sourceType( String.format("%s(%s)", SQLSERVER_DATETIMEOFFSET, typeDefine.getScale())); - builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); builder.scale(typeDefine.getScale()); break; case SQLSERVER_SMALLDATETIME: @@ -458,6 +459,7 @@ public BasicTypeDefine reconvert(Column column) { builder.dataType(SQLSERVER_TIME); break; case TIMESTAMP: + // NTZ: maps to DATETIME2 if (column.getScale() != null && column.getScale() > 0) { int timestampScale = column.getScale(); if (timestampScale > MAX_TIMESTAMP_SCALE) { @@ -479,6 +481,29 @@ public BasicTypeDefine reconvert(Column column) { } builder.dataType(SQLSERVER_DATETIME2); break; + case TIMESTAMP_TZ: + // LTZ: maps to DATETIMEOFFSET + if (column.getScale() != null && column.getScale() > 0) { + int timestampTzScale = column.getScale(); + if (timestampTzScale > MAX_TIMESTAMP_SCALE) { + timestampTzScale = MAX_TIMESTAMP_SCALE; + log.warn( + "The timestamp_tz column {} type datetimeoffset({}) is out of" + + " range, which exceeds the maximum scale of {}, " + + "it will be converted to datetimeoffset({})", + column.getName(), + column.getScale(), + MAX_TIMESTAMP_SCALE, + timestampTzScale); + } + builder.columnType( + String.format("%s(%s)", SQLSERVER_DATETIMEOFFSET, timestampTzScale)); + builder.scale(timestampTzScale); + } else { + builder.columnType(SQLSERVER_DATETIMEOFFSET); + } + builder.dataType(SQLSERVER_DATETIMEOFFSET); + break; default: throw CommonError.convertToConnectorTypeError( DatabaseIdentifier.SQLSERVER, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlserverJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlserverJdbcRowConverter.java index 28826e700c1e..304465f3cbcc 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlserverJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlserverJdbcRowConverter.java @@ -49,6 +49,24 @@ protected LocalTime readTime(ResultSet rs, int resultSetIndex) throws SQLExcepti .map(e -> e.toLocalDateTime().toLocalTime()) .orElse(null); } + + @Override + protected void setValueToStatementByDataType( + Object value, + PreparedStatement statement, + SeaTunnelDataType seaTunnelDataType, + int statementIndex, + @Nullable String sourceType) + throws SQLException { + if (seaTunnelDataType.getSqlType().equals(SqlType.TIMESTAMP_TZ)) { + // DATETIMEOFFSET supports OffsetDateTime directly via setObject + statement.setObject(statementIndex, (java.time.OffsetDateTime) value); + } else { + super.setValueToStatementByDataType( + value, statement, seaTunnelDataType, statementIndex, sourceType); + } + } + /** * {@inheritDoc} * diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguJdbcRowConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguJdbcRowConverter.java index 4590761965c8..f64b6e33a306 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguJdbcRowConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguJdbcRowConverter.java @@ -17,13 +17,55 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.xugu; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SqlType; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.AbstractJdbcRowConverter; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nullable; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.OffsetDateTime; + +@Slf4j public class XuguJdbcRowConverter extends AbstractJdbcRowConverter { + private static final java.util.concurrent.atomic.AtomicBoolean TIMESTAMP_TZ_WARNED = + new java.util.concurrent.atomic.AtomicBoolean(false); + @Override public String converterName() { return DatabaseIdentifier.XUGU; } + + @Override + protected void setValueToStatementByDataType( + Object value, + PreparedStatement statement, + SeaTunnelDataType seaTunnelDataType, + int statementIndex, + @Nullable String sourceType) + throws SQLException { + if (seaTunnelDataType.getSqlType().equals(SqlType.TIMESTAMP_TZ)) { + // LOSSY PATH: Xugu JDBC driver crashes on batch writes of OffsetDateTime / + // timezone-formatted strings for TIMESTAMP WITH TIME ZONE columns (bug [E19138]). + // The timezone offset is intentionally dropped; only the wall-clock value is stored. + // This is a documented limitation — not silent data loss. + if (TIMESTAMP_TZ_WARNED.compareAndSet(false, true)) { + log.warn( + "TIMESTAMP_TZ is written to Xugu as a plain TIMESTAMP: the timezone" + + " offset is dropped and only the wall-clock value is stored." + + " This is a known Xugu JDBC driver limitation (bug [E19138])."); + } + OffsetDateTime odt = (OffsetDateTime) value; + statement.setTimestamp(statementIndex, Timestamp.valueOf(odt.toLocalDateTime())); + } else { + super.setValueToStatementByDataType( + value, statement, seaTunnelDataType, statementIndex, sourceType); + } + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguTypeConverter.java index 38b0553124d9..8074e6ce4e72 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguTypeConverter.java @@ -187,11 +187,12 @@ public Column convert(BasicTypeDefine typeDefine) { builder.dataType(LocalTimeType.LOCAL_TIME_TYPE); break; case XUGU_DATETIME: - case XUGU_DATETIME_WITH_TIME_ZONE: builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); break; + case XUGU_DATETIME_WITH_TIME_ZONE: + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); + break; case XUGU_TIMESTAMP: - case XUGU_TIMESTAMP_WITH_TIME_ZONE: builder.dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE); if (typeDefine.getScale() == null) { builder.scale(TIMESTAMP_DEFAULT_SCALE); @@ -199,6 +200,14 @@ public Column convert(BasicTypeDefine typeDefine) { builder.scale(typeDefine.getScale()); } break; + case XUGU_TIMESTAMP_WITH_TIME_ZONE: + builder.dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE); + if (typeDefine.getScale() == null) { + builder.scale(TIMESTAMP_DEFAULT_SCALE); + } else { + builder.scale(typeDefine.getScale()); + } + break; default: throw CommonError.convertToSeaTunnelTypeError( DatabaseIdentifier.XUGU, xuguDataType, typeDefine.getName()); @@ -373,6 +382,28 @@ public BasicTypeDefine reconvert(Column column) { } builder.dataType(XUGU_TIMESTAMP); break; + case TIMESTAMP_TZ: + if (column.getScale() == null || column.getScale() <= 0) { + builder.columnType(XUGU_TIMESTAMP_WITH_TIME_ZONE); + } else { + int timestampTzScale = column.getScale(); + if (column.getScale() > MAX_TIMESTAMP_SCALE) { + timestampTzScale = MAX_TIMESTAMP_SCALE; + log.warn( + "The timestamp_tz column {} type timestamp_tz({}) is out of" + + " range, which exceeds the maximum scale of {}, " + + "it will be converted to timestamp_tz({})", + column.getName(), + column.getScale(), + MAX_TIMESTAMP_SCALE, + timestampTzScale); + } + builder.columnType( + String.format("TIMESTAMP(%s) WITH TIME ZONE", timestampTzScale)); + builder.scale(timestampTzScale); + } + builder.dataType(XUGU_TIMESTAMP_WITH_TIME_ZONE); + break; default: throw CommonError.convertToConnectorTypeError( DatabaseIdentifier.XUGU, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcFieldTypeUtils.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcFieldTypeUtils.java index 92874511714e..08e875a462eb 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcFieldTypeUtils.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcFieldTypeUtils.java @@ -23,10 +23,13 @@ import java.sql.Time; import java.sql.Timestamp; import java.time.Instant; +import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; +import java.util.Calendar; +import java.util.TimeZone; public final class JdbcFieldTypeUtils { @@ -96,6 +99,52 @@ public static Timestamp getTimestamp(ResultSet resultSet, int columnIndex) throw return resultSet.getTimestamp(columnIndex); } + /** + * Reads a NTZ (No Time Zone) timestamp column as {@link LocalDateTime}, free from JVM default + * timezone influence. + * + *

Strategy: + * + *

    + *
  1. Try {@code getObject(index, LocalDateTime.class)} first — supported by modern JDBC + * drivers (PostgreSQL ≥ 42.2, MySQL Connector/J ≥ 8.0, MariaDB Connector/J ≥ 3.x). This + * returns the wall-clock value exactly as stored, with no timezone conversion. + *
  2. Try {@code getTimestamp(index, utcCalendar)}: passing a UTC {@link Calendar} forces the + * driver to treat the raw bytes as UTC epoch millis, then {@link + * Timestamp#toLocalDateTime()} reconstructs the wall-clock via UTC — again + * timezone-neutral. A new {@link Calendar} is created per call to avoid thread-safety + * issues with a shared mutable instance. Not supported by all drivers (e.g. Hive JDBC). + *
  3. Last resort: plain {@code getTimestamp(index)} — may be affected by JVM timezone for + * drivers that apply session/JVM timezone conversion internally (e.g. Hive JDBC). + *
+ * + * @param resultSet the JDBC result set + * @param columnIndex 1-based column index + * @return the wall-clock {@link LocalDateTime} exactly as stored in the DB, or {@code null} + */ + public static LocalDateTime getLocalDateTime(ResultSet resultSet, int columnIndex) + throws SQLException { + // Prefer the modern JDBC 4.2 API — returns wall-clock value directly, no TZ involved + try { + return resultSet.getObject(columnIndex, LocalDateTime.class); + } catch (SQLException | UnsupportedOperationException ignored) { + // Driver does not support getObject(index, LocalDateTime.class) — fall back + } + // Try UTC Calendar to avoid JVM-default-timezone influence. + // A new Calendar is created per call to avoid thread-safety issues with a shared instance. + try { + Timestamp ts = + resultSet.getTimestamp( + columnIndex, Calendar.getInstance(TimeZone.getTimeZone("UTC"))); + return ts == null ? null : ts.toLocalDateTime(); + } catch (SQLException | UnsupportedOperationException ignored) { + // Driver does not support getTimestamp(index, Calendar) — fall back (e.g. Hive JDBC) + } + // Last resort: plain getTimestamp() — may be affected by JVM timezone for some drivers + Timestamp ts = resultSet.getTimestamp(columnIndex); + return ts == null ? null : ts.toLocalDateTime(); + } + public static byte[] getBytes(ResultSet resultSet, int columnIndex) throws SQLException { return resultSet.getBytes(columnIndex); } @@ -123,8 +172,20 @@ public static OffsetDateTime getOffsetDateTime(ResultSet resultSet, int columnIn } // Handle java.sql.Timestamp + // Avoid using Timestamp.toInstant() directly because the Timestamp was constructed + // with JVM-default-timezone semantics, which would shift the value by the JVM offset. + // Instead, try to re-read the column as a string and parse it with timezone info preserved. if (obj instanceof Timestamp) { - return ((Timestamp) obj).toInstant().atOffset(ZoneOffset.UTC); + String strVal = resultSet.getString(columnIndex); + if (strVal == null) { + return null; + } + try { + return parseOffsetDateTimeFromString(strVal); + } catch (Exception e) { + // Last resort: use the instant-based conversion (may shift by JVM offset) + return ((Timestamp) obj).toInstant().atOffset(ZoneOffset.UTC); + } } // Handle java.util.Date @@ -137,6 +198,23 @@ public static OffsetDateTime getOffsetDateTime(ResultSet resultSet, int columnIn return Instant.ofEpochMilli((Long) obj).atOffset(ZoneOffset.UTC); } + // Handle Oracle-specific TIMESTAMPLTZ / TIMESTAMPTZ types. + // oracle.sql.TIMESTAMPLTZ and oracle.sql.TIMESTAMPTZ do not implement standard interfaces + // and their toString() returns the Java object reference (e.g. + // "oracle.sql.TIMESTAMPLTZ@xxx"). + // Fall back to ResultSet.getTimestamp() which the Oracle JDBC driver converts correctly. + String objClassName = obj.getClass().getName(); + if (objClassName.equals("oracle.sql.TIMESTAMPLTZ") + || objClassName.equals("oracle.sql.TIMESTAMPTZ")) { + Timestamp oracleTs = + resultSet.getTimestamp( + columnIndex, Calendar.getInstance(TimeZone.getTimeZone("UTC"))); + if (oracleTs == null) { + return null; + } + return oracleTs.toInstant().atOffset(ZoneOffset.UTC); + } + // Try to parse as string String str = obj.toString(); try { @@ -201,7 +279,11 @@ private static String normalizeOffsetDateTimeString(String value) { if (normalized.endsWith(" UTC")) { normalized = normalized.substring(0, normalized.length() - 4) + "Z"; } - normalized = normalized.replace(' ', 'T'); + // Only replace the first space (between date and time) with 'T'. + // Then remove any space before the timezone offset (+/-). + // e.g. "2026-04-15 04:53:44.407 +08:00" → "2026-04-15T04:53:44.407+08:00" + normalized = normalized.replaceFirst(" ", "T"); + normalized = normalized.replace(" +", "+").replace(" -", "-"); if (normalized.matches(".*[+-]\\d{2}$")) { normalized = normalized + ":00"; } else if (normalized.matches(".*[+-]\\d{4}$")) { diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCreateTableSqlBuilderTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCreateTableSqlBuilderTest.java index 255ef1ae9e00..54589f8cf579 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCreateTableSqlBuilderTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCreateTableSqlBuilderTest.java @@ -115,14 +115,17 @@ public void testBuild() { List sqls = oracleCreateTableSqlBuilder.build(tablePath); String createTableSql = sqls.get(0); // create table sql is change; The old unit tests are no longer applicable + // After the NTZ/LTZ fix (#10685), LOCAL_DATE_TIME_TYPE (NTZ) maps to TIMESTAMP + // (without timezone), while OFFSET_DATE_TIME_TYPE (LTZ) maps to TIMESTAMP WITH LOCAL + // TIME ZONE. String expect = "CREATE TABLE \"test_table\" (\n" + "\"id\" INTEGER NOT NULL,\n" + "\"name\" VARCHAR2(128) NOT NULL,\n" + "\"age\" INTEGER,\n" + "\"blob_v\" BLOB,\n" - + "\"createTime\" TIMESTAMP WITH LOCAL TIME ZONE,\n" - + "\"lastUpdateTime\" TIMESTAMP WITH LOCAL TIME ZONE,\n" + + "\"createTime\" TIMESTAMP,\n" + + "\"lastUpdateTime\" TIMESTAMP,\n" + "CONSTRAINT id_9a8b PRIMARY KEY (\"id\")\n" + ")"; @@ -146,8 +149,8 @@ public void testBuild() { + "\"name\" VARCHAR2(128) NOT NULL,\n" + "\"age\" INTEGER,\n" + "\"blob_v\" BLOB,\n" - + "\"createTime\" TIMESTAMP WITH LOCAL TIME ZONE,\n" - + "\"lastUpdateTime\" TIMESTAMP WITH LOCAL TIME ZONE\n" + + "\"createTime\" TIMESTAMP,\n" + + "\"lastUpdateTime\" TIMESTAMP\n" + ")"; CONSOLE.println(expectSkipIndex); Assertions.assertEquals(expectSkipIndex, createTableSqlSkipIndex); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbTypeConverterTest.java index 0fa537d08fa2..75d42d071d33 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbTypeConverterTest.java @@ -600,7 +600,8 @@ public void testConvertDatetime() { .build(); column = DmdbTypeConverter.INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // DATETIME WITH TIME ZONE is LTZ → maps to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(typeDefine.getScale(), column.getScale()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType().toLowerCase()); @@ -613,7 +614,8 @@ public void testConvertDatetime() { .build(); column = DmdbTypeConverter.INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // DATETIME WITH TIME ZONE is LTZ → maps to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(typeDefine.getScale(), column.getScale()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType().toLowerCase()); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/duckdb/DuckDBTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/duckdb/DuckDBTypeConverterTest.java index 4024612d8a1e..bcb0b00f15f5 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/duckdb/DuckDBTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/duckdb/DuckDBTypeConverterTest.java @@ -242,7 +242,8 @@ void testConvertTimestamp() { @Test void testConvertTimestampWithTimezone() { Column column = convert("f_timestamp_tz", "timestamp with time zone"); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // TIMESTAMP WITH TIME ZONE is LTZ → must map to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); } @Test @@ -467,6 +468,21 @@ void testReconvertTimestamp() { Assertions.assertEquals(DuckDBTypeConverter.DUCKDB_TIMESTAMP, typeDefine.getDataType()); } + @Test + void testReconvertTimestampTz() { + BasicTypeDefine typeDefine = + DuckDBTypeConverter.INSTANCE.reconvert( + PhysicalColumn.builder() + .name("f_timestamp_tz") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .build()); + // OFFSET_DATE_TIME_TYPE → DUCKDB_TIMESTAMP_WITH_TZ + Assertions.assertEquals( + DuckDBTypeConverter.DUCKDB_TIMESTAMP_WITH_TZ, typeDefine.getColumnType()); + Assertions.assertEquals( + DuckDBTypeConverter.DUCKDB_TIMESTAMP_WITH_TZ, typeDefine.getDataType()); + } + @Test void testReconvertUnsupportedType() { Column mapColumn = diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseTypeConverterTest.java index 17176db80986..03302b48b9ea 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/kingbase/KingbaseTypeConverterTest.java @@ -862,4 +862,19 @@ public void testReconvertArray() { KingbaseTypeConverter.PG_SMALLINT_ARRAY, typeDefine.getColumnType()); Assertions.assertEquals(KingbaseTypeConverter.PG_SMALLINT_ARRAY, typeDefine.getDataType()); } + + @Test + public void testConvertSqlServerDatetimeoffsetIsLtz() { + // SQL Server DATETIMEOFFSET (LTZ) compatibility in Kingbase → must map to + // OFFSET_DATE_TIME_TYPE + BasicTypeDefine typeDefine = + BasicTypeDefine.builder() + .name("test") + .columnType("DATETIMEOFFSET") + .dataType("DATETIMEOFFSET") + .build(); + Column column = KingbaseTypeConverter.INSTANCE.convert(typeDefine); + Assertions.assertEquals(typeDefine.getName(), column.getName()); + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlTypeConverterTest.java index 1093285d6ebf..1401a5894a0a 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MySqlTypeConverterTest.java @@ -626,6 +626,7 @@ public void testConvertDatetime() { @Test public void testConvertTimestamp() { + // MySQL TIMESTAMP is LTZ → should map to OFFSET_DATE_TIME_TYPE (TIMESTAMP_TZ) BasicTypeDefine typeDefine = BasicTypeDefine.builder() .name("test") @@ -634,7 +635,7 @@ public void testConvertTimestamp() { .build(); Column column = MySqlTypeConverter.DEFAULT_INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); typeDefine = @@ -646,7 +647,7 @@ public void testConvertTimestamp() { .build(); column = MySqlTypeConverter.DEFAULT_INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(typeDefine.getScale(), column.getScale()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); } @@ -1088,6 +1089,39 @@ public void testReconvertDatetimeForV55() { Assertions.assertEquals(MySqlTypeConverter.MYSQL_DATETIME, typeDefine.getDataType()); } + @Test + public void testReconvertTimestampTz() { + // TIMESTAMP_TZ (LTZ) should map back to MySQL TIMESTAMP + Column column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .build(); + + BasicTypeDefine typeDefine = + MySqlTypeConverter.DEFAULT_INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals(MysqlType.TIMESTAMP, typeDefine.getNativeType()); + Assertions.assertEquals(MySqlTypeConverter.MYSQL_TIMESTAMP, typeDefine.getColumnType()); + Assertions.assertEquals(MySqlTypeConverter.MYSQL_TIMESTAMP, typeDefine.getDataType()); + + column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .scale(3) + .build(); + + typeDefine = MySqlTypeConverter.DEFAULT_INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals(MysqlType.TIMESTAMP, typeDefine.getNativeType()); + Assertions.assertEquals( + String.format("%s(%s)", MySqlTypeConverter.MYSQL_TIMESTAMP, column.getScale()), + typeDefine.getColumnType()); + Assertions.assertEquals(MySqlTypeConverter.MYSQL_TIMESTAMP, typeDefine.getDataType()); + Assertions.assertEquals(column.getScale(), typeDefine.getScale()); + } + @Test public void testConvertSet() { BasicTypeDefine typeDefine = diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlTypeConverterTest.java new file mode 100644 index 000000000000..6104e3b0baf9 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oceanbase/OceanBaseMySqlTypeConverterTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oceanbase; + +import org.apache.seatunnel.api.table.catalog.Column; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.converter.BasicTypeDefine; +import org.apache.seatunnel.api.table.type.LocalTimeType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link OceanBaseMySqlTypeConverter} verifying the NTZ/LTZ timestamp split + * introduced by the fix for https://github.com/apache/seatunnel/issues/10685. + */ +public class OceanBaseMySqlTypeConverterTest { + + @Test + public void testConvertDatetimeIsNtz() { + BasicTypeDefine typeDefine = + BasicTypeDefine.builder() + .name("test") + .columnType("DATETIME") + .dataType("DATETIME") + .build(); + Column column = OceanBaseMySqlTypeConverter.INSTANCE.convert(typeDefine); + Assertions.assertEquals(typeDefine.getName(), column.getName()); + // DATETIME is NTZ → must map to LOCAL_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + } + + @Test + public void testConvertTimestampIsLtz() { + BasicTypeDefine typeDefine = + BasicTypeDefine.builder() + .name("test") + .columnType("TIMESTAMP") + .dataType("TIMESTAMP") + .build(); + Column column = OceanBaseMySqlTypeConverter.INSTANCE.convert(typeDefine); + Assertions.assertEquals(typeDefine.getName(), column.getName()); + // TIMESTAMP is LTZ → must map to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); + } + + @Test + public void testReconvertDatetime() { + Column column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE) + .build(); + BasicTypeDefine typeDefine = OceanBaseMySqlTypeConverter.INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + // LOCAL_DATE_TIME_TYPE (NTZ) → DATETIME + Assertions.assertEquals( + OceanBaseMySqlTypeConverter.MYSQL_DATETIME, typeDefine.getColumnType()); + Assertions.assertEquals( + OceanBaseMySqlTypeConverter.MYSQL_DATETIME, typeDefine.getDataType()); + } + + @Test + public void testReconvertDatetimeTz() { + Column column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .build(); + BasicTypeDefine typeDefine = OceanBaseMySqlTypeConverter.INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + // OFFSET_DATE_TIME_TYPE (LTZ) → TIMESTAMP + Assertions.assertEquals( + OceanBaseMySqlTypeConverter.MYSQL_TIMESTAMP, typeDefine.getColumnType()); + Assertions.assertEquals( + OceanBaseMySqlTypeConverter.MYSQL_TIMESTAMP, typeDefine.getDataType()); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleTypeConverterTest.java index eddbddde89f8..a25dd327a7d5 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleTypeConverterTest.java @@ -612,7 +612,8 @@ public void testConvertDatetime() { column = INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // TIMESTAMP WITH TIME ZONE is LTZ → should map to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(6, column.getScale()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); @@ -626,7 +627,8 @@ public void testConvertDatetime() { column = INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // TIMESTAMP WITH LOCAL TIME ZONE is LTZ → should map to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(6, column.getScale()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); } @@ -882,12 +884,42 @@ public void testReconvertDate() { @Test public void testReconvertDatetime() { + // TIMESTAMP (NTZ) should map to ORACLE_TIMESTAMP Column column = PhysicalColumn.builder() .name("test") .dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE) .build(); + BasicTypeDefine typeDefine = INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals(OracleTypeConverter.ORACLE_TIMESTAMP, typeDefine.getColumnType()); + Assertions.assertEquals(OracleTypeConverter.ORACLE_TIMESTAMP, typeDefine.getDataType()); + + column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE) + .scale(3) + .build(); + + typeDefine = INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals( + String.format("TIMESTAMP(%s)", column.getScale()), typeDefine.getColumnType()); + Assertions.assertEquals(OracleTypeConverter.ORACLE_TIMESTAMP, typeDefine.getDataType()); + Assertions.assertEquals(column.getScale(), typeDefine.getScale()); + } + + @Test + public void testReconvertTimestampTz() { + // TIMESTAMP_TZ (LTZ) should map to ORACLE_TIMESTAMP_WITH_LOCAL_TIME_ZONE + Column column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .build(); + BasicTypeDefine typeDefine = INSTANCE.reconvert(column); Assertions.assertEquals(column.getName(), typeDefine.getName()); Assertions.assertEquals( @@ -900,7 +932,7 @@ public void testReconvertDatetime() { column = PhysicalColumn.builder() .name("test") - .dataType(LocalTimeType.LOCAL_DATE_TIME_TYPE) + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) .scale(3) .build(); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftTypeConverterTest.java index e491e1532840..8b3e9fc1f92d 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/redshift/RedshiftTypeConverterTest.java @@ -359,7 +359,8 @@ public void testConvertTimestamp() { .build(); column = RedshiftTypeConverter.INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // TIMESTAMP WITH TIME ZONE is LTZ → must map to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(RedshiftTypeConverter.MAX_TIMESTAMP_SCALE, column.getScale()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); } @@ -640,4 +641,35 @@ public void testReconvertDatetime() { Assertions.assertEquals(RedshiftTypeConverter.REDSHIFT_TIMESTAMP, typeDefine.getDataType()); Assertions.assertEquals(RedshiftTypeConverter.MAX_TIMESTAMP_SCALE, typeDefine.getScale()); } + + @Test + public void testReconvertDatetimeTz() { + Column column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .build(); + + BasicTypeDefine typeDefine = RedshiftTypeConverter.INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals( + RedshiftTypeConverter.REDSHIFT_TIMESTAMPTZ, typeDefine.getColumnType()); + Assertions.assertEquals( + RedshiftTypeConverter.REDSHIFT_TIMESTAMPTZ, typeDefine.getDataType()); + + column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .scale(3) + .build(); + + typeDefine = RedshiftTypeConverter.INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals( + RedshiftTypeConverter.REDSHIFT_TIMESTAMPTZ, typeDefine.getColumnType()); + Assertions.assertEquals( + RedshiftTypeConverter.REDSHIFT_TIMESTAMPTZ, typeDefine.getDataType()); + Assertions.assertEquals(3, typeDefine.getScale()); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerTypeConverterTest.java index 308a80497671..6ebaae9aabb0 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerTypeConverterTest.java @@ -540,7 +540,8 @@ public void testConvertDatetime() { .build(); column = SqlServerTypeConverter.INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // DATETIMEOFFSET is LTZ → should map to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(typeDefine.getScale(), column.getScale()); Assertions.assertEquals( String.format("%s(%s)", typeDefine.getDataType(), typeDefine.getScale()), @@ -872,4 +873,39 @@ public void testReconvertDatetime() { SqlServerTypeConverter.SQLSERVER_DATETIME2, typeDefine.getDataType()); Assertions.assertEquals(7, typeDefine.getScale()); } + + @Test + public void testReconvertTimestampTz() { + // TIMESTAMP_TZ (LTZ) should map to DATETIMEOFFSET + Column column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .build(); + + BasicTypeDefine typeDefine = SqlServerTypeConverter.INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals( + SqlServerTypeConverter.SQLSERVER_DATETIMEOFFSET, typeDefine.getColumnType()); + Assertions.assertEquals( + SqlServerTypeConverter.SQLSERVER_DATETIMEOFFSET, typeDefine.getDataType()); + + column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .scale(3) + .build(); + + typeDefine = SqlServerTypeConverter.INSTANCE.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals( + String.format( + "%s(%s)", + SqlServerTypeConverter.SQLSERVER_DATETIMEOFFSET, column.getScale()), + typeDefine.getColumnType()); + Assertions.assertEquals( + SqlServerTypeConverter.SQLSERVER_DATETIMEOFFSET, typeDefine.getDataType()); + Assertions.assertEquals(column.getScale(), typeDefine.getScale()); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguTypeConverterTest.java index 9dfd7079dfd2..41e3fafdec28 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/xugu/XuguTypeConverterTest.java @@ -343,7 +343,8 @@ public void testConvertTimestamp() { .build(); column = XuguTypeConverter.INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // DATETIME WITH TIME ZONE is LTZ → maps to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); typeDefine = @@ -379,7 +380,8 @@ public void testConvertTimestamp() { .build(); column = XuguTypeConverter.INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // TIMESTAMP WITH TIME ZONE is LTZ → maps to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(3, column.getScale()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); @@ -392,7 +394,8 @@ public void testConvertTimestamp() { .build(); column = XuguTypeConverter.INSTANCE.convert(typeDefine); Assertions.assertEquals(typeDefine.getName(), column.getName()); - Assertions.assertEquals(LocalTimeType.LOCAL_DATE_TIME_TYPE, column.getDataType()); + // TIMESTAMP(n) WITH TIME ZONE is LTZ → maps to OFFSET_DATE_TIME_TYPE + Assertions.assertEquals(LocalTimeType.OFFSET_DATE_TIME_TYPE, column.getDataType()); Assertions.assertEquals(typeDefine.getScale(), column.getScale()); Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcFieldTypeUtilsTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcFieldTypeUtilsTest.java index 8135b4b61877..f9ac9cb819b8 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcFieldTypeUtilsTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcFieldTypeUtilsTest.java @@ -40,6 +40,9 @@ public void testGetOffsetDateTimeFromTimestampUsesInstant() throws SQLException ResultSet rs = mock(ResultSet.class); when(rs.getObject(1)).thenReturn(timestamp); + // getString is called as the primary parse path for Timestamp objects; + // return an ISO-8601 string so parseOffsetDateTimeFromString can convert it. + when(rs.getString(1)).thenReturn("2025-01-01T00:00:00Z"); OffsetDateTime result = JdbcFieldTypeUtils.getOffsetDateTime(rs, 1); assertEquals(instant, result.toInstant()); diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/utils/RowConverter.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/utils/RowConverter.java index e25de32f24bc..7dfe1856fb02 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/utils/RowConverter.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/utils/RowConverter.java @@ -49,15 +49,19 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; import org.apache.paimon.utils.DateTimeUtils; import java.math.BigDecimal; import java.math.RoundingMode; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -309,6 +313,24 @@ public static SeaTunnelRow convert( Timestamp timestamp = rowData.getTimestamp(i, precision); objects[i] = timestamp.toLocalDateTime(); break; + case TIMESTAMP_TZ: + int tzPrecision = LocalZonedTimestampType.DEFAULT_PRECISION; + Optional tzPrecisionOptional = + tableSchema.fields().stream() + .filter(dataField -> dataField.name().equals(fieldName)) + .findFirst(); + if (tzPrecisionOptional.isPresent() + && tzPrecisionOptional.get().type() + instanceof LocalZonedTimestampType) { + tzPrecision = + ((LocalZonedTimestampType) tzPrecisionOptional.get().type()) + .getPrecision(); + } + Timestamp tzTimestamp = rowData.getTimestamp(i, tzPrecision); + objects[i] = + Instant.ofEpochMilli(tzTimestamp.getMillisecond()) + .atOffset(ZoneOffset.UTC); + break; case ARRAY: InternalArray paimonArray = rowData.getArray(i); ArrayType seatunnelArray = (ArrayType) fieldType; @@ -451,6 +473,17 @@ private static InternalRow reconvert( binaryWriter.writeTimestamp( i, Timestamp.fromLocalDateTime(datetime), precision); break; + case TIMESTAMP_TZ: + DataField tzDataField = SchemaUtil.getDataField(sinkFields, fieldName); + int tzWritePrecision = + ((LocalZonedTimestampType) tzDataField.type()).getPrecision(); + Instant instant = ((OffsetDateTime) fieldValue).toInstant(); + binaryWriter.writeTimestamp( + i, + Timestamp.fromEpochMillis( + instant.toEpochMilli(), instant.getNano() % 1_000_000), + tzWritePrecision); + break; case TIME: LocalTime time = (LocalTime) fieldValue; BinaryWriter.createValueSetter(DataTypes.TIME()) diff --git a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/utils/RowTypeConverter.java b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/utils/RowTypeConverter.java index df437f1ab5d2..34db4eddf8de 100644 --- a/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/utils/RowTypeConverter.java +++ b/seatunnel-connectors-v2/connector-paimon/src/main/java/org/apache/seatunnel/connectors/seatunnel/paimon/utils/RowTypeConverter.java @@ -222,10 +222,14 @@ public static RowType reconvert(SeaTunnelRowType seaTunnelRowType, TableSchema t DataType dataType = SeaTunnelTypeToPaimonVisitor.INSTANCE.visit(fieldName, fieldTypes[i]); DataTypeRoot typeRoot = dataType.getTypeRoot(); - if (typeRoot.equals(DataTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE) - || typeRoot.equals(DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) { + if (typeRoot.equals(DataTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE)) { DataField dataField = SchemaUtil.getDataField(fields, fieldName); dataType = new TimestampType(((TimestampType) dataField.type()).getPrecision()); + } else if (typeRoot.equals(DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) { + DataField dataField = SchemaUtil.getDataField(fields, fieldName); + dataType = + new LocalZonedTimestampType( + ((LocalZonedTimestampType) dataField.type()).getPrecision()); } if (typeRoot.equals(DataTypeRoot.TIME_WITHOUT_TIME_ZONE)) { DataField dataField = SchemaUtil.getDataField(fields, fieldName); @@ -290,6 +294,19 @@ public BasicTypeDefine visit(Column column) { builder.scale(timestampScale); builder.length(column.getColumnLength()); return builder.build(); + case TIMESTAMP_TZ: + int tzScale = + Objects.isNull(scale) + ? LocalZonedTimestampType.DEFAULT_PRECISION + : scale; + LocalZonedTimestampType tzType = + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(tzScale); + builder.nativeType(tzType.copy(column.isNullable())); + builder.dataType(tzType.getTypeRoot().name()); + builder.columnType(tzType.toString()); + builder.scale(tzScale); + builder.length(column.getColumnLength()); + return builder.build(); case TIME: int timeScale = Objects.isNull(scale) ? TimeType.DEFAULT_PRECISION : scale; TimeType timeType = DataTypes.TIME(timeScale); @@ -404,6 +421,9 @@ public DataType visit(String fieldName, SeaTunnelDataType dataType) { return DataTypes.TIME(TimeType.MAX_PRECISION); case TIMESTAMP: return DataTypes.TIMESTAMP(TimestampType.MAX_PRECISION); + case TIMESTAMP_TZ: + return DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE( + LocalZonedTimestampType.MAX_PRECISION); case MAP: SeaTunnelDataType keyType = ((org.apache.seatunnel.api.table.type.MapType) dataType) @@ -530,7 +550,7 @@ public SeaTunnelDataType visit(TimeType timeType) { @Override public SeaTunnelDataType visit(LocalZonedTimestampType localZonedTimestampType) { - return LocalTimeType.LOCAL_DATE_TIME_TYPE; + return LocalTimeType.OFFSET_DATE_TIME_TYPE; } @Override diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/datatypes/StarRocksTypeConverter.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/datatypes/StarRocksTypeConverter.java index 6d31e4337aea..16e552a09ce5 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/datatypes/StarRocksTypeConverter.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/datatypes/StarRocksTypeConverter.java @@ -319,6 +319,12 @@ public BasicTypeDefine reconvert(Column column) { builder.columnType(SR_DATETIME); builder.dataType(SR_DATETIME); break; + case TIMESTAMP_TZ: + // StarRocks DATETIME does not store timezone info; + // TIMESTAMP_TZ (LTZ) is mapped to DATETIME with potential timezone loss. + builder.columnType(SR_DATETIME); + builder.dataType(SR_DATETIME); + break; case MAP: reconvertMap(column, builder); break; @@ -550,6 +556,13 @@ private void reconvertBuildArrayInternal( builder.columnType(SR_DATETIME_ARRAY); builder.dataType(SR_DATETIME_ARRAY); break; + case TIMESTAMP_TZ: + // StarRocks DATETIME does not store timezone info; + // TIMESTAMP_TZ (LTZ) array is mapped to DATETIME array with potential timezone + // loss. + builder.columnType(SR_DATETIME_ARRAY); + builder.dataType(SR_DATETIME_ARRAY); + break; default: throw CommonError.convertToConnectorTypeError( identifier(), elementType.getSqlType().name(), columnName); diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/serialize/StarRocksBaseSerializer.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/serialize/StarRocksBaseSerializer.java index 04b5b4ec218b..871c01f9eda7 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/serialize/StarRocksBaseSerializer.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/serialize/StarRocksBaseSerializer.java @@ -64,6 +64,10 @@ protected Object convert(SeaTunnelDataType dataType, Object val) { return TimeUtils.toString((LocalTime) val, timeFormatter); case TIMESTAMP: return ((LocalDateTime) val).format(dateTimeFormatter); + case TIMESTAMP_TZ: + // StarRocks DATETIME does not store timezone info; + // convert OffsetDateTime to local wall-clock string. + return ((java.time.OffsetDateTime) val).toLocalDateTime().format(dateTimeFormatter); case ARRAY: case MAP: return JsonUtils.toJsonString(val); diff --git a/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksTypeConverterTest.java b/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksTypeConverterTest.java index 8bd4ca5f21bc..53bafec36983 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksTypeConverterTest.java @@ -1026,6 +1026,23 @@ public void testReconvertDatetime() { Assertions.assertEquals(SR_DATETIME, typeDefine.getDataType()); } + @Test + public void testReconvertDatetimeTz() { + // OFFSET_DATE_TIME_TYPE (TIMESTAMP_TZ / LTZ) → StarRocks DATETIME + // StarRocks DATETIME does not support timezone, so LTZ maps to DATETIME with potential + // timezone loss. + Column column = + PhysicalColumn.builder() + .name("test") + .dataType(LocalTimeType.OFFSET_DATE_TIME_TYPE) + .build(); + + BasicTypeDefine typeDefine = converter.reconvert(column); + Assertions.assertEquals(column.getName(), typeDefine.getName()); + Assertions.assertEquals(SR_DATETIME, typeDefine.getColumnType()); + Assertions.assertEquals(SR_DATETIME, typeDefine.getDataType()); + } + @Test public void testReconvertArray() { Column column = diff --git a/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/serialize/StarRocksJsonSerializerTest.java b/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/serialize/StarRocksJsonSerializerTest.java index 54c4401b8d0d..d85af7743e52 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/serialize/StarRocksJsonSerializerTest.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/serialize/StarRocksJsonSerializerTest.java @@ -29,6 +29,8 @@ import org.junit.jupiter.api.Test; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.temporal.ChronoField; @@ -70,4 +72,24 @@ public void serialize() { "{\"id\":1,\"name\":\"Tom\",\"array\":[\"tag1\",\"tag2\"],\"map\":{\"key1\":\"value1\"},\"timestamp\":\"2024-01-25 07:55:45.123\"}", jsonString); } + + @Test + public void serializeTimestampTz() { + // TIMESTAMP_TZ (OffsetDateTime / LTZ) → StarRocks DATETIME string (wall-clock, no tz) + String[] fieldNames = {"id", "ts_tz"}; + SeaTunnelDataType[] fieldTypes = { + BasicType.LONG_TYPE, LocalTimeType.OFFSET_DATE_TIME_TYPE + }; + + SeaTunnelRowType seaTunnelRowType = new SeaTunnelRowType(fieldNames, fieldTypes); + StarRocksJsonSerializer serializer = new StarRocksJsonSerializer(seaTunnelRowType, false); + + // 2026-04-15T04:15:23Z → toLocalDateTime() → "2026-04-15 04:15:23" + OffsetDateTime odt = OffsetDateTime.of(2026, 4, 15, 4, 15, 23, 0, ZoneOffset.UTC); + Object[] fields = {1L, odt}; + SeaTunnelRow row = new SeaTunnelRow(fields); + + String jsonString = serializer.serialize(row); + Assertions.assertEquals("{\"id\":1,\"ts_tz\":\"2026-04-15 04:15:23\"}", jsonString); + } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml index 11c147432d03..ce344d435a0f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml @@ -105,6 +105,19 @@ test + + org.testcontainers + postgresql + ${testcontainer.version} + test + + + + org.postgresql + postgresql + test + + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/JdbcToIcebergTimestampIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/JdbcToIcebergTimestampIT.java new file mode 100644 index 000000000000..fc233ddcefad --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/JdbcToIcebergTimestampIT.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.connector.iceberg; + +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; +import org.apache.seatunnel.e2e.common.container.EngineType; +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer; +import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.stream.Stream; + +/** + * E2E test verifying that NTZ (No Time Zone) and LTZ (Local Time Zone) timestamp columns from JDBC + * sources are stored with the correct Iceberg timestamp type: + * + *
    + *
  • NTZ → Iceberg {@code TimestampType.withoutZone()} (e.g. MySQL DATETIME, PG timestamp) + *
  • LTZ → Iceberg {@code TimestampType.withZone()} (e.g. MySQL TIMESTAMP, PG timestamptz) + *
+ * + *

Covers the fix for https://github.com/apache/seatunnel/issues/10685 + */ +@DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = + "Spark engine does not support TIMESTAMP_TZ (OffsetDateTime) natively; " + + "TIMESTAMP_TZ is serialized as a custom Decimal struct in Spark translation layer, " + + "which is incompatible with standard Sink connectors. " + + "Tested on Zeta and Flink engines only.") +@DisabledOnOs(OS.WINDOWS) +public class JdbcToIcebergTimestampIT extends TestSuiteBase implements TestResource { + + private static final Logger log = LoggerFactory.getLogger(JdbcToIcebergTimestampIT.class); + + // ------------------------------------------------------------------------- + // Catalog directories (inside the SeaTunnel container) + // ------------------------------------------------------------------------- + private static final String MYSQL_CATALOG_DIR = "/tmp/seatunnel_mnt/iceberg/hadoop-ts-mysql/"; + + private static final String PG_CATALOG_DIR = "/tmp/seatunnel_mnt/iceberg/hadoop-ts-pg/"; + + // ------------------------------------------------------------------------- + // MySQL container + // ------------------------------------------------------------------------- + private static final String MYSQL_IMAGE = "mysql:8.0"; + private static final String MYSQL_HOST = "mysql_timestamp_e2e"; + private static final String MYSQL_DATABASE = "ts_test"; + private static final String MYSQL_USER = "root"; + private static final String MYSQL_PASSWORD = "root"; + + private static final MySQLContainer MYSQL_CONTAINER = + new MySQLContainer<>(DockerImageName.parse(MYSQL_IMAGE)) + .withDatabaseName(MYSQL_DATABASE) + .withUsername(MYSQL_USER) + .withPassword(MYSQL_PASSWORD) + .withNetwork(NETWORK) + .withNetworkAliases(MYSQL_HOST) + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger("mysql-timestamp-image"))); + + // ------------------------------------------------------------------------- + // PostgreSQL container + // ------------------------------------------------------------------------- + private static final String PG_IMAGE = "postgres:14-alpine"; + private static final String PG_HOST = "pg_timestamp_e2e"; + private static final String PG_DATABASE = "ts_test"; + private static final String PG_USER = "postgres"; + private static final String PG_PASSWORD = "postgres"; + + private static final PostgreSQLContainer PG_CONTAINER = + new PostgreSQLContainer<>(DockerImageName.parse(PG_IMAGE)) + .withDatabaseName(PG_DATABASE) + .withUsername(PG_USER) + .withPassword(PG_PASSWORD) + .withNetwork(NETWORK) + .withNetworkAliases(PG_HOST) + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger("pg-timestamp-image"))); + + // ------------------------------------------------------------------------- + // Driver / plugin JARs downloaded into the SeaTunnel container + // ------------------------------------------------------------------------- + private static final String MYSQL_DRIVER_URL = + "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; + + private static final String PG_DRIVER_URL = + "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.3.3/postgresql-42.3.3.jar"; + + private static final String ZSTD_URL = + "https://repo1.maven.org/maven2/com/github/luben/zstd-jni/1.5.5-5/zstd-jni-1.5.5-5.jar"; + + // ------------------------------------------------------------------------- + // Container setup: create Iceberg dirs + download driver JARs + // ------------------------------------------------------------------------- + @TestContainerExtension + protected final ContainerExtendedFactory extendedFactory = + container -> { + for (String dir : + new String[] { + MYSQL_CATALOG_DIR + "seatunnel_namespace/mysql_ts_sink/data", + MYSQL_CATALOG_DIR + "seatunnel_namespace/mysql_ts_sink/metadata", + PG_CATALOG_DIR + "seatunnel_namespace/pg_ts_sink/data", + PG_CATALOG_DIR + "seatunnel_namespace/pg_ts_sink/metadata", + }) { + container.execInContainer("sh", "-c", "mkdir -p " + dir); + } + container.execInContainer("sh", "-c", "chmod -R 777 /tmp/seatunnel_mnt/iceberg/"); + + // Download Iceberg compression codec + container.execInContainer( + "sh", + "-c", + "mkdir -p /tmp/seatunnel/plugins/Iceberg/lib" + + " && cd /tmp/seatunnel/plugins/Iceberg/lib" + + " && wget -q " + + ZSTD_URL); + + // Download JDBC drivers into the Jdbc plugin directory + container.execInContainer( + "sh", + "-c", + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib" + + " && cd /tmp/seatunnel/plugins/Jdbc/lib" + + " && wget -q " + + MYSQL_DRIVER_URL + + " && wget -q " + + PG_DRIVER_URL); + }; + + // ------------------------------------------------------------------------- + // Lifecycle + // ------------------------------------------------------------------------- + @BeforeAll + @Override + public void startUp() throws Exception { + log.info("Starting MySQL and PostgreSQL containers..."); + Startables.deepStart(Stream.of(MYSQL_CONTAINER, PG_CONTAINER)).join(); + log.info("DB containers started. Initializing test data..."); + initMysqlData(); + initPostgresData(); + log.info("Test data initialised."); + } + + @AfterAll + @Override + public void tearDown() { + if (MYSQL_CONTAINER != null) { + MYSQL_CONTAINER.close(); + } + if (PG_CONTAINER != null) { + PG_CONTAINER.close(); + } + } + + // ------------------------------------------------------------------------- + // Test: MySQL DATETIME (NTZ) → Iceberg withoutZone() + // ------------------------------------------------------------------------- + @TestTemplate + public void testMysqlDatetimeToIcebergNtz(TestContainer container) + throws IOException, InterruptedException { + // Step 1: Run job to write data from MySQL to Iceberg + org.testcontainers.containers.Container.ExecResult result = + container.executeJob("/iceberg/mysql_jdbc_to_iceberg_timestamp.conf"); + Assertions.assertEquals( + 0, result.getExitCode(), "Write job failed:\n" + result.getStderr()); + + // Step 2: Run verification job (Iceberg -> Assert) + // This job verifies that the data in Iceberg matches expected types and values + org.testcontainers.containers.Container.ExecResult verifyResult = + container.executeJob("/iceberg/mysql_iceberg_to_assert.conf"); + Assertions.assertEquals( + 0, + verifyResult.getExitCode(), + "Verification job failed:\n" + verifyResult.getStderr()); + } + + // ------------------------------------------------------------------------- + // Test: PostgreSQL timestamp (NTZ) → Iceberg withoutZone() + // PostgreSQL timestamptz (LTZ) → Iceberg withZone() + // ------------------------------------------------------------------------- + @TestTemplate + public void testPgTimestampToIceberg(TestContainer container) + throws IOException, InterruptedException { + // Step 1: Run job to write data from PostgreSQL to Iceberg + org.testcontainers.containers.Container.ExecResult result = + container.executeJob("/iceberg/pg_jdbc_to_iceberg_timestamp.conf"); + Assertions.assertEquals( + 0, result.getExitCode(), "Write job failed:\n" + result.getStderr()); + + // Step 2: Run verification job (Iceberg -> Assert) + org.testcontainers.containers.Container.ExecResult verifyResult = + container.executeJob("/iceberg/pg_iceberg_to_assert.conf"); + Assertions.assertEquals( + 0, + verifyResult.getExitCode(), + "Verification job failed:\n" + verifyResult.getStderr()); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + private void initMysqlData() throws Exception { + try (Connection conn = + DriverManager.getConnection( + MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWORD); + Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE IF NOT EXISTS ts_table (" + + " id INT PRIMARY KEY," + + " dt_col DATETIME," + + " ts_col TIMESTAMP" + + ")"); + stmt.execute( + "INSERT INTO ts_table (id, dt_col, ts_col) VALUES" + + " (1, '2026-01-01 00:00:00', '2026-01-01 00:00:00')"); + } + } + + private void initPostgresData() throws Exception { + try (Connection conn = + DriverManager.getConnection( + PG_CONTAINER.getJdbcUrl(), PG_USER, PG_PASSWORD); + Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE IF NOT EXISTS ts_table (" + + " id INT PRIMARY KEY," + + " ts_col TIMESTAMP WITHOUT TIME ZONE," + + " tstz_col TIMESTAMP WITH TIME ZONE" + + ")"); + stmt.execute( + "INSERT INTO ts_table (id, ts_col, tstz_col) VALUES" + + " (1, '2026-01-01 00:00:00', '2026-01-01 00:00:00+00')"); + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/iceberg_source.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/iceberg_source.conf index fcec73e5d01e..3e42c56f69de 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/iceberg_source.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/iceberg_source.conf @@ -34,7 +34,7 @@ source { f5 = "float" f6 = "double" f7 = "date" - f9 = "timestamp" + f9 = "timestamp_tz" f10 = "timestamp" f11 = "string" f12 = "bytes" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/mysql_iceberg_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/mysql_iceberg_to_assert.conf new file mode 100644 index 000000000000..3ba2afb306fa --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/mysql_iceberg_to_assert.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Iceberg { + catalog_name = "seatunnel_test" + iceberg.catalog.config = { + "type" = "hadoop" + "warehouse" = "file:///tmp/seatunnel_mnt/iceberg/hadoop-ts-mysql/" + } + namespace = "seatunnel_namespace" + table = "mysql_ts_sink" + } +} + +sink { + Assert { + rules = { + row_rules = [ + { rule_type = MIN_ROW, rule_value = 1 }, + { rule_type = MAX_ROW, rule_value = 1 } + ] + field_rules = [ + { + field_name = dt_col + field_type = timestamp + field_value = [ + { rule_type = NOT_NULL, equals_to = "2026-01-01T00:00:00" } + ] + }, + { + field_name = ts_col + field_type = timestamp_tz + field_value = [ + { rule_type = NOT_NULL, equals_to = "2026-01-01T00:00:00Z" } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/mysql_jdbc_to_iceberg_timestamp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/mysql_jdbc_to_iceberg_timestamp.conf new file mode 100644 index 000000000000..545a54e0e74b --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/mysql_jdbc_to_iceberg_timestamp.conf @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + url = "jdbc:mysql://mysql_timestamp_e2e:3306/ts_test?useSSL=false&serverTimezone=UTC" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "root" + query = "SELECT id, dt_col, ts_col FROM ts_table" + plugin_output = "jdbc_source" + } +} + +sink { + Iceberg { + catalog_name = "seatunnel_test" + iceberg.catalog.config = { + "type" = "hadoop" + "warehouse" = "file:///tmp/seatunnel_mnt/iceberg/hadoop-ts-mysql/" + } + namespace = "seatunnel_namespace" + table = "mysql_ts_sink" + plugin_input = "jdbc_source" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/pg_iceberg_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/pg_iceberg_to_assert.conf new file mode 100644 index 000000000000..647dc1da8e96 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/pg_iceberg_to_assert.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Iceberg { + catalog_name = "seatunnel_test" + iceberg.catalog.config = { + "type" = "hadoop" + "warehouse" = "file:///tmp/seatunnel_mnt/iceberg/hadoop-ts-pg/" + } + namespace = "seatunnel_namespace" + table = "pg_ts_sink" + } +} + +sink { + Assert { + rules = { + row_rules = [ + { rule_type = MIN_ROW, rule_value = 1 }, + { rule_type = MAX_ROW, rule_value = 1 } + ] + field_rules = [ + { + field_name = ts_col + field_type = timestamp + field_value = [ + { rule_type = NOT_NULL, equals_to = "2026-01-01T00:00:00" } + ] + }, + { + field_name = tstz_col + field_type = timestamp_tz + field_value = [ + { rule_type = NOT_NULL, equals_to = "2026-01-01T00:00:00Z" } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/pg_jdbc_to_iceberg_timestamp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/pg_jdbc_to_iceberg_timestamp.conf new file mode 100644 index 000000000000..7640356e601c --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/pg_jdbc_to_iceberg_timestamp.conf @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + url = "jdbc:postgresql://pg_timestamp_e2e:5432/ts_test" + driver = "org.postgresql.Driver" + user = "postgres" + password = "postgres" + query = "SELECT id, ts_col, tstz_col FROM ts_table" + plugin_output = "jdbc_source" + } +} + +sink { + Iceberg { + catalog_name = "seatunnel_test" + iceberg.catalog.config = { + "type" = "hadoop" + "warehouse" = "file:///tmp/seatunnel_mnt/iceberg/hadoop-ts-pg/" + } + namespace = "seatunnel_namespace" + table = "pg_ts_sink" + plugin_input = "jdbc_source" + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/src/test/resources/iceberg/iceberg_source.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/src/test/resources/iceberg/iceberg_source.conf index 1430d77e505c..0be4ac65ed53 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/src/test/resources/iceberg/iceberg_source.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/src/test/resources/iceberg/iceberg_source.conf @@ -38,7 +38,7 @@ source { f5 = "float" f6 = "double" f7 = "date" - f9 = "timestamp" + f9 = "timestamp_tz" f10 = "timestamp" f11 = "string" f12 = "bytes" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/resources/iceberg/iceberg_source.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/resources/iceberg/iceberg_source.conf index 6b50aba96fbe..36386dd25f43 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/resources/iceberg/iceberg_source.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/resources/iceberg/iceberg_source.conf @@ -38,7 +38,7 @@ source { f5 = "float" f6 = "double" f7 = "date" - f9 = "timestamp" + f9 = "timestamp_tz" f10 = "timestamp" f11 = "string" f12 = "bytes" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java index 31274185eab5..c28e640918c9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java @@ -367,9 +367,25 @@ public void tearDown() throws SQLException { } } + /** + * Hook for subclasses to skip testJdbcDb for specific engine types without overriding + * the @TestTemplate method itself. Overriding a @TestTemplate method in a subclass causes JUnit + * 5 to register and run the test twice (once from the parent, once from the child), leading to + * duplicate data in the sink table and incorrect row-count assertions. + * + * @param container the current test container + * @return true if testJdbcDb should be skipped for this container + */ + protected boolean isDisabledOnContainer(TestContainer container) { + return false; + } + @TestTemplate public void testJdbcDb(TestContainer container) throws IOException, InterruptedException, SQLException { + if (isDisabledOnContainer(container)) { + return; + } List configFiles = jdbcCase.getConfigFile(); for (String configFile : configFiles) { try { @@ -574,7 +590,23 @@ private Object checkData(Object data) throws SQLException, IOException { javaArray[index] = checkData(jdbcArray[index]); } return javaArray; + } else if (data instanceof java.time.OffsetDateTime) { + // Normalize OffsetDateTime to Timestamp for comparison + return java.sql.Timestamp.valueOf(((java.time.OffsetDateTime) data).toLocalDateTime()); } else { + // oracle.sql.TIMESTAMPLTZ / TIMESTAMPTZ objects do not override equals() correctly + // for cross-object comparison. Normalize to byte[] via toBytes() so that + // assertArrayEquals() can compare the raw timestamp bytes directly. + String className = data.getClass().getName(); + if (className.equals("oracle.sql.TIMESTAMPLTZ") + || className.equals("oracle.sql.TIMESTAMPTZ")) { + try { + java.lang.reflect.Method toBytes = data.getClass().getMethod("toBytes"); + return toBytes.invoke(data); + } catch (Exception e) { + return data; + } + } return data; } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java index 4448e6e90670..99d1826937a3 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java @@ -481,6 +481,23 @@ private List> querySink(String sql) { Object object = resultSet.getObject(i); if (object instanceof NClob) { objects.add(readNClobAsString((NClob) object)); + } else if (object instanceof java.time.OffsetDateTime) { + // TIMESTAMP_TZ (OffsetDateTime) → normalize to Timestamp for comparison + // with MySQL source which returns java.sql.Timestamp + objects.add( + java.sql.Timestamp.valueOf( + ((java.time.OffsetDateTime) object).toLocalDateTime())); + } else if (object != null + && object.getClass().getName().equals("microsoft.sql.DateTimeOffset")) { + // SQL Server DATETIMEOFFSET → normalize to Timestamp for comparison + // microsoft.sql.DateTimeOffset.getTimestamp() returns java.sql.Timestamp + try { + java.lang.reflect.Method getTimestamp = + object.getClass().getMethod("getTimestamp"); + objects.add(getTimestamp.invoke(object)); + } catch (Exception e) { + objects.add(object); + } } else { objects.add(object); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlTimestampIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlTimestampIT.java new file mode 100644 index 000000000000..233c9b8cef9e --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlTimestampIT.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; +import org.apache.seatunnel.e2e.common.container.EngineType; +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer; +import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.awaitility.Awaitility.given; + +/** + * E2E test verifying that MySQL NTZ/LTZ timestamp types are correctly distinguished by the JDBC + * connector after the fix for https://github.com/apache/seatunnel/issues/10685. + * + *

    + *
  • MySQL {@code DATETIME} (NTZ) → SeaTunnel internal {@code TIMESTAMP} type + *
  • MySQL {@code TIMESTAMP} (LTZ) → SeaTunnel internal {@code TIMESTAMP_TZ} type + *
+ * + *

The Assert sink's {@code field_type} check is used to validate the internal type mapping. + */ +@DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = + "Spark engine does not support TIMESTAMP_TZ (OffsetDateTime) natively; " + + "TIMESTAMP_TZ is serialized as a custom Decimal struct in Spark translation layer, " + + "which is incompatible with standard Sink connectors. " + + "Tested on Zeta and Flink engines only.") +@Slf4j +public class JdbcMysqlTimestampIT extends TestSuiteBase implements TestResource { + + private static final String MYSQL_IMAGE = "mysql:8.0"; + private static final String MYSQL_HOST = "mysql_ts_e2e"; + private static final String MYSQL_DATABASE = "ts_test"; + private static final String MYSQL_USER = "root"; + private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; + + private static final String MYSQL_DRIVER_URL = + "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; + + private MySQLContainer mysqlContainer; + + @TestContainerExtension + private final ContainerExtendedFactory extendedFactory = + container -> { + Container.ExecResult result = + container.execInContainer( + "bash", + "-c", + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib" + + " && cd /tmp/seatunnel/plugins/Jdbc/lib" + + " && wget -q " + + MYSQL_DRIVER_URL); + Assertions.assertEquals( + 0, + result.getExitCode(), + "Failed to download MySQL driver: " + result.getStderr()); + }; + + @BeforeAll + @Override + public void startUp() throws Exception { + mysqlContainer = + new MySQLContainer<>(DockerImageName.parse(MYSQL_IMAGE)) + .withDatabaseName(MYSQL_DATABASE) + .withUsername(MYSQL_USER) + .withPassword(MYSQL_PASSWORD) + .withNetwork(NETWORK) + .withNetworkAliases(MYSQL_HOST) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); + + Startables.deepStart(Stream.of(mysqlContainer)).join(); + + given().ignoreExceptions() + .await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted(() -> initMysqlData()); + log.info("MySQL container started and test data initialised."); + } + + @AfterAll + @Override + public void tearDown() { + if (mysqlContainer != null) { + mysqlContainer.close(); + } + } + + /** + * Verifies that MySQL {@code DATETIME} (NTZ) columns are read as SeaTunnel {@code TIMESTAMP} + * (i.e. {@code LOCAL_DATE_TIME_TYPE}), not {@code TIMESTAMP_TZ}. + * + *

The Assert sink's {@code field_type = timestamp} assertion will fail if the connector + * incorrectly maps {@code DATETIME} to {@code TIMESTAMP_TZ}. + */ + @TestTemplate + public void testMysqlDatetimeIsNtz(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = container.executeJob("/jdbc_mysql_datetime_to_assert.conf"); + Assertions.assertEquals( + 0, + result.getExitCode(), + "MySQL DATETIME (NTZ) assertion failed:\n" + result.getStderr()); + } + + /** + * Verifies that MySQL {@code TIMESTAMP} (LTZ) columns are read as SeaTunnel {@code + * TIMESTAMP_TZ} (i.e. {@code OFFSET_DATE_TIME_TYPE}). + * + *

The Assert sink's {@code field_type = timestamp_tz} assertion will fail if the connector + * incorrectly maps {@code TIMESTAMP} to plain {@code TIMESTAMP}. + */ + @TestTemplate + public void testMysqlTimestampIsLtz(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = container.executeJob("/jdbc_mysql_timestamp_to_assert.conf"); + Assertions.assertEquals( + 0, + result.getExitCode(), + "MySQL TIMESTAMP (LTZ) assertion failed:\n" + result.getStderr()); + } + + /** + * Core fix scenario: verifies that MySQL {@code TIMESTAMP} (LTZ) preserves the correct UTC + * instant when the JDBC connection uses a non-UTC {@code serverTimezone} (Asia/Seoul, +09:00). + * + *

Before the fix, {@code JdbcFieldTypeUtils.getOffsetDateTime()} applied the JVM default + * timezone during {@code ResultSet} traversal. In a Seoul-timezone session a value stored as + * UTC midnight would be shifted by +09:00 and read back as {@code 2026-01-01T09:00:00Z} instead + * of {@code 2026-01-01T00:00:00Z} — a 9-hour epoch error. + */ + @TestTemplate + public void testMysqlTimestampIsLtzInNonUtcSession(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = + container.executeJob("/jdbc_mysql_timestamp_non_utc_to_assert.conf"); + Assertions.assertEquals( + 0, + result.getExitCode(), + "MySQL TIMESTAMP (LTZ) assertion failed with non-UTC serverTimezone (Asia/Seoul):\n" + + result.getStderr()); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private void initMysqlData() throws Exception { + String jdbcUrl = + String.format( + "jdbc:mysql://%s:%d/%s?useSSL=false&serverTimezone=UTC", + mysqlContainer.getHost(), + mysqlContainer.getFirstMappedPort(), + MYSQL_DATABASE); + try (Connection conn = DriverManager.getConnection(jdbcUrl, MYSQL_USER, MYSQL_PASSWORD); + Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE IF NOT EXISTS ts_source (" + + " id INT PRIMARY KEY," + + " dt_col DATETIME," + + " ts_col TIMESTAMP NULL" + + ")"); + // Insert a fixed wall-clock value: 2026-01-01 00:00:00 + // DATETIME stores it as-is (NTZ); TIMESTAMP stores UTC and displays in session TZ. + stmt.execute( + "INSERT INTO ts_source (id, dt_col, ts_col) VALUES" + + " (1, '2026-01-01 00:00:00', '2026-01-01 00:00:00')"); + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java index 7e3e6fe5959a..8730cd9525ba 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java @@ -28,6 +28,7 @@ import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.oracle.OracleDialect; import org.apache.seatunnel.connectors.seatunnel.jdbc.source.JdbcSourceTable; +import org.apache.seatunnel.e2e.common.container.EngineType; import org.apache.seatunnel.e2e.common.container.TestContainer; import org.junit.jupiter.api.Assertions; @@ -167,6 +168,16 @@ public void testSampleDataFromColumnSuccess() throws Exception { dialect.sampleDataFromColumn(connection, table, "INTEGER_COL", 1, 1024); } + /** + * Disabled on Spark: TIMESTAMP WITH LOCAL TIME ZONE is now mapped to TIMESTAMP_TZ + * (OffsetDateTime). Spark encodes TIMESTAMP_TZ as DecimalType(18, 5) internally, causing + * byte-level mismatch on Oracle round-trip. See JdbcMysqlTimestampIT for the same limitation. + */ + @Override + protected boolean isDisabledOnContainer(TestContainer container) { + return container.identifier().getEngineType() == EngineType.SPARK; + } + @TestTemplate public void testOracleWithoutDecimalTypeNarrowing(TestContainer container) throws Exception { Container.ExecResult execResult = diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_datetime_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_datetime_to_assert.conf new file mode 100644 index 000000000000..cb2d002a3a8f --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_datetime_to_assert.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Verifies that MySQL DATETIME (NTZ) is read as SeaTunnel TIMESTAMP (LOCAL_DATE_TIME_TYPE). +# Covers the fix for https://github.com/apache/seatunnel/issues/10685 + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + url = "jdbc:mysql://mysql_ts_e2e:3306/ts_test?useSSL=false&serverTimezone=UTC" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" + query = "SELECT id, dt_col FROM ts_source" + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + } + ] + field_rules = [ + { + # dt_col is MySQL DATETIME → must map to SeaTunnel TIMESTAMP (NTZ) + field_name = dt_col + field_type = timestamp + field_value = [ + { + rule_type = NOT_NULL + equals_to = "2026-01-01T00:00:00" + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_timestamp_non_utc_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_timestamp_non_utc_to_assert.conf new file mode 100644 index 000000000000..1e1e85b19f8e --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_timestamp_non_utc_to_assert.conf @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Verifies that MySQL TIMESTAMP (LTZ) preserves the correct UTC instant even when the +# JDBC connection is made with a non-UTC serverTimezone (Asia/Seoul, +09:00). +# +# Core fix scenario: before the fix, JdbcFieldTypeUtils.getOffsetDateTime() used the +# JVM default timezone during ResultSet traversal, so a value stored as UTC midnight +# would be read back as 09:00 (+09:00) in a Seoul-timezone environment — a 9-hour shift. +# After the fix, the UTC instant must be preserved regardless of serverTimezone. +# +# Covers the fix for https://github.com/apache/seatunnel/issues/10685 + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + # serverTimezone=Asia/Seoul simulates a non-UTC JDBC session. + # The stored UTC epoch must still be read correctly as TIMESTAMP_TZ. + url = "jdbc:mysql://mysql_ts_e2e:3306/ts_test?useSSL=false&serverTimezone=Asia%2FSeoul" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" + query = "SELECT id, ts_col FROM ts_source" + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + } + ] + field_rules = [ + { + # ts_col is MySQL TIMESTAMP → must still map to TIMESTAMP_TZ in non-UTC session. + # If the fix regresses, the field_type check will fail because the value will + # be read as a plain TIMESTAMP (LocalDateTime) shifted by the Seoul offset. + field_name = ts_col + field_type = timestamp_tz + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_timestamp_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_timestamp_to_assert.conf new file mode 100644 index 000000000000..9cdec934bbfd --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_timestamp_to_assert.conf @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Verifies that MySQL TIMESTAMP (LTZ) is read as SeaTunnel TIMESTAMP_TZ (OFFSET_DATE_TIME_TYPE). +# Covers the fix for https://github.com/apache/seatunnel/issues/10685 + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + url = "jdbc:mysql://mysql_ts_e2e:3306/ts_test?useSSL=false&serverTimezone=UTC" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" + query = "SELECT id, ts_col FROM ts_source" + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + } + ] + field_rules = [ + { + # ts_col is MySQL TIMESTAMP → must map to SeaTunnel TIMESTAMP_TZ (LTZ) + field_name = ts_col + field_type = timestamp_tz + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresTimestampIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresTimestampIT.java new file mode 100644 index 000000000000..b9f14c4d1dac --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresTimestampIT.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; +import org.apache.seatunnel.e2e.common.container.EngineType; +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer; +import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.awaitility.Awaitility.given; + +/** + * E2E test verifying that PostgreSQL NTZ/LTZ timestamp types are correctly distinguished by the + * JDBC connector after the fix for https://github.com/apache/seatunnel/issues/10685. + * + *

    + *
  • PostgreSQL {@code TIMESTAMP WITHOUT TIME ZONE} (NTZ) → SeaTunnel internal {@code TIMESTAMP} + *
  • PostgreSQL {@code TIMESTAMP WITH TIME ZONE} (LTZ) → SeaTunnel internal {@code TIMESTAMP_TZ} + *
+ * + *

The Assert sink's {@code field_type} check is used to validate the internal type mapping. + */ +@DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = + "Spark engine does not support TIMESTAMP_TZ (OffsetDateTime) natively; " + + "TIMESTAMP_TZ is serialized as a custom Decimal struct in Spark translation layer, " + + "which is incompatible with standard Sink connectors. " + + "Tested on Zeta and Flink engines only.") +@Slf4j +public class JdbcPostgresTimestampIT extends TestSuiteBase implements TestResource { + + private static final String PG_IMAGE = "postgres:14-alpine"; + private static final String PG_HOST = "pg_ts_e2e"; + private static final String PG_DATABASE = "ts_test"; + private static final String PG_USER = "postgres"; + private static final String PG_PASSWORD = "postgres"; + + private static final String PG_DRIVER_URL = + "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.3.3/postgresql-42.3.3.jar"; + + private PostgreSQLContainer pgContainer; + + @TestContainerExtension + private final ContainerExtendedFactory extendedFactory = + container -> { + Container.ExecResult result = + container.execInContainer( + "bash", + "-c", + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib" + + " && cd /tmp/seatunnel/plugins/Jdbc/lib" + + " && wget -q " + + PG_DRIVER_URL); + Assertions.assertEquals( + 0, + result.getExitCode(), + "Failed to download PostgreSQL driver: " + result.getStderr()); + }; + + @BeforeAll + @Override + public void startUp() throws Exception { + pgContainer = + new PostgreSQLContainer<>(DockerImageName.parse(PG_IMAGE)) + .withDatabaseName(PG_DATABASE) + .withUsername(PG_USER) + .withPassword(PG_PASSWORD) + .withNetwork(NETWORK) + .withNetworkAliases(PG_HOST) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(PG_IMAGE))); + + Startables.deepStart(Stream.of(pgContainer)).join(); + + given().ignoreExceptions() + .await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted(() -> initPgData()); + log.info("PostgreSQL container started and test data initialised."); + } + + @AfterAll + @Override + public void tearDown() { + if (pgContainer != null) { + pgContainer.close(); + } + } + + /** + * Verifies that PostgreSQL {@code TIMESTAMP WITHOUT TIME ZONE} (NTZ) columns are read as + * SeaTunnel {@code TIMESTAMP} (i.e. {@code LOCAL_DATE_TIME_TYPE}). + * + *

The Assert sink's {@code field_type = timestamp} assertion will fail if the connector + * incorrectly maps plain {@code TIMESTAMP} to {@code TIMESTAMP_TZ}. + */ + @TestTemplate + public void testPgTimestampIsNtz(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = container.executeJob("/jdbc_pg_timestamp_to_assert.conf"); + Assertions.assertEquals( + 0, + result.getExitCode(), + "PostgreSQL TIMESTAMP (NTZ) assertion failed:\n" + result.getStderr()); + } + + /** + * Verifies that PostgreSQL {@code TIMESTAMP WITH TIME ZONE} (LTZ) columns are read as SeaTunnel + * {@code TIMESTAMP_TZ} (i.e. {@code OFFSET_DATE_TIME_TYPE}). + * + *

The Assert sink's {@code field_type = timestamp_tz} assertion will fail if the connector + * incorrectly maps {@code TIMESTAMPTZ} to plain {@code TIMESTAMP}. + */ + @TestTemplate + public void testPgTimestamptzIsLtz(TestContainer container) + throws IOException, InterruptedException { + Container.ExecResult result = container.executeJob("/jdbc_pg_timestamptz_to_assert.conf"); + Assertions.assertEquals( + 0, + result.getExitCode(), + "PostgreSQL TIMESTAMPTZ (LTZ) assertion failed:\n" + result.getStderr()); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private void initPgData() throws Exception { + String jdbcUrl = + String.format( + "jdbc:postgresql://%s:%d/%s", + pgContainer.getHost(), pgContainer.getFirstMappedPort(), PG_DATABASE); + try (Connection conn = DriverManager.getConnection(jdbcUrl, PG_USER, PG_PASSWORD); + Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE IF NOT EXISTS ts_source (" + + " id INT PRIMARY KEY," + + " ts_col TIMESTAMP WITHOUT TIME ZONE," + + " tstz_col TIMESTAMP WITH TIME ZONE" + + ")"); + // ts_col: wall-clock value stored as-is (NTZ, no timezone conversion) + // tstz_col: value with explicit UTC offset stored in UTC internally (LTZ) + stmt.execute( + "INSERT INTO ts_source (id, ts_col, tstz_col) VALUES" + + " (1, '2026-01-01 00:00:00', '2026-01-01 00:00:00+00')"); + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_pg_timestamp_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_pg_timestamp_to_assert.conf new file mode 100644 index 000000000000..f3de0481c7a0 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_pg_timestamp_to_assert.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Verifies that PostgreSQL TIMESTAMP WITHOUT TIME ZONE (NTZ) is read as SeaTunnel TIMESTAMP. +# Covers the fix for https://github.com/apache/seatunnel/issues/10685 + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + url = "jdbc:postgresql://pg_ts_e2e:5432/ts_test?loggerLevel=OFF" + driver = "org.postgresql.Driver" + user = "postgres" + password = "postgres" + query = "SELECT id, ts_col FROM ts_source" + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + } + ] + field_rules = [ + { + # ts_col is TIMESTAMP WITHOUT TIME ZONE → must map to SeaTunnel TIMESTAMP (NTZ) + field_name = ts_col + field_type = timestamp + field_value = [ + { + rule_type = NOT_NULL + equals_to = "2026-01-01T00:00:00" + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_pg_timestamptz_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_pg_timestamptz_to_assert.conf new file mode 100644 index 000000000000..802c798cc7aa --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_pg_timestamptz_to_assert.conf @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Verifies that PostgreSQL TIMESTAMP WITH TIME ZONE (LTZ) is read as SeaTunnel TIMESTAMP_TZ. +# Covers the fix for https://github.com/apache/seatunnel/issues/10685 + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + url = "jdbc:postgresql://pg_ts_e2e:5432/ts_test?loggerLevel=OFF" + driver = "org.postgresql.Driver" + user = "postgres" + password = "postgres" + query = "SELECT id, tstz_col FROM ts_source" + } +} + +sink { + Assert { + rules { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + } + ] + field_rules = [ + { + # tstz_col is TIMESTAMP WITH TIME ZONE → must map to SeaTunnel TIMESTAMP_TZ (LTZ) + field_name = tstz_col + field_type = timestamp_tz + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksSchemaChangeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksSchemaChangeIT.java index f5b7522499b9..8d5d8a927f69 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksSchemaChangeIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksSchemaChangeIT.java @@ -54,6 +54,7 @@ import java.sql.Statement; import java.sql.Timestamp; import java.time.LocalDateTime; +import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; @@ -443,17 +444,21 @@ private List> query(String sql, Connection connection) { while (resultSet.next()) { ArrayList objects = new ArrayList<>(); for (int i = 1; i <= columnCount; i++) { - if (resultSet.getObject(i) instanceof Timestamp) { - Timestamp timestamp = resultSet.getTimestamp(i); - objects.add(timestamp.toLocalDateTime().format(DATE_TIME_FORMATTER)); - break; + Object obj = resultSet.getObject(i); + if (obj instanceof Timestamp) { + objects.add( + ((Timestamp) obj).toLocalDateTime().format(DATE_TIME_FORMATTER)); + } else if (obj instanceof LocalDateTime) { + objects.add(((LocalDateTime) obj).format(DATE_TIME_FORMATTER)); + } else if (obj instanceof OffsetDateTime) { + // TIMESTAMP_TZ (LTZ) → normalize to wall-clock string for comparison + objects.add( + ((OffsetDateTime) obj) + .toLocalDateTime() + .format(DATE_TIME_FORMATTER)); + } else { + objects.add(obj); } - if (resultSet.getObject(i) instanceof LocalDateTime) { - LocalDateTime localDateTime = resultSet.getObject(i, LocalDateTime.class); - objects.add(localDateTime.format(DATE_TIME_FORMATTER)); - break; - } - objects.add(resultSet.getObject(i)); } log.debug(String.format("Print query, sql: %s, data: %s", sql, objects)); result.add(objects); diff --git a/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java b/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java index 1eb73b34ca4e..ba3b7be105c0 100644 --- a/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java +++ b/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java @@ -46,6 +46,8 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; @@ -264,6 +266,8 @@ private Object convert( return objectArrayList.toArray(new LocalTime[0]); case TIMESTAMP: return objectArrayList.toArray(new LocalDateTime[0]); + case TIMESTAMP_TZ: + return objectArrayList.toArray(new OffsetDateTime[0]); default: throw new SeaTunnelCsvFormatException( CommonErrorCode.UNSUPPORTED_DATA_TYPE, @@ -315,6 +319,8 @@ private Object convert( return parseTime(field); case TIMESTAMP: return parseTimestamp(field, fieldName); + case TIMESTAMP_TZ: + return parseTimestampTz(field, fieldName); case ROW: Map splitsMap = splitLineBySeaTunnelRowType(field, (SeaTunnelRowType) fieldType, level + 1); @@ -374,4 +380,21 @@ private LocalDateTime parseTimestamp(String field, String fieldName) { parsedTimestamp.query(TemporalQueries.localDate()), parsedTimestamp.query(TemporalQueries.localTime())); } + + private OffsetDateTime parseTimestampTz(String field, String fieldName) { + try { + return OffsetDateTime.parse(field, DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } catch (DateTimeParseException ignored) { + // Fallback: data written by old SeaTunnel (wall-clock, no offset). + DateTimeFormatter fallbackFmt = DateTimeUtils.matchDateTimeFormatter(field); + if (fallbackFmt == null) { + throw CommonError.formatDateTimeError(field, fieldName); + } + TemporalAccessor ta = fallbackFmt.parse(field); + return LocalDateTime.of( + ta.query(TemporalQueries.localDate()), + ta.query(TemporalQueries.localTime())) + .atOffset(ZoneOffset.UTC); + } + } } diff --git a/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvSerializationSchema.java b/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvSerializationSchema.java index 10d5162efe92..d853c75e772f 100644 --- a/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvSerializationSchema.java +++ b/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvSerializationSchema.java @@ -44,6 +44,8 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.Map; import java.util.stream.Collectors; @@ -58,6 +60,8 @@ public class CsvSerializationSchema implements SerializationSchema { private final Charset charset; private final String nullValue; private final CsvStringQuoteMode quoteMode; + /** When true, TIMESTAMP_TZ is serialized as wall-clock (no offset) for DB sinks like Doris. */ + private final boolean wallClockTimestampTz; private CsvSerializationSchema( @NonNull SeaTunnelRowType seaTunnelRowType, @@ -67,7 +71,8 @@ private CsvSerializationSchema( TimeUtils.Formatter timeFormatter, Charset charset, String nullValue, - CsvStringQuoteMode quoteMode) { + CsvStringQuoteMode quoteMode, + boolean wallClockTimestampTz) { this.seaTunnelRowType = seaTunnelRowType; this.separators = separators; this.dateFormatter = dateFormatter; @@ -76,6 +81,7 @@ private CsvSerializationSchema( this.charset = charset; this.nullValue = nullValue; this.quoteMode = quoteMode; + this.wallClockTimestampTz = wallClockTimestampTz; } public static Builder builder() { @@ -92,6 +98,7 @@ public static class Builder { private Charset charset = StandardCharsets.UTF_8; private String nullValue = ""; private CsvStringQuoteMode quoteMode = CsvStringQuoteMode.MINIMAL; + private boolean wallClockTimestampTz = false; private Builder() {} @@ -140,6 +147,15 @@ public Builder quoteMode(CsvStringQuoteMode quoteMode) { return this; } + /** + * When set to true, TIMESTAMP_TZ fields are serialized as wall-clock local datetime + * (without offset) for timezone-unaware DB sinks such as Doris. + */ + public Builder wallClockTimestampTz(boolean wallClockTimestampTz) { + this.wallClockTimestampTz = wallClockTimestampTz; + return this; + } + public CsvSerializationSchema build() { return new CsvSerializationSchema( seaTunnelRowType, @@ -149,7 +165,8 @@ public CsvSerializationSchema build() { timeFormatter, charset, nullValue, - quoteMode); + quoteMode, + wallClockTimestampTz); } } @@ -194,6 +211,12 @@ private String convert(Object field, SeaTunnelDataType fieldType, int level) return TimeUtils.toString((LocalTime) field, timeFormatter); case TIMESTAMP: return DateTimeUtils.toString((LocalDateTime) field, dateTimeFormatter); + case TIMESTAMP_TZ: + OffsetDateTime odt = (OffsetDateTime) field; + if (wallClockTimestampTz) { + return DateTimeUtils.toString(odt.toLocalDateTime(), dateTimeFormatter); + } + return odt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); case NULL: return ""; case BYTES: diff --git a/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java b/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java index 3bcd9a53c5e9..f5032921c0a5 100644 --- a/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java +++ b/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java @@ -38,6 +38,8 @@ import java.nio.file.Path; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -295,4 +297,52 @@ public void testCsvFileDeserialization() throws Exception { "Amount should be a valid integer at line " + (i + 1)); } } + + @Test + void testTimestampTzRoundTrip() throws IOException { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"ts_tz"}, + new SeaTunnelDataType[] {LocalTimeType.OFFSET_DATE_TIME_TYPE}); + + CsvSerializationSchema ser = + CsvSerializationSchema.builder().seaTunnelRowType(rowType).delimiter(",").build(); + CsvDeserializationSchema deser = + CsvDeserializationSchema.builder().seaTunnelRowType(rowType).delimiter(",").build(); + + OffsetDateTime[] cases = { + OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.ofHours(9)), + OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.ofHours(-8)), + OffsetDateTime.of(2024, 6, 15, 0, 0, 0, 0, ZoneOffset.UTC), + }; + + for (OffsetDateTime original : cases) { + SeaTunnelRow row = new SeaTunnelRow(new Object[] {original}); + byte[] serialized = ser.serialize(row); + SeaTunnelRow deserialized = deser.deserialize(serialized); + OffsetDateTime result = (OffsetDateTime) deserialized.getField(0); + Assertions.assertEquals( + original.toInstant(), result.toInstant(), "Epoch mismatch for " + original); + Assertions.assertEquals( + original.getOffset(), result.getOffset(), "Offset mismatch for " + original); + } + } + + @Test + void testTimestampTzBackwardCompatFallback() throws IOException { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"ts_tz"}, + new SeaTunnelDataType[] {LocalTimeType.OFFSET_DATE_TIME_TYPE}); + CsvDeserializationSchema deser = + CsvDeserializationSchema.builder().seaTunnelRowType(rowType).delimiter(",").build(); + + SeaTunnelRow row = deser.deserialize("2024-01-01 03:00:00".getBytes()); + OffsetDateTime result = (OffsetDateTime) row.getField(0); + Assertions.assertNotNull(result); + Assertions.assertEquals(ZoneOffset.UTC, result.getOffset()); + Assertions.assertEquals( + java.time.LocalDateTime.of(2024, 1, 1, 3, 0, 0).toInstant(ZoneOffset.UTC), + result.toInstant()); + } } diff --git a/seatunnel-formats/seatunnel-format-json/src/main/java/org/apache/seatunnel/format/json/JsonSerializationSchema.java b/seatunnel-formats/seatunnel-format-json/src/main/java/org/apache/seatunnel/format/json/JsonSerializationSchema.java index 2459452f2181..18dc91306a01 100644 --- a/seatunnel-formats/seatunnel-format-json/src/main/java/org/apache/seatunnel/format/json/JsonSerializationSchema.java +++ b/seatunnel-formats/seatunnel-format-json/src/main/java/org/apache/seatunnel/format/json/JsonSerializationSchema.java @@ -68,6 +68,14 @@ public JsonSerializationSchema(SeaTunnelRowType rowType, String nullValue) { this.charset = StandardCharsets.UTF_8; } + public JsonSerializationSchema(SeaTunnelRowType rowType, boolean serializeTimestampTzAsLocal) { + this.rowType = rowType; + this.runtimeConverter = + new RowToJsonConverters(serializeTimestampTzAsLocal) + .createConverter(checkNotNull(rowType)); + this.charset = StandardCharsets.UTF_8; + } + { mapper.configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true); } diff --git a/seatunnel-formats/seatunnel-format-json/src/main/java/org/apache/seatunnel/format/json/RowToJsonConverters.java b/seatunnel-formats/seatunnel-format-json/src/main/java/org/apache/seatunnel/format/json/RowToJsonConverters.java index 13a30442d172..5aaf0a49950d 100644 --- a/seatunnel-formats/seatunnel-format-json/src/main/java/org/apache/seatunnel/format/json/RowToJsonConverters.java +++ b/seatunnel-formats/seatunnel-format-json/src/main/java/org/apache/seatunnel/format/json/RowToJsonConverters.java @@ -53,6 +53,16 @@ public class RowToJsonConverters implements Serializable { private String nullValue; + private final boolean serializeTimestampTzAsLocal; + + public RowToJsonConverters() { + this.serializeTimestampTzAsLocal = false; + } + + public RowToJsonConverters(boolean serializeTimestampTzAsLocal) { + this.serializeTimestampTzAsLocal = serializeTimestampTzAsLocal; + } + public RowToJsonConverter createConverter(SeaTunnelDataType type) { return wrapIntoNullableConverter(createNotNullConverter(type)); } @@ -186,6 +196,17 @@ public JsonNode convert(ObjectMapper mapper, JsonNode reuse, Object value) { } }; case TIMESTAMP_TZ: + if (serializeTimestampTzAsLocal) { + return new RowToJsonConverter() { + @Override + public JsonNode convert(ObjectMapper mapper, JsonNode reuse, Object value) { + return mapper.getNodeFactory() + .textNode( + ISO_LOCAL_DATE_TIME.format( + ((OffsetDateTime) value).toLocalDateTime())); + } + }; + } return new RowToJsonConverter() { @Override public JsonNode convert(ObjectMapper mapper, JsonNode reuse, Object value) { diff --git a/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextDeserializationSchema.java b/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextDeserializationSchema.java index d073f09b6b8b..10c8a7c1d316 100644 --- a/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextDeserializationSchema.java +++ b/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextDeserializationSchema.java @@ -46,8 +46,11 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.time.temporal.TemporalQueries; @@ -257,6 +260,8 @@ private Object convert( return objectArrayList.toArray(new LocalTime[0]); case TIMESTAMP: return objectArrayList.toArray(new LocalDateTime[0]); + case TIMESTAMP_TZ: + return objectArrayList.toArray(new OffsetDateTime[0]); default: throw new SeaTunnelTextFormatException( CommonErrorCode.UNSUPPORTED_DATA_TYPE, @@ -330,6 +335,22 @@ private Object convert( LocalTime localTime = parsedTimestamp.query(TemporalQueries.localTime()); LocalDate localDate = parsedTimestamp.query(TemporalQueries.localDate()); return LocalDateTime.of(localDate, localTime); + case TIMESTAMP_TZ: + try { + return OffsetDateTime.parse(field, DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } catch (DateTimeParseException ignored) { + // Fallback: data written by old SeaTunnel (wall-clock, no offset). + // Parse as LocalDateTime and attach UTC — offset info is already lost. + DateTimeFormatter fallbackFmt = DateTimeUtils.matchDateTimeFormatter(field); + if (fallbackFmt == null) { + throw CommonError.formatDateTimeError(field, fieldName); + } + TemporalAccessor ta = fallbackFmt.parse(field); + return LocalDateTime.of( + ta.query(TemporalQueries.localDate()), + ta.query(TemporalQueries.localTime())) + .atOffset(ZoneOffset.UTC); + } case ROW: Map splitsMap = splitLineBySeaTunnelRowType(field, (SeaTunnelRowType) fieldType, level + 1); diff --git a/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextSerializationSchema.java b/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextSerializationSchema.java index 08f7bd7eab67..f249f5efd100 100644 --- a/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextSerializationSchema.java +++ b/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextSerializationSchema.java @@ -38,6 +38,8 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.Map; import java.util.stream.Collectors; @@ -50,6 +52,8 @@ public class TextSerializationSchema implements SerializationSchema { private final TimeUtils.Formatter timeFormatter; private final Charset charset; private final String nullValue; + /** When true, TIMESTAMP_TZ is serialized as wall-clock (no offset) for DB sinks like Doris. */ + private final boolean wallClockTimestampTz; private TextSerializationSchema( @NonNull SeaTunnelRowType seaTunnelRowType, @@ -58,7 +62,8 @@ private TextSerializationSchema( DateTimeUtils.Formatter dateTimeFormatter, TimeUtils.Formatter timeFormatter, Charset charset, - String nullValue) { + String nullValue, + boolean wallClockTimestampTz) { this.seaTunnelRowType = seaTunnelRowType; this.separators = separators; this.dateFormatter = dateFormatter; @@ -66,6 +71,7 @@ private TextSerializationSchema( this.timeFormatter = timeFormatter; this.charset = charset; this.nullValue = nullValue; + this.wallClockTimestampTz = wallClockTimestampTz; } public static Builder builder() { @@ -81,6 +87,7 @@ public static class Builder { private TimeUtils.Formatter timeFormatter = TimeUtils.Formatter.HH_MM_SS; private Charset charset = StandardCharsets.UTF_8; private String nullValue = ""; + private boolean wallClockTimestampTz = false; private Builder() {} @@ -124,6 +131,16 @@ public Builder nullValue(String nullValue) { return this; } + /** + * When set to true, TIMESTAMP_TZ fields are serialized as wall-clock local datetime + * (dropping the timezone offset). Use this for DB sinks whose column type has no native + * timezone support (e.g. Doris DATETIME). + */ + public Builder wallClockTimestampTz(boolean wallClockTimestampTz) { + this.wallClockTimestampTz = wallClockTimestampTz; + return this; + } + public TextSerializationSchema build() { return new TextSerializationSchema( seaTunnelRowType, @@ -132,7 +149,8 @@ public TextSerializationSchema build() { dateTimeFormatter, timeFormatter, charset, - nullValue); + nullValue, + wallClockTimestampTz); } } @@ -175,6 +193,12 @@ private String convert(Object field, SeaTunnelDataType fieldType, int level) return TimeUtils.toString((LocalTime) field, timeFormatter); case TIMESTAMP: return DateTimeUtils.toString((LocalDateTime) field, dateTimeFormatter); + case TIMESTAMP_TZ: + OffsetDateTime odt = (OffsetDateTime) field; + if (wallClockTimestampTz) { + return DateTimeUtils.toString(odt.toLocalDateTime(), dateTimeFormatter); + } + return odt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); case NULL: return ""; case BYTES: diff --git a/seatunnel-formats/seatunnel-format-text/src/test/java/org/apache/seatunnel/format/text/TextFormatSchemaTest.java b/seatunnel-formats/seatunnel-format-text/src/test/java/org/apache/seatunnel/format/text/TextFormatSchemaTest.java index 3e57e3ce8306..691f71a88e9f 100644 --- a/seatunnel-formats/seatunnel-format-text/src/test/java/org/apache/seatunnel/format/text/TextFormatSchemaTest.java +++ b/seatunnel-formats/seatunnel-format-text/src/test/java/org/apache/seatunnel/format/text/TextFormatSchemaTest.java @@ -34,6 +34,8 @@ import java.io.IOException; import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.Map; @@ -252,6 +254,63 @@ public void testSerializationWithRequireEscapeCharacters() throws Exception { Assertions.assertEquals("tyrantlucifer", seaTunnelRow.getField(1)); } + @Test + void testTimestampTzRoundTrip() throws IOException { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"ts_tz"}, + new SeaTunnelDataType[] {LocalTimeType.OFFSET_DATE_TIME_TYPE}); + + TextSerializationSchema ser = + TextSerializationSchema.builder().seaTunnelRowType(rowType).delimiter(",").build(); + TextDeserializationSchema deser = + TextDeserializationSchema.builder() + .seaTunnelRowType(rowType) + .delimiter(",") + .build(); + + OffsetDateTime[] cases = { + OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.ofHours(9)), + OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.ofHours(-8)), + OffsetDateTime.of(2024, 6, 15, 0, 0, 0, 0, ZoneOffset.UTC), + OffsetDateTime.of(2024, 3, 10, 8, 30, 0, 0, ZoneOffset.ofHoursMinutes(5, 30)), + }; + + for (OffsetDateTime original : cases) { + SeaTunnelRow row = new SeaTunnelRow(new Object[] {original}); + byte[] serialized = ser.serialize(row); + SeaTunnelRow deserialized = deser.deserialize(serialized); + OffsetDateTime result = (OffsetDateTime) deserialized.getField(0); + Assertions.assertEquals( + original.toInstant(), result.toInstant(), "Epoch mismatch for " + original); + Assertions.assertEquals( + original.getOffset(), result.getOffset(), "Offset mismatch for " + original); + } + } + + @Test + void testTimestampTzBackwardCompatFallback() throws IOException { + // Old SeaTunnel wrote TIMESTAMP_TZ as wall-clock ("2024-01-01 03:00:00"). + // The new deserialization must not throw; it falls back to LocalDateTime.atOffset(UTC). + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"ts_tz"}, + new SeaTunnelDataType[] {LocalTimeType.OFFSET_DATE_TIME_TYPE}); + TextDeserializationSchema deser = + TextDeserializationSchema.builder() + .seaTunnelRowType(rowType) + .delimiter(",") + .build(); + + SeaTunnelRow row = deser.deserialize("2024-01-01 03:00:00".getBytes()); + OffsetDateTime result = (OffsetDateTime) row.getField(0); + Assertions.assertNotNull(result); + Assertions.assertEquals(ZoneOffset.UTC, result.getOffset()); + Assertions.assertEquals( + java.time.LocalDateTime.of(2024, 1, 1, 3, 0, 0).toInstant(ZoneOffset.UTC), + result.toInstant()); + } + @Test void testFormatDecimal() { // test 0000.01000 diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFilter.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFilter.java index 934ed230aec8..9446e323d071 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFilter.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFilter.java @@ -44,6 +44,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -156,6 +157,11 @@ private boolean inExpr(InExpression inExpression, Object[] inputFields) { if (((Number) leftValue).doubleValue() == ((Number) rightValue).doubleValue()) { return !inExpression.isNot(); } + } else if (leftValue instanceof OffsetDateTime + && rightValue instanceof OffsetDateTime) { + if (((OffsetDateTime) leftValue).isEqual((OffsetDateTime) rightValue)) { + return !inExpression.isNot(); + } } else if (leftValue.equals(rightValue)) { return !inExpression.isNot(); } @@ -273,6 +279,9 @@ boolean equalsToExpr(Pair pair) { if (leftVal instanceof Number && rightVal instanceof Number) { return ((Number) leftVal).doubleValue() == ((Number) rightVal).doubleValue(); } + if (leftVal instanceof OffsetDateTime && rightVal instanceof OffsetDateTime) { + return ((OffsetDateTime) leftVal).isEqual((OffsetDateTime) rightVal); + } return leftVal.equals(rightVal); } @@ -285,6 +294,9 @@ private boolean notEqualsToExpr(Pair pair) { if (leftVal instanceof Number && rightVal instanceof Number) { return ((Number) leftVal).doubleValue() != ((Number) rightVal).doubleValue(); } + if (leftVal instanceof OffsetDateTime && rightVal instanceof OffsetDateTime) { + return !((OffsetDateTime) leftVal).isEqual((OffsetDateTime) rightVal); + } return !leftVal.equals(rightVal); } @@ -300,6 +312,9 @@ private boolean greaterThanExpr(Pair pair) { if (leftVal instanceof String && rightVal instanceof String) { return ((String) leftVal).compareTo((String) rightVal) > 0; } + if (leftVal instanceof OffsetDateTime && rightVal instanceof OffsetDateTime) { + return ((OffsetDateTime) leftVal).isAfter((OffsetDateTime) rightVal); + } if (leftVal instanceof LocalDateTime && rightVal instanceof LocalDateTime) { return ((LocalDateTime) leftVal).isAfter((LocalDateTime) rightVal); } @@ -328,6 +343,10 @@ private boolean greaterThanEqualsExpr(Pair pair) { if (leftVal instanceof String && rightVal instanceof String) { return ((String) leftVal).compareTo((String) rightVal) >= 0; } + if (leftVal instanceof OffsetDateTime && rightVal instanceof OffsetDateTime) { + return ((OffsetDateTime) leftVal).isAfter((OffsetDateTime) rightVal) + || ((OffsetDateTime) leftVal).isEqual((OffsetDateTime) rightVal); + } if (leftVal instanceof LocalDateTime && rightVal instanceof LocalDateTime) { return ((LocalDateTime) leftVal).isAfter((LocalDateTime) rightVal) || ((LocalDateTime) leftVal).isEqual((LocalDateTime) rightVal); @@ -352,6 +371,9 @@ private boolean minorThanExpr(Pair pair) { if (leftVal == null || rightVal == null) { return false; } + if (leftVal instanceof OffsetDateTime && rightVal instanceof OffsetDateTime) { + return ((OffsetDateTime) leftVal).isBefore((OffsetDateTime) rightVal); + } if (leftVal instanceof LocalDateTime && rightVal instanceof LocalDateTime) { return ((LocalDateTime) leftVal).isBefore((LocalDateTime) rightVal); } @@ -380,6 +402,10 @@ private boolean minorThanEqualsExpr(Pair pair) { if (leftVal == null || rightVal == null) { return false; } + if (leftVal instanceof OffsetDateTime && rightVal instanceof OffsetDateTime) { + return ((OffsetDateTime) leftVal).isBefore((OffsetDateTime) rightVal) + || ((OffsetDateTime) leftVal).isEqual((OffsetDateTime) rightVal); + } if (leftVal instanceof LocalDateTime && rightVal instanceof LocalDateTime) { return ((LocalDateTime) leftVal).isBefore((LocalDateTime) rightVal) || ((LocalDateTime) leftVal).isEqual((LocalDateTime) rightVal); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CastFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CastFunction.java index 9950515cd7f6..03335f6d451e 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CastFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CastFunction.java @@ -52,6 +52,7 @@ public class CastFunction { public static final String DATE = "DATE"; public static final String TIME = "TIME"; public static final String BOOLEAN = "BOOLEAN"; + public static final String TIMESTAMP_TZ = "TIMESTAMP_TZ"; public static final List INT_CAST_TYPE = Arrays.asList( @@ -78,6 +79,8 @@ public class CastFunction { SqlType.TINYINT, SqlType.FLOAT, SqlType.DOUBLE); + public static final List TIMESTAMP_TZ_CAST_TYPES = + Arrays.asList(SqlType.TIMESTAMP, SqlType.TIMESTAMP_TZ, SqlType.BIGINT, SqlType.STRING); public static final List DATETIME_CAST_TYPES = Arrays.asList(SqlType.TIMESTAMP, SqlType.TIMESTAMP_TZ, SqlType.BIGINT); public static final List DATE_CAST_TYPES = @@ -132,6 +135,11 @@ public static SeaTunnelDataType getCastType(SqlType originType, ColDataType c case BYTES: case BINARY: return PrimitiveByteArrayType.INSTANCE; + case TIMESTAMP_TZ: + if (TIMESTAMP_TZ_CAST_TYPES.contains(originType)) { + return LocalTimeType.OFFSET_DATE_TIME_TYPE; + } + break; case TIMESTAMP: case DATETIME: if (DATETIME_CAST_TYPES.contains(originType)) { diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunction.java index 4539acd09378..b179da623995 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunction.java @@ -78,6 +78,9 @@ public static Object dateadd(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).plusYears(count); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).plusYears(count); + } break; case "MONTH": if (datetime instanceof LocalDate) { @@ -86,6 +89,9 @@ public static Object dateadd(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).plusMonths(count); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).plusMonths(count); + } break; case "WEEK": if (datetime instanceof LocalDate) { @@ -94,6 +100,9 @@ public static Object dateadd(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).plusWeeks(count); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).plusWeeks(count); + } break; case "DAY": if (datetime instanceof LocalDate) { @@ -102,6 +111,9 @@ public static Object dateadd(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).plusDays(count); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).plusDays(count); + } break; case "HOUR": if (datetime instanceof LocalTime) { @@ -110,6 +122,9 @@ public static Object dateadd(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).plusHours(count); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).plusHours(count); + } break; case "MINUTE": if (datetime instanceof LocalTime) { @@ -118,6 +133,9 @@ public static Object dateadd(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).plusMinutes(count); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).plusMinutes(count); + } break; case "SECOND": if (datetime instanceof LocalTime) { @@ -126,6 +144,9 @@ public static Object dateadd(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).plusSeconds(count); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).plusSeconds(count); + } break; case "MILLISECOND": if (datetime instanceof LocalTime) { @@ -134,6 +155,9 @@ public static Object dateadd(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).plusNanos(count * 1000_000L); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).plusNanos(count * 1000_000L); + } break; default: throw new TransformException( @@ -169,14 +193,16 @@ public static Long datediff(List args) { || "DAY".equals(datetimeField)) { if (datetime1 instanceof LocalDateTime) { date1 = ((LocalDateTime) datetime1).toLocalDate(); - } - if (datetime1 instanceof LocalDate) { + } else if (datetime1 instanceof OffsetDateTime) { + date1 = ((OffsetDateTime) datetime1).toLocalDate(); + } else if (datetime1 instanceof LocalDate) { date1 = (LocalDate) datetime1; } if (datetime2 instanceof LocalDateTime) { date2 = ((LocalDateTime) datetime2).toLocalDate(); - } - if (datetime2 instanceof LocalDate) { + } else if (datetime2 instanceof OffsetDateTime) { + date2 = ((OffsetDateTime) datetime2).toLocalDate(); + } else if (datetime2 instanceof LocalDate) { date2 = (LocalDate) datetime2; } } @@ -223,10 +249,14 @@ public static Long datediff(List args) { } public static LocalDateTime dateTrunc(List args) { - LocalDateTime datetime = (LocalDateTime) args.get(0); - if (datetime == null) { + Object raw = args.get(0); + if (raw == null) { return null; } + LocalDateTime datetime = + raw instanceof OffsetDateTime + ? ((OffsetDateTime) raw).toLocalDateTime() + : (LocalDateTime) raw; String datetimeField = "DAY"; if (args.size() >= 2) { String df = (String) args.get(1); @@ -310,6 +340,8 @@ private static LocalDate convertToLocalDate(Temporal datetime) { LocalDate localDate = null; if (datetime instanceof LocalDateTime) { localDate = ((LocalDateTime) datetime).toLocalDate(); + } else if (datetime instanceof OffsetDateTime) { + localDate = ((OffsetDateTime) datetime).toLocalDate(); } else if (datetime instanceof LocalDate) { localDate = (LocalDate) datetime; } @@ -521,6 +553,9 @@ public static Integer extract(List args) { if (datetime instanceof LocalDateTime) { return ((LocalDateTime) datetime).getDayOfWeek().getValue(); } + if (datetime instanceof OffsetDateTime) { + return ((OffsetDateTime) datetime).getDayOfWeek().getValue(); + } break; case "DOY": case "DAYOFYEAR": @@ -543,6 +578,10 @@ public static Integer extract(List args) { LocalDate date = ((LocalDateTime) datetime).toLocalDate(); return date.get(WeekFields.ISO.weekBasedYear()); } + if (datetime instanceof OffsetDateTime) { + LocalDate date = ((OffsetDateTime) datetime).toLocalDate(); + return date.get(WeekFields.ISO.weekBasedYear()); + } break; case "MILLENNIUM": if (datetime instanceof LocalDate) { @@ -553,6 +592,10 @@ public static Integer extract(List args) { int year = ((LocalDateTime) datetime).getYear(); return (year > 0) ? (year - 1) / 1000 + 1 : year / 1000; } + if (datetime instanceof OffsetDateTime) { + int year = ((OffsetDateTime) datetime).getYear(); + return (year > 0) ? (year - 1) / 1000 + 1 : year / 1000; + } break; default: throw new TransformException( @@ -587,6 +630,8 @@ private static LocalTime convertToLocalTime(Temporal datetime) { LocalTime localTime = null; if (datetime instanceof LocalDateTime) { localTime = ((LocalDateTime) datetime).toLocalTime(); + } else if (datetime instanceof OffsetDateTime) { + localTime = ((OffsetDateTime) datetime).toLocalTime(); } else if (datetime instanceof LocalTime) { localTime = (LocalTime) datetime; } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java index 95e2c564e865..7917613b2a96 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java @@ -31,7 +31,11 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -130,6 +134,8 @@ public static Object castAs(List args) { case "LONG": if (v1 instanceof String) { return Long.parseLong(v1.toString()); + } else if (v1 instanceof OffsetDateTime) { + return ((OffsetDateTime) v1).toInstant().toEpochMilli(); } else if (v1 instanceof Number) { return ((Number) v1).longValue(); } else { @@ -151,6 +157,9 @@ public static Object castAs(List args) { if (v1 instanceof LocalDateTime) { return v1; } + if (v1 instanceof OffsetDateTime) { + return ((OffsetDateTime) v1).toLocalDateTime(); + } if (v1 instanceof Long) { Instant instant = Instant.ofEpochMilli(((Long) v1).longValue()); ZoneId zone = ZoneId.systemDefault(); @@ -159,10 +168,37 @@ public static Object castAs(List args) { throw new TransformException( CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION, String.format("Unsupported CAST AS type: %s", v2)); + case "TIMESTAMP_TZ": + if (v1 instanceof OffsetDateTime) { + return v1; + } + if (v1 instanceof LocalDateTime) { + return ((LocalDateTime) v1).atOffset(ZoneOffset.UTC); + } + if (v1 instanceof Long) { + return OffsetDateTime.ofInstant( + Instant.ofEpochMilli(((Long) v1).longValue()), ZoneOffset.UTC); + } + if (v1 instanceof String) { + try { + return OffsetDateTime.parse( + (String) v1, DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } catch (DateTimeParseException ignored) { + return LocalDateTime.parse( + (String) v1, DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .atOffset(ZoneOffset.UTC); + } + } + throw new TransformException( + CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION, + String.format("Unsupported CAST AS type: %s", v2)); case "DATE": if (v1 instanceof LocalDateTime) { return ((LocalDateTime) v1).toLocalDate(); } + if (v1 instanceof OffsetDateTime) { + return ((OffsetDateTime) v1).toLocalDate(); + } if (v1 instanceof LocalDate) { return v1; } @@ -180,6 +216,9 @@ public static Object castAs(List args) { if (v1 instanceof LocalDateTime) { return ((LocalDateTime) v1).toLocalTime(); } + if (v1 instanceof OffsetDateTime) { + return ((OffsetDateTime) v1).toLocalTime(); + } if (v1 instanceof LocalTime) { return v1; } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunctionTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunctionTest.java index 42a7d1faa120..b980df25d33f 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunctionTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunctionTest.java @@ -34,6 +34,8 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -343,6 +345,41 @@ public void testCastAsTimestampAndDateTimeVariants() { Assertions.assertEquals(now.toLocalTime(), time); } + @Test + public void testCastAsFromOffsetDateTime() { + OffsetDateTime odt = OffsetDateTime.of(2024, 3, 15, 10, 30, 0, 0, ZoneOffset.ofHours(9)); + + // CAST(odt AS TIMESTAMP) → wall-clock LocalDateTime + List args = new ArrayList<>(); + args.add(odt); + args.add("TIMESTAMP"); + Object ts = SystemFunction.castAs(args); + Assertions.assertTrue(ts instanceof LocalDateTime); + Assertions.assertEquals(odt.toLocalDateTime(), ts); + + // CAST(odt AS DATE) → wall-clock LocalDate + args.clear(); + args.add(odt); + args.add("DATE"); + Object date = SystemFunction.castAs(args); + Assertions.assertEquals(odt.toLocalDate(), date); + + // CAST(odt AS TIME) → wall-clock LocalTime + args.clear(); + args.add(odt); + args.add("TIME"); + Object time = SystemFunction.castAs(args); + Assertions.assertEquals(odt.toLocalTime(), time); + + // CAST(odt AS BIGINT) → epoch millis (UTC) + args.clear(); + args.add(odt); + args.add("BIGINT"); + Object epochMillis = SystemFunction.castAs(args); + Assertions.assertTrue(epochMillis instanceof Long); + Assertions.assertEquals(odt.toInstant().toEpochMilli(), epochMillis); + } + @Test public void testCastAsBooleanFromNumberStringAndBoolean() { List args = new ArrayList<>(); diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/utils/OffsetDateTimeUtils.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/utils/OffsetDateTimeUtils.java index ea5a58654497..30136eb14591 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/utils/OffsetDateTimeUtils.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/utils/OffsetDateTimeUtils.java @@ -20,7 +20,6 @@ import org.apache.spark.sql.types.DecimalType; import java.math.BigDecimal; -import java.math.BigInteger; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -29,22 +28,28 @@ public class OffsetDateTimeUtils { public static final String LOGICAL_TIMESTAMP_WITH_OFFSET_TYPE_FLAG = "logical_timestamp_with_offset_type"; - // epochMilli length 13, timezone offset length 5 - public static final DecimalType OFFSET_DATETIME_WITH_DECIMAL = new DecimalType(18, 5); + // Shift applied to totalSeconds so the fractional part is always positive. + // Offset range: [-43200, +50400] seconds → shifted range: [56800, 150400] (6 digits, always > + // 0) + static final int OFFSET_SHIFT = 100_000; + + // scale=6 to hold the 6-digit shifted offset; precision=20 for 13-digit epochMilli + 6 decimal + public static final DecimalType OFFSET_DATETIME_WITH_DECIMAL = new DecimalType(20, 6); public static BigDecimal toBigDecimal(OffsetDateTime time) { - return new BigDecimal( - time.toInstant().toEpochMilli() + "." + time.getOffset().getTotalSeconds()); + long epochMilli = time.toInstant().toEpochMilli(); + int shiftedOffset = time.getOffset().getTotalSeconds() + OFFSET_SHIFT; + // epochMilli may be negative; shiftedOffset is always a positive 6-digit integer. + // String.format guarantees no sign in the fractional part. + return new BigDecimal(epochMilli + "." + String.format("%06d", shiftedOffset)); } public static OffsetDateTime toOffsetDateTime(BigDecimal timeWithDecimal) { - BigInteger epochMilli = - timeWithDecimal.unscaledValue().divide(BigInteger.TEN.pow(timeWithDecimal.scale())); - BigInteger offset = - timeWithDecimal - .unscaledValue() - .remainder(BigInteger.TEN.pow(timeWithDecimal.scale())); - return Instant.ofEpochMilli(epochMilli.longValue()) - .atOffset(ZoneOffset.ofTotalSeconds(offset.intValue())); + BigDecimal normalized = timeWithDecimal.setScale(6); + long epochMilli = normalized.longValue(); // truncates toward zero — correct for ±epoch + BigDecimal fractional = normalized.subtract(BigDecimal.valueOf(epochMilli)).abs(); + int shiftedOffset = fractional.movePointRight(6).intValue(); + return Instant.ofEpochMilli(epochMilli) + .atOffset(ZoneOffset.ofTotalSeconds(shiftedOffset - OFFSET_SHIFT)); } } diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/test/java/org/apache/seatunnel/translation/spark/utils/OffsetDateTimeUtilsTest.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/test/java/org/apache/seatunnel/translation/spark/utils/OffsetDateTimeUtilsTest.java new file mode 100644 index 000000000000..2b8e2dc03405 --- /dev/null +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/test/java/org/apache/seatunnel/translation/spark/utils/OffsetDateTimeUtilsTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.translation.spark.utils; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class OffsetDateTimeUtilsTest { + + private static void assertRoundTrip(OffsetDateTime original) { + BigDecimal encoded = OffsetDateTimeUtils.toBigDecimal(original); + OffsetDateTime decoded = OffsetDateTimeUtils.toOffsetDateTime(encoded); + assertEquals(original.toInstant(), decoded.toInstant(), "Epoch mismatch for " + original); + assertEquals(original.getOffset(), decoded.getOffset(), "Offset mismatch for " + original); + } + + @Test + void positiveOffset() { + assertRoundTrip(OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.ofHours(9))); + } + + @Test + void negativeOffset() { + // This was broken before the fix: -08:00 produced "epochMilli.-28800" (invalid BigDecimal) + assertRoundTrip(OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.ofHours(-8))); + } + + @Test + void utcOffset() { + assertRoundTrip(OffsetDateTime.of(2024, 6, 15, 0, 0, 0, 0, ZoneOffset.UTC)); + } + + @Test + void maxPositiveOffset() { + assertRoundTrip(OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, ZoneOffset.ofHours(14))); + } + + @Test + void maxNegativeOffset() { + assertRoundTrip(OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, ZoneOffset.ofHours(-12))); + } + + @Test + void fractionalOffset() { + // India Standard Time: +05:30 + assertRoundTrip( + OffsetDateTime.of(2024, 3, 10, 8, 30, 0, 0, ZoneOffset.ofHoursMinutes(5, 30))); + } +} From 1863e337921e1c9e93422d12b72180905d3da1c1 Mon Sep 17 00:00:00 2001 From: loupipalien Date: Mon, 29 Jun 2026 22:21:31 +0800 Subject: [PATCH 078/375] [Improve][Transform-V2][Embedding]Enhance multimodal embeddings (#9996) --- docs/en/transforms/embedding.md | 54 ++- docs/zh/transforms/embedding.md | 54 ++- .../embedding_transform_multimodal.conf | 78 ++++ .../embedding/EmbeddingTransform.java | 98 +++-- .../nlpmodel/embedding/SrcField.java | 84 +++++ .../{FieldSpec.java => SrcFieldSpec.java} | 65 ++-- .../nlpmodel/embedding/VectorFieldSpec.java | 101 +++++ .../multimodal/MultimodalFieldValue.java | 43 +-- .../embedding/remote/doubao/DoubaoModel.java | 64 ++-- .../embedding/DoubaoMultimodalModelTest.java | 355 +++++++++++++----- .../transform/embedding/FieldSpecTest.java | 114 ------ .../embedding/MultimodalConfigTest.java | 112 +++++- .../embedding/VectorFieldSpecTest.java | 200 ++++++++++ 13 files changed, 1066 insertions(+), 356 deletions(-) create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/SrcField.java rename seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/{FieldSpec.java => SrcFieldSpec.java} (65%) create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/VectorFieldSpec.java delete mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/FieldSpecTest.java create mode 100644 seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/VectorFieldSpecTest.java diff --git a/docs/en/transforms/embedding.md b/docs/en/transforms/embedding.md index 98dfa0af01be..6823332d4e8e 100644 --- a/docs/en/transforms/embedding.md +++ b/docs/en/transforms/embedding.md @@ -149,6 +149,58 @@ vectorization_fields { } ``` +**Multi-field Mixing Multimodal Vectorization:** +> Note: Currently, only the `DOUBAO` provider supports multimodal data processing. +```hocon +vectorization_fields { + # Multi-field text + multi_field_text_vector = [product_name, description] + + # Multi-field image + multi_field_image_vector = [ + { + field = product_image_url + modality = jpeg + format = url + }, + { + field = thumbnail_image + modality = png + format = url + } + ] + + # Multi-field video + multi_field_video_vector = [ + { + field = product_video_url + modality = mp4 + format = url + }, + { + field = promotional_video + modality = mov + format = url + } + ] + + # Multi-field mix multimodal + multi_field_mix_vector = [ + product_name, + { + field = product_image_url + modality = jpeg + format = url + }, + { + field = product_video_url + modality = mp4 + format = url + } + ] +} +``` + **Field Specification Formats:** **Supported Modality Types:** @@ -162,7 +214,7 @@ vectorization_fields { - `binary` - Binary data format **Automatic Modality Detection:** -When `modality` is not explicitly specified and `format` is not `binary`, the system automatically detects the modality type based on the file suffix of the field value: +When `modality` is not explicitly specified and `format` is `url`, the system automatically detects the modality type based on the file suffix of the field value: > **Important:** When using multimodal fields (image or video), ensure your model provider supports multimodal embedding. Image and video fields must contain valid URLs or binary data. Currently, `DOUBAO` provider supports multimodal data processing. diff --git a/docs/zh/transforms/embedding.md b/docs/zh/transforms/embedding.md index b8ace6ca6b8b..e80f96a575a8 100644 --- a/docs/zh/transforms/embedding.md +++ b/docs/zh/transforms/embedding.md @@ -134,6 +134,58 @@ vectorization_fields { } ``` +**多字段混合多模态向量化：** +> 注意: 目前，仅 `DOUBAO` 提供商支持多模态数据处理 +```hocon +vectorization_fields { + # 多字段文本 + multi_field_text_vector = [product_name, description] + + # 多字段图片 + multi_field_image_vector = [ + { + field = product_image_url + modality = jpeg + format = url + }, + { + field = thumbnail_image + modality = png + format = url + } + ] + + # 多字段视频 + multi_field_video_vector = [ + { + field = product_video_url + modality = mp4 + format = url + }, + { + field = promotional_video + modality = mov + format = url + } + ] + + # 多字段混合多模态 + multi_field_mix_vector = [ + product_name, + { + field = product_image_url + modality = jpeg + format = url + }, + { + field = product_video_url + modality = mp4 + format = url + } + ] +} +``` + **字段规范格式：** **支持的模态类型：** @@ -147,7 +199,7 @@ vectorization_fields { - `binary` - 二进制数据格式 **自动模态检测：** -当未显式指定 `modality` 且 `format` 不是 `binary` 时，系统会根据字段值的文件后缀自动检测模态类型： +当未显式指定 `modality` 且 `format` 是 `url` 时，系统会根据字段值的文件后缀自动检测模态类型： > **重要：** 使用多模态字段（图片或视频）时，请确保您的模型提供商支持多模态 embedding。图片和视频字段必须包含有效的 URL 或二进制数据。目前，`DOUBAO` 提供商支持多模态数据处理。 diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-1/src/test/resources/embedding_transform_multimodal.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-1/src/test/resources/embedding_transform_multimodal.conf index efc72731457d..cada8b7ca27b 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-1/src/test/resources/embedding_transform_multimodal.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-1/src/test/resources/embedding_transform_multimodal.conf @@ -154,6 +154,48 @@ transform { } product_name_vector = product_name + + multi_field_text_vector = [product_name, description] + + multi_field_image_vector = [ + { + field = product_image_url + modality = jpeg + format = url + }, + { + field = thumbnail_image + modality = png + format = url + } + ] + + multi_field_video_vector = [ + { + field = product_video_url + modality = mp4 + format = url + }, + { + field = promotional_video + modality = mov + format = url + } + ] + + multi_field_mix_vector = [ + product_name, + { + field = product_image_url + modality = jpeg + format = url + }, + { + field = product_video_url + modality = mp4 + format = url + } + ] } plugin_output = "multimodal_embedding_output" @@ -219,6 +261,42 @@ sink { } ] }, + { + field_name = multi_field_text_vector + field_type = float_vector + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = multi_field_image_vector + field_type = float_vector + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = multi_field_video_vector + field_type = float_vector + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = multi_field_mix_vector + field_type = float_vector + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, { field_name = category field_type = string diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java index 6e7ba72a3aa8..bcf09b67bb6a 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java @@ -52,22 +52,22 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; @Slf4j public class EmbeddingTransform extends MultipleFieldOutputTransform { private final ReadonlyConfig config; - private List fieldOriginalIndexes; private transient Model model; private Integer dimension; private boolean isMultimodalFields = false; - private Map fieldSpecMap; + private Map> fieldSpecMap; private List fieldNames; private final Map> binaryFileCache = new ConcurrentHashMap<>(); @@ -204,30 +204,35 @@ public void open() { } private void initOutputFields(SeaTunnelRowType inputRowType, ReadonlyConfig config) { - Map fieldSpecMap = new HashMap<>(); - List fieldNames = new ArrayList<>(); Map fieldsConfig = config.get(EmbeddingTransformConfig.VECTORIZATION_FIELDS); if (fieldsConfig == null || fieldsConfig.isEmpty()) { throw new IllegalArgumentException("vectorization_fields configuration is required"); } - for (Map.Entry field : fieldsConfig.entrySet()) { - FieldSpec fieldSpec = new FieldSpec(field); - log.info("Field spec: {}", fieldSpec.toString()); - String srcField = fieldSpec.getFieldName(); - int srcFieldIndex; - try { - srcFieldIndex = inputRowType.indexOf(srcField); - } catch (IllegalArgumentException e) { - throw TransformCommonError.cannotFindInputFieldError(getPluginName(), srcField); - } - if (fieldSpec.isMultimodalField()) { - isMultimodalFields = true; + List fieldNames = new ArrayList<>(); + Map> fieldSpecMap = new LinkedHashMap<>(); + for (Map.Entry fieldConfig : fieldsConfig.entrySet()) { + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(fieldConfig); + log.info("Vector field spec: {}", vectorFieldSpec); + List srcFieldNames = + vectorFieldSpec.getSrcFieldSpecs().stream() + .map(SrcFieldSpec::getFieldName) + .collect(Collectors.toList()); + List srcFieldIndexes = new ArrayList<>(); + for (String srcFieldName : srcFieldNames) { + try { + srcFieldIndexes.add(inputRowType.indexOf(srcFieldName)); + } catch (IllegalArgumentException e) { + throw TransformCommonError.cannotFindInputFieldsError( + getPluginName(), srcFieldNames); + } } - fieldSpecMap.put(srcFieldIndex, fieldSpec); - fieldNames.add(field.getKey()); + fieldSpecMap.put(vectorFieldSpec, srcFieldIndexes); + fieldNames.add(vectorFieldSpec.getFieldName()); } + this.isMultimodalFields = + fieldSpecMap.keySet().stream().anyMatch(VectorFieldSpec::isMultimodalField); this.fieldSpecMap = fieldSpecMap; this.fieldNames = fieldNames; } @@ -239,19 +244,28 @@ protected Object[] getOutputFieldValues(SeaTunnelRowAccessor inputRow) { if (MetadataUtil.isBinaryFormat(inputRow)) { return vectorizationBinaryRow(inputRow); } - Set fieldOriginalIndexes = fieldSpecMap.keySet(); - Object[] fieldValues = new Object[fieldOriginalIndexes.size()]; - List vectorization; + + Set vectorFieldSpecs = fieldSpecMap.keySet(); + Object[] fieldValues = new Object[vectorFieldSpecs.size()]; int i = 0; - for (Integer fieldOriginalIndex : fieldOriginalIndexes) { - FieldSpec fieldSpec = fieldSpecMap.get(fieldOriginalIndex); - Object value = inputRow.getField(fieldOriginalIndex); + for (VectorFieldSpec vectorFieldSpec : vectorFieldSpecs) { + List srcFieldSpecs = vectorFieldSpec.getSrcFieldSpecs(); + List srcFieldIndexes = fieldSpecMap.get(vectorFieldSpec); + List srcFields = new ArrayList<>(); + for (int j = 0; j < srcFieldSpecs.size(); j++) { + srcFields.add( + new SrcField( + srcFieldSpecs.get(j), + inputRow.getField(srcFieldIndexes.get(j)))); + } fieldValues[i++] = - isMultimodalFields ? new MultimodalFieldValue(fieldSpec, value) : value; + isMultimodalFields + ? new MultimodalFieldValue(srcFields) + : srcFields.get(0).getFieldValue(); } - vectorization = model.vectorization(fieldValues); + List vectorization = model.vectorization(fieldValues); return vectorization.toArray(); } catch (Exception e) { throw new RuntimeException("Failed to data vectorization", e); @@ -289,32 +303,34 @@ public boolean isMultimodalFields() { /** Process a row in binary format: [data, relativePath, partIndex] */ private Object[] vectorizationBinaryRow(SeaTunnelRowAccessor inputRow) throws Exception { - byte[] completeData = processBinaryRow(inputRow); if (completeData == null) { return null; } - Set fieldOriginalIndexes = fieldSpecMap.keySet(); - Object[] fieldValues = new Object[fieldOriginalIndexes.size()]; + + Set vectorFieldSpecs = fieldSpecMap.keySet(); + Object[] fieldValues = new Object[vectorFieldSpecs.size()]; int i = 0; - for (Integer fieldOriginalIndex : fieldOriginalIndexes) { - FieldSpec fieldSpec = fieldSpecMap.get(fieldOriginalIndex); - if (fieldSpec.isBinary()) { - fieldValues[i++] = new MultimodalFieldValue(fieldSpec, completeData); - } else { - log.warn( - "Non-binary field {} configured in binary format data", - fieldSpec.getFieldName()); - fieldValues[i++] = null; + for (VectorFieldSpec vectorFieldSpec : vectorFieldSpecs) { + List srcFieldSpecs = vectorFieldSpec.getSrcFieldSpecs(); + List srcFields = new ArrayList<>(); + for (SrcFieldSpec srcFieldSpec : srcFieldSpecs) { + if (srcFieldSpec.isBinary()) { + srcFields.add(new SrcField(srcFieldSpec, completeData)); + } else { + log.warn( + "Non-binary field {} configured in binary format data", + srcFieldSpec.getFieldName()); + } } + fieldValues[i++] = srcFields.isEmpty() ? null : new MultimodalFieldValue(srcFields); } try { return model.vectorization(fieldValues).toArray(); } catch (Exception e) { - throw new RuntimeException( - "Failed to vectorize binary data for file: " + inputRow.toString(), e); + throw new RuntimeException("Failed to vectorize binary data for file: " + inputRow, e); } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/SrcField.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/SrcField.java new file mode 100644 index 000000000000..908ad5aad18e --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/SrcField.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel.embedding; + +import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.ModalityType; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.io.Serializable; +import java.util.Base64; + +@Data +@Slf4j +public class SrcField implements Serializable { + + private static final long serialVersionUID = 1L; + + private SrcFieldSpec fieldSpec; + + private Object fieldValue; + + public SrcField(SrcFieldSpec spec, Object value) { + // create a new object avoid to mutate original src field spec + this.fieldSpec = + new SrcFieldSpec( + spec.getFieldName(), + spec.getModalityType(), + spec.getPayloadFormat(), + spec.isModalityTypeExplicitlyConfigured()); + this.fieldValue = value; + determineModalityType(); + } + + /** + * Determine the actual modality type based on field spec and value. The configured modality + * type is always respected when it was explicitly provided by the user. Auto-detection from the + * value suffix only happens for URL payloads whose modality type was not explicitly configured, + * so that plain text values are never misclassified as image/video by their content. + */ + private void determineModalityType() { + if (fieldSpec.isModalityTypeExplicitlyConfigured() || !fieldSpec.isUrl()) { + return; + } + if (fieldValue != null) { + String valueStr = fieldValue.toString(); + ModalityType detectedType = ModalityType.fromFileSuffix(valueStr); + if (detectedType != null) { + log.debug( + "Auto-detected modality type '{}' from value: {}", detectedType, valueStr); + fieldSpec.setModalityType(detectedType); + } + } + } + + public String toBase64() { + if (fieldSpec == null || !fieldSpec.isBinary()) { + throw new IllegalArgumentException("Payload format must be binary"); + } + if (fieldValue == null) { + throw new IllegalArgumentException("Binary data cannot be null or empty"); + } + if (fieldValue instanceof byte[]) { + return Base64.getEncoder().encodeToString((byte[]) fieldValue); + } else { + return Base64.getEncoder().encodeToString(fieldValue.toString().getBytes()); + } + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/FieldSpec.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/SrcFieldSpec.java similarity index 65% rename from seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/FieldSpec.java rename to seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/SrcFieldSpec.java index 94ee65329ebc..cf5a6ae5c81b 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/FieldSpec.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/SrcFieldSpec.java @@ -26,7 +26,7 @@ import java.util.Map; @Data -public class FieldSpec implements Serializable { +public class SrcFieldSpec implements Serializable { private static final long serialVersionUID = 1L; @@ -34,51 +34,31 @@ public class FieldSpec implements Serializable { private ModalityType modalityType; private PayloadFormat payloadFormat; - public FieldSpec(String fieldName) { - this.fieldName = fieldName; - this.modalityType = ModalityType.TEXT; - this.payloadFormat = PayloadFormat.TEXT; - } - - public FieldSpec(Map.Entry fieldConfig) { - String outputFieldName = fieldConfig.getKey(); - if (outputFieldName == null) { - throw new IllegalArgumentException("Field spec cannot be null"); - } - Object fieldValue = fieldConfig.getValue(); - try { - if (fieldValue instanceof String) { - parseBasicFieldSpec((String) fieldValue); - } else { - Map fieldSpecConfig = (Map) fieldValue; - parseMultimodalFieldSpec(fieldSpecConfig); - } - } catch (Exception e) { - String errorMessage = - String.format( - "Invalid field spec for output field '%s': %s", - outputFieldName, fieldConfig); - throw new IllegalArgumentException(errorMessage, e); - } - } + /** + * Whether the modality type was explicitly configured by the user. When false, the actual + * modality type can be auto-detected from the runtime value suffix; when true, the configured + * modality type must be respected and never overridden. + */ + private boolean modalityTypeExplicitlyConfigured; /** Parse basic field spec: just the field name, defaults to TEXT modality and default format */ - private void parseBasicFieldSpec(String fieldSpec) { - if (fieldSpec == null || fieldSpec.trim().isEmpty()) { - throw new IllegalArgumentException("Field spec cannot be null or empty"); + public SrcFieldSpec(String fieldName) { + if (fieldName == null || fieldName.trim().isEmpty()) { + throw new IllegalArgumentException("Field name cannot be null or empty"); } - this.fieldName = fieldSpec.trim(); + this.fieldName = fieldName.trim(); this.modalityType = ModalityType.TEXT; this.payloadFormat = PayloadFormat.TEXT; + this.modalityTypeExplicitlyConfigured = false; } /** * Parse multimodal field spec: field name, modality, and format Supports both formats: 1. * Separate modality and format */ - private void parseMultimodalFieldSpec(Map fieldConfig) { + public SrcFieldSpec(Map fieldConfig) { if (fieldConfig == null || fieldConfig.isEmpty()) { - throw new IllegalArgumentException("Field configuration cannot be null or empty"); + throw new IllegalArgumentException("Field config cannot be null or empty"); } Object fieldNameObj = fieldConfig.get("field"); @@ -94,12 +74,14 @@ private void parseMultimodalFieldSpec(Map fieldConfig) { Object modalityObj = fieldConfig.get("modality"); if (modalityObj != null) { this.modalityType = ModalityType.ofName(modalityObj.toString()); + this.modalityTypeExplicitlyConfigured = true; Object formatObj = fieldConfig.get("format"); if (formatObj != null) { this.payloadFormat = PayloadFormat.ofName(formatObj.toString()); } } else { this.modalityType = ModalityType.TEXT; + this.modalityTypeExplicitlyConfigured = false; Object formatObj = fieldConfig.get("format"); if (formatObj != null) { this.payloadFormat = PayloadFormat.ofName(formatObj.toString()); @@ -109,11 +91,22 @@ private void parseMultimodalFieldSpec(Map fieldConfig) { } } - public boolean isMultimodalField() { - return !ModalityType.TEXT.equals(modalityType); + public SrcFieldSpec( + String fieldName, + ModalityType modalityType, + PayloadFormat payloadFormat, + boolean modalityTypeExplicitlyConfigured) { + this.fieldName = fieldName; + this.modalityType = modalityType; + this.payloadFormat = payloadFormat; + this.modalityTypeExplicitlyConfigured = modalityTypeExplicitlyConfigured; } public boolean isBinary() { return PayloadFormat.BINARY.equals(payloadFormat); } + + public boolean isUrl() { + return PayloadFormat.URL.equals(payloadFormat); + } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/VectorFieldSpec.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/VectorFieldSpec.java new file mode 100644 index 000000000000..bf8bd937f0ca --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/VectorFieldSpec.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.nlpmodel.embedding; + +import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; + +import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.ModalityType; + +import lombok.Data; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +@Data +public class VectorFieldSpec implements Serializable { + + private static final long serialVersionUID = 1L; + + private String fieldName; + + private List srcFieldSpecs; + + public VectorFieldSpec(Map.Entry fieldConfig) { + this.fieldName = fieldConfig.getKey(); + if (StringUtils.isBlank(fieldName)) { + throw new IllegalArgumentException("Field config name cannot be null or empty"); + } + Object fieldConfigValue = fieldConfig.getValue(); + if (fieldConfigValue == null) { + throw new IllegalArgumentException( + "Field config value cannot be null for field: " + fieldName); + } + + srcFieldSpecs = new ArrayList<>(); + try { + if (fieldConfigValue instanceof String) { + srcFieldSpecs.add(new SrcFieldSpec((String) fieldConfigValue)); + } else if (fieldConfigValue instanceof Map) { + srcFieldSpecs.add(new SrcFieldSpec((Map) fieldConfigValue)); + } else { + List fieldConfigValues = (List) fieldConfigValue; + for (Object fieldConfigValueItem : fieldConfigValues) { + if (fieldConfigValueItem instanceof String) { + srcFieldSpecs.add(new SrcFieldSpec((String) fieldConfigValueItem)); + } else if (fieldConfigValueItem instanceof Map) { + srcFieldSpecs.add( + new SrcFieldSpec((Map) fieldConfigValueItem)); + } else { + String errorMessage = + String.format( + "Invalid field spec for output field '%s': %s", + fieldName, fieldConfig); + throw new IllegalArgumentException(errorMessage); + } + } + } + } catch (Exception e) { + String errorMessage = + String.format( + "Invalid field spec for output field '%s': %s", fieldName, fieldConfig); + throw new IllegalArgumentException(errorMessage, e); + } + } + + public boolean isMultimodalField() { + return srcFieldSpecs.size() > 1 + || srcFieldSpecs.stream() + .anyMatch(f -> !ModalityType.TEXT.equals(f.getModalityType())); + } + + @Override + public boolean equals(Object object) { + if (this == object) return true; + if (object == null || getClass() != object.getClass()) return false; + VectorFieldSpec that = (VectorFieldSpec) object; + return Objects.equals(fieldName, that.fieldName); + } + + @Override + public int hashCode() { + return Objects.hash(fieldName); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/MultimodalFieldValue.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/MultimodalFieldValue.java index 01c3e5040323..195db4a4201f 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/MultimodalFieldValue.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/MultimodalFieldValue.java @@ -17,54 +17,21 @@ package org.apache.seatunnel.transform.nlpmodel.embedding.multimodal; -import org.apache.seatunnel.transform.nlpmodel.embedding.FieldSpec; +import org.apache.seatunnel.transform.nlpmodel.embedding.SrcField; import lombok.Getter; -import lombok.extern.slf4j.Slf4j; import java.io.Serializable; -import java.util.Base64; +import java.util.List; -@Slf4j @Getter public class MultimodalFieldValue implements Serializable { private static final long serialVersionUID = 1L; - private final FieldSpec fieldSpec; - private final Object value; + private final List srcFields; - public MultimodalFieldValue(FieldSpec fieldSpec, Object value) { - this.value = value; - fieldSpec.setModalityType(determineModalityType(fieldSpec, value)); - this.fieldSpec = fieldSpec; - } - - /** - * Determine the actual modality type based on field spec and value If not binary format, - * analyze the value suffix to determine modality type - */ - private ModalityType determineModalityType(FieldSpec fieldSpec, Object value) { - - if (fieldSpec.isBinary()) { - return fieldSpec.getModalityType(); - } - if (value != null) { - String valueStr = value.toString(); - ModalityType detectedType = ModalityType.fromFileSuffix(valueStr); - if (detectedType != null) { - log.debug( - "Auto-detected modality type '{}' from value: {}", detectedType, valueStr); - return detectedType; - } - } - return fieldSpec.getModalityType(); - } - - public String toBase64() { - if (value == null) { - throw new IllegalArgumentException("Binary data cannot be null or empty"); - } - return Base64.getEncoder().encodeToString(value.toString().getBytes()); + public MultimodalFieldValue(List srcFields) { + this.srcFields = srcFields; } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java index e02bbe74df88..f175b7c9b2da 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java @@ -28,7 +28,8 @@ import org.apache.seatunnel.transform.nlpmodel.ModelInvocationException; import org.apache.seatunnel.transform.nlpmodel.ModelInvocationOptions; import org.apache.seatunnel.transform.nlpmodel.ProviderAdapter; -import org.apache.seatunnel.transform.nlpmodel.embedding.FieldSpec; +import org.apache.seatunnel.transform.nlpmodel.embedding.SrcField; +import org.apache.seatunnel.transform.nlpmodel.embedding.SrcFieldSpec; import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.ModalityType; import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.MultimodalFieldValue; import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.MultimodalModel; @@ -45,6 +46,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; public class DoubaoModel extends MultimodalModel { @@ -143,7 +145,7 @@ protected List> textVector(Object[] fields) throws IOException { public List> multimodalVector(Object[] fields) throws IOException { if (singleVectorizedInputNumber > 1) { throw new IllegalArgumentException( - "Doubao does not support batch multimodal vectorization in a single request. "); + "Doubao does not support batch multimodal vectorization in a single request."); } List> vectors = new ArrayList<>(); for (Object field : fields) { @@ -188,7 +190,10 @@ public Integer dimension() throws IOException { ? multimodalVector( new Object[] { new MultimodalFieldValue( - new FieldSpec(DIMENSION_EXAMPLE), DIMENSION_EXAMPLE) + Collections.singletonList( + new SrcField( + new SrcFieldSpec(DIMENSION_EXAMPLE), + DIMENSION_EXAMPLE))) }) .get(0) .size() @@ -355,35 +360,40 @@ public ObjectNode multimodalBody(MultimodalFieldValue field) { ObjectNode requestNode = OBJECT_MAPPER.createObjectNode(); requestNode.put("model", model); requestNode.put("encoding_format", "float"); - ArrayNode inputDatas = OBJECT_MAPPER.createArrayNode(); - inputDatas.add(inputRawData(field)); - requestNode.set("input", inputDatas); + ArrayNode inputNode = OBJECT_MAPPER.createArrayNode(); + inputNode.addAll(inputRawData(field)); + requestNode.set("input", inputNode); return requestNode; } - protected ObjectNode inputRawData(MultimodalFieldValue field) { - ObjectNode rawDataNode = OBJECT_MAPPER.createObjectNode(); - FieldSpec fieldSpec = field.getFieldSpec(); - String fieldValue = field.getValue().toString().trim(); - ModalityType fieldSpecModalityType = fieldSpec.getModalityType(); - String modalityParamName = getModalityParamName(fieldSpecModalityType); - rawDataNode.put("type", modalityParamName); - if (ModalityType.TEXT == fieldSpecModalityType) { - rawDataNode.put(modalityParamName, fieldValue); - return rawDataNode; - } + protected List inputRawData(MultimodalFieldValue field) { + List rawDataNodes = new ArrayList<>(); + List srcFields = field.getSrcFields(); + for (SrcField srcField : srcFields) { + ObjectNode rawDataNode = OBJECT_MAPPER.createObjectNode(); + String fieldValue = srcField.getFieldValue().toString().trim(); + ModalityType fieldSpecModalityType = srcField.getFieldSpec().getModalityType(); + String modalityParamName = getModalityParamName(fieldSpecModalityType); + rawDataNode.put("type", modalityParamName); + if (ModalityType.TEXT == fieldSpecModalityType) { + rawDataNode.put(modalityParamName, fieldValue); + rawDataNodes.add(rawDataNode); + continue; + } - if (fieldSpec.isBinary()) { - fieldValue = - String.format( - BASE64_PARAM_TEMPLATE, - fieldSpecModalityType.getGroup().name().toLowerCase(), - fieldSpecModalityType.getName(), - field.toBase64()); + if (srcField.getFieldSpec().isBinary()) { + fieldValue = + String.format( + BASE64_PARAM_TEMPLATE, + fieldSpecModalityType.getGroup().name().toLowerCase(), + fieldSpecModalityType.getName(), + srcField.toBase64()); + } + rawDataNode.set( + modalityParamName, OBJECT_MAPPER.createObjectNode().put("url", fieldValue)); + rawDataNodes.add(rawDataNode); } - rawDataNode.set(modalityParamName, OBJECT_MAPPER.createObjectNode().put("url", fieldValue)); - - return rawDataNode; + return rawDataNodes; } private String getModalityParamName(ModalityType inputType) { diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java index b9ae009e8ae0..0489a76ace4e 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java @@ -17,40 +17,59 @@ package org.apache.seatunnel.transform.embedding; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.seatunnel.transform.nlpmodel.embedding.FieldSpec; +import org.apache.seatunnel.transform.nlpmodel.embedding.SrcField; +import org.apache.seatunnel.transform.nlpmodel.embedding.VectorFieldSpec; import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.ModalityType; import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.MultimodalFieldValue; import org.apache.seatunnel.transform.nlpmodel.embedding.remote.doubao.DoubaoModel; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; public class DoubaoMultimodalModelTest { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private DoubaoModel model; - @Test - void testMultimodalBodyWithText() throws IOException { - DoubaoModel model = + @BeforeEach + void setUp() { + this.model = new DoubaoModel( "test-api-key", "doubao-embedding-vision", "https://ark.cn-beijing.volces.com/api/v3/embeddings", 1); + } + @AfterEach + void tearDown() throws IOException { + if (model != null) { + model.close(); + } + } + + @Test + void testMultimodalBodyWithText() { Map.Entry textFieldEntry = - new java.util.AbstractMap.SimpleEntry<>("text_vector", "Hello world"); - FieldSpec fieldSpec = new FieldSpec(textFieldEntry); + new java.util.AbstractMap.SimpleEntry<>("text_vector", "text_field"); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(textFieldEntry); MultimodalFieldValue multimodalFieldValue = - new MultimodalFieldValue(fieldSpec, "Hello world"); + new MultimodalFieldValue( + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), "Hello world"))); ObjectNode result = model.multimodalBody(multimodalFieldValue); @@ -63,36 +82,29 @@ void testMultimodalBodyWithText() throws IOException { Assertions.assertEquals("Hello world", inputNode.get("text").asText()); Assertions.assertFalse(inputNode.has("image_url")); Assertions.assertFalse(inputNode.has("video_url")); - - model.close(); } /** - * { "model" : "doubao-embedding-vision", "encoding_format" : "float", "input" : [ { "type" : - * "image_url", "image_url" : { "url" : - * "https://ck-test.tos-cn-beijing.volces.com/vlm/pexels-photo-27163466.jpeg" } }] } + * { "model": "doubao-embedding-vision", "encoding_format": "float", "input": [ { "type": + * "image_url", "image_url": { "url": + * "https://ck-test.tos-cn-beijing.volces.com/vlm/pexels-photo-27163466.jpeg" } } ] } */ @Test - void testMultimodalBodyWithImage() throws IOException { - DoubaoModel model = - new DoubaoModel( - "test-api-key", - "doubao-embedding-vision", - "https://ark.cn-beijing.volces.com/api/v3/embeddings", - 1); - + void testMultimodalBodyWithImage() { Map imageFieldConfig = new HashMap<>(); imageFieldConfig.put("field", "image_field"); imageFieldConfig.put("modality", "jpeg"); imageFieldConfig.put("format", "url"); - Map.Entry imageFieldEntry = new java.util.AbstractMap.SimpleEntry<>("image_vector", imageFieldConfig); - FieldSpec fieldSpec = new FieldSpec(imageFieldEntry); + + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(imageFieldEntry); MultimodalFieldValue multimodalFieldValue = new MultimodalFieldValue( - fieldSpec, - "https://ck-test.tos-cn-beijing.volces.com/vlm/pexels-photo-27163466.jpeg"); + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), + "https://ck-test.tos-cn-beijing.volces.com/vlm/pexels-photo-27163466.jpeg"))); ObjectNode result = model.multimodalBody(multimodalFieldValue); @@ -110,33 +122,28 @@ void testMultimodalBodyWithImage() throws IOException { inputNode.get("image_url").get("url").asText()); Assertions.assertFalse(inputNode.has("text")); Assertions.assertFalse(inputNode.has("video_url")); - - model.close(); } /** - * { "model" : "doubao-embedding-vision", "encoding_format" : "float", "input" : [ { "type" : - * "video_url", "video_url" : { "url" : "https://example.com/video.mp4" } } ] } + * { "model": "doubao-embedding-vision", "encoding_format": "float", "input": [ { "type": + * "video_url", "video_url": { "url": "https://example.com/video.mp4" } } ] } */ @Test - void testMultimodalBodyWithVideo() throws IOException { - DoubaoModel model = - new DoubaoModel( - "test-api-key", - "doubao-embedding-vision", - "https://ark.cn-beijing.volces.com/api/v3/embeddings", - 1); - + void testMultimodalBodyWithVideo() { Map videoFieldConfig = new HashMap<>(); videoFieldConfig.put("field", "video_field"); videoFieldConfig.put("modality", "mP4"); videoFieldConfig.put("format", "url"); - Map.Entry videoFieldEntry = new java.util.AbstractMap.SimpleEntry<>("video_vector", videoFieldConfig); - FieldSpec fieldSpec = new FieldSpec(videoFieldEntry); + + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(videoFieldEntry); MultimodalFieldValue multimodalFieldValue = - new MultimodalFieldValue(fieldSpec, "https://example.com/video.mp4"); + new MultimodalFieldValue( + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), + "https://example.com/video.mp4"))); ObjectNode result = model.multimodalBody(multimodalFieldValue); @@ -151,8 +158,6 @@ void testMultimodalBodyWithVideo() throws IOException { "https://example.com/video.mp4", inputNode.get("video_url").get("url").asText()); Assertions.assertFalse(inputNode.has("text")); Assertions.assertFalse(inputNode.has("image_url")); - - model.close(); } /** @@ -160,50 +165,149 @@ void testMultimodalBodyWithVideo() throws IOException { * f"data:image/;base64,{base64_image}" } } */ @Test - void testMultimodalBodyWithBinaryImage() throws IOException { - DoubaoModel model = - new DoubaoModel( - "test-api-key", - "doubao-embedding-vision-250615", - "https://ark.cn-beijing.volces.com/api/v3/embeddings", - 1); - + void testMultimodalBodyWithBinaryImage() { Map binaryImageFieldConfig = new HashMap<>(); binaryImageFieldConfig.put("field", "binary_image_field"); binaryImageFieldConfig.put("modality", "png"); binaryImageFieldConfig.put("format", "binary"); - Map.Entry binaryImageFieldEntry = new java.util.AbstractMap.SimpleEntry<>( "binary_image_vector", binaryImageFieldConfig); - FieldSpec fieldSpec = new FieldSpec(binaryImageFieldEntry); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(binaryImageFieldEntry); byte[] mockImageData = "mock-image-data".getBytes(java.nio.charset.StandardCharsets.UTF_8); MultimodalFieldValue multimodalFieldValue = - new MultimodalFieldValue(fieldSpec, mockImageData); + new MultimodalFieldValue( + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), mockImageData))); ObjectNode result = model.multimodalBody(multimodalFieldValue); - - Assertions.assertEquals("doubao-embedding-vision-250615", result.get("model").asText()); + Assertions.assertEquals("doubao-embedding-vision", result.get("model").asText()); Assertions.assertEquals("float", result.get("encoding_format").asText()); Assertions.assertEquals(1, result.get("input").size()); ObjectNode inputNode = (ObjectNode) result.get("input").get(0); Assertions.assertEquals("image_url", inputNode.get("type").asText()); Assertions.assertTrue(inputNode.has("image_url")); + Assertions.assertTrue( + inputNode + .get("image_url") + .get("url") + .asText() + .endsWith(Base64.getEncoder().encodeToString(mockImageData))); + } + + /** + * { "model": "doubao-embedding-vision", "encoding_format": "float", "input": [ { "type": + * "text", "text": "Hello world 1" }, { "type": "text", "text": "Hello world 2" } ] } + */ + @Test + void testMultimodalBodyWithSameModalityList() { + Map.Entry vectorFieldEntry = + new java.util.AbstractMap.SimpleEntry<>( + "same_multimodal_vector", Arrays.asList("text_field_1", "text_field_2")); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(vectorFieldEntry); + MultimodalFieldValue multimodalFieldValue = + new MultimodalFieldValue( + Arrays.asList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), "Hello world 1"), + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(1), + "Hello world 2"))); + + ObjectNode result = model.multimodalBody(multimodalFieldValue); + Assertions.assertEquals("doubao-embedding-vision", result.get("model").asText()); + Assertions.assertEquals("float", result.get("encoding_format").asText()); + Assertions.assertEquals(2, result.get("input").size()); - model.close(); + ObjectNode inputNode = (ObjectNode) result.get("input").get(0); + Assertions.assertEquals("text", inputNode.get("type").asText()); + Assertions.assertEquals("Hello world 1", inputNode.get("text").asText()); + Assertions.assertFalse(inputNode.has("image_url")); + Assertions.assertFalse(inputNode.has("video_url")); + + inputNode = (ObjectNode) result.get("input").get(1); + Assertions.assertEquals("text", inputNode.get("type").asText()); + Assertions.assertEquals("Hello world 2", inputNode.get("text").asText()); + Assertions.assertFalse(inputNode.has("image_url")); + Assertions.assertFalse(inputNode.has("video_url")); } + /** + * { "model": "doubao-embedding-vision", "encoding_format": "float", "input": [ { "type": + * "text", "text": "Hello world" }, { "type": "image_url", "image_url": { "url": + * "https://ck-test.tos-cn-beijing.volces.com/vlm/pexels-photo-27163466.jpeg" } }, { "type": + * "video_url", "video_url": { "url": "https://example.com/video.mp4" } } ] } + */ @Test - void testParseMultimodalVectorResponseSuccess() throws IOException { - DoubaoModel model = - new DoubaoModel( - "test-api-key", - "doubao-embedding-vision", - "https://ark.cn-beijing.volces.com/api/v3/embeddings", - 1); + void testMultimodalBodyWithDifferentModalityList() { + Object textFieldConfig = "text_field"; + if (ThreadLocalRandom.current().nextBoolean()) { + Map textFieldConfigMap = new HashMap<>(); + textFieldConfigMap.put("field", "text_field"); + textFieldConfigMap.put("modality", "text"); + textFieldConfigMap.put("format", "text"); + textFieldConfig = textFieldConfigMap; + } + Map imageFieldConfig = new HashMap<>(); + imageFieldConfig.put("field", "image_field"); + imageFieldConfig.put("modality", "jpeg"); + imageFieldConfig.put("format", "url"); + Map videoFieldConfig = new HashMap<>(); + videoFieldConfig.put("field", "video_field"); + videoFieldConfig.put("modality", "mp4"); + videoFieldConfig.put("format", "url"); + Map.Entry vectorFieldEntry = + new java.util.AbstractMap.SimpleEntry<>( + "different_multimodal_vector", + Arrays.asList(textFieldConfig, imageFieldConfig, videoFieldConfig)); + + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(vectorFieldEntry); + MultimodalFieldValue multimodalFieldValue = + new MultimodalFieldValue( + Arrays.asList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), "Hello world"), + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(1), + "https://ck-test.tos-cn-beijing.volces.com/vlm/pexels-photo-27163466.jpeg"), + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(2), + "https://example.com/video.mp4"))); + ObjectNode result = model.multimodalBody(multimodalFieldValue); + Assertions.assertEquals("doubao-embedding-vision", result.get("model").asText()); + Assertions.assertEquals("float", result.get("encoding_format").asText()); + Assertions.assertEquals(3, result.get("input").size()); + + ObjectNode inputNode = (ObjectNode) result.get("input").get(0); + Assertions.assertEquals("text", inputNode.get("type").asText()); + Assertions.assertEquals("Hello world", inputNode.get("text").asText()); + Assertions.assertFalse(inputNode.has("image_url")); + Assertions.assertFalse(inputNode.has("video_url")); + + inputNode = (ObjectNode) result.get("input").get(1); + Assertions.assertEquals("image_url", inputNode.get("type").asText()); + Assertions.assertTrue(inputNode.has("image_url")); + Assertions.assertEquals( + "https://ck-test.tos-cn-beijing.volces.com/vlm/pexels-photo-27163466.jpeg", + inputNode.get("image_url").get("url").asText()); + Assertions.assertFalse(inputNode.has("text")); + Assertions.assertFalse(inputNode.has("video_url")); + + inputNode = (ObjectNode) result.get("input").get(2); + Assertions.assertEquals("video_url", inputNode.get("type").asText()); + Assertions.assertTrue(inputNode.has("video_url")); + Assertions.assertEquals( + "https://example.com/video.mp4", inputNode.get("video_url").get("url").asText()); + Assertions.assertFalse(inputNode.has("text")); + Assertions.assertFalse(inputNode.has("image_url")); + } + + @Test + void testParseMultimodalVectorResponseSuccess() throws IOException { String successResponse = "{\n" + " \"created\": 1743575029,\n" @@ -236,79 +340,136 @@ void testParseMultimodalVectorResponseSuccess() throws IOException { Assertions.assertEquals(-0.318359375f, result.get(2), 0.0001f); Assertions.assertEquals(0.255859375f, result.get(3), 0.0001f); Assertions.assertEquals(1.5f, result.get(4), 0.0001f); - - model.close(); } @Test - void testUrlAutoDetectModality() throws IOException { - DoubaoModel model = - new DoubaoModel( - "test-api-key", - "doubao-embedding-vision", - "https://ark.cn-beijing.volces.com/api/v3/embeddings", - 1); - + void testUrlAutoDetectModality() { + // Explicitly configured modality (png) must be respected and NOT overridden by the runtime + // value suffix (.jpg). Map fieldConfig = new HashMap<>(); fieldConfig.put("field", "image_field"); fieldConfig.put("format", "url"); fieldConfig.put("modality", "png"); - Map.Entry fieldEntry = + Map.Entry imageFieldEntry = new java.util.AbstractMap.SimpleEntry<>("image_vector", fieldConfig); - FieldSpec fieldSpec = new FieldSpec(fieldEntry); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(imageFieldEntry); MultimodalFieldValue multimodalFieldValue = - new MultimodalFieldValue(fieldSpec, "https://example.com/photo.jpg"); + new MultimodalFieldValue( + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), + "https://example.com/photo.jpg"))); Assertions.assertEquals( - ModalityType.JPEG, multimodalFieldValue.getFieldSpec().getModalityType()); + ModalityType.PNG, + multimodalFieldValue.getSrcFields().get(0).getFieldSpec().getModalityType()); ObjectNode result = model.multimodalBody(multimodalFieldValue); ObjectNode inputNode = (ObjectNode) result.get("input").get(0); Assertions.assertEquals("image_url", inputNode.get("type").asText()); + // No modality configured -> auto-detect from the value suffix (.jpg -> jpeg). Map fieldConfig2 = new HashMap<>(); fieldConfig2.put("field", "image_field"); fieldConfig2.put("format", "url"); - fieldEntry = new java.util.AbstractMap.SimpleEntry<>("image_vector", fieldConfig2); - fieldSpec = new FieldSpec(fieldEntry); - - multimodalFieldValue = new MultimodalFieldValue(fieldSpec, "https://example.com/photo.jpg"); + imageFieldEntry = new java.util.AbstractMap.SimpleEntry<>("image_vector", fieldConfig2); + vectorFieldSpec = new VectorFieldSpec(imageFieldEntry); + multimodalFieldValue = + new MultimodalFieldValue( + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), + "https://example.com/photo.jpg"))); Assertions.assertEquals( - ModalityType.JPEG, multimodalFieldValue.getFieldSpec().getModalityType()); + ModalityType.JPEG, + multimodalFieldValue.getSrcFields().get(0).getFieldSpec().getModalityType()); result = model.multimodalBody(multimodalFieldValue); inputNode = (ObjectNode) result.get("input").get(0); Assertions.assertEquals("image_url", inputNode.get("type").asText()); + } - model.close(); + @Test + void testExplicitModalityNotOverriddenBySuffix() { + // Regression: modality = png + runtime value photo.jpg should stay png. + Map fieldConfig = new HashMap<>(); + fieldConfig.put("field", "image_field"); + fieldConfig.put("format", "url"); + fieldConfig.put("modality", "png"); + Map.Entry imageFieldEntry = + new java.util.AbstractMap.SimpleEntry<>("image_vector", fieldConfig); + + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(imageFieldEntry); + SrcField srcField = + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), "https://example.com/photo.jpg"); + + Assertions.assertEquals(ModalityType.PNG, srcField.getFieldSpec().getModalityType()); + Assertions.assertTrue(srcField.getFieldSpec().isModalityTypeExplicitlyConfigured()); } @Test - void testBinaryAutoDetectModality() throws IOException { - DoubaoModel model = - new DoubaoModel( - "test-api-key", - "doubao-embedding-vision", - "https://ark.cn-beijing.volces.com/api/v3/embeddings", - 1); + void testMixedConfigTextFieldWithImageSuffixStaysText() { + // Regression: in a mixed multimodal job, a plain text field whose value happens to end with + // a known image suffix (foo.jpg) must NOT be misclassified as an image. + Map imageFieldConfig = new HashMap<>(); + imageFieldConfig.put("field", "image_field"); + imageFieldConfig.put("modality", "jpeg"); + imageFieldConfig.put("format", "url"); + + Map.Entry entry = + new java.util.AbstractMap.SimpleEntry<>( + "mix_vector", Arrays.asList("text_field", imageFieldConfig)); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + + // first src field is the plain text field, with a value that ends with .jpg + SrcField textSrcField = + new SrcField(vectorFieldSpec.getSrcFieldSpecs().get(0), "this is foo.jpg"); + Assertions.assertEquals(ModalityType.TEXT, textSrcField.getFieldSpec().getModalityType()); + Assertions.assertFalse(textSrcField.getFieldSpec().isModalityTypeExplicitlyConfigured()); + + MultimodalFieldValue multimodalFieldValue = + new MultimodalFieldValue(Collections.singletonList(textSrcField)); + ObjectNode result = model.multimodalBody(multimodalFieldValue); + ObjectNode inputNode = (ObjectNode) result.get("input").get(0); + Assertions.assertEquals("text", inputNode.get("type").asText()); + Assertions.assertEquals("this is foo.jpg", inputNode.get("text").asText()); + } + + @Test + void testNoModalityPlainTextValueStaysText() { + // No modality configured and value has no recognizable suffix -> stays TEXT. + Map.Entry entry = + new java.util.AbstractMap.SimpleEntry<>("text_vector", "hello world"); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + SrcField srcField = new SrcField(vectorFieldSpec.getSrcFieldSpecs().get(0), "hello world"); + + Assertions.assertEquals(ModalityType.TEXT, srcField.getFieldSpec().getModalityType()); + Assertions.assertFalse(srcField.getFieldSpec().isModalityTypeExplicitlyConfigured()); + } + @Test + void testBinaryAutoDetectModality() { Map fieldConfig = new HashMap<>(); fieldConfig.put("field", "image_field"); fieldConfig.put("format", "binary"); fieldConfig.put("modality", "png"); - Map.Entry fieldEntry = + Map.Entry imageFieldEntry = new java.util.AbstractMap.SimpleEntry<>("image_vector", fieldConfig); - FieldSpec fieldSpec = new FieldSpec(fieldEntry); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(imageFieldEntry); MultimodalFieldValue multimodalFieldValue = - new MultimodalFieldValue(fieldSpec, "https://example.com/photo.jpg"); + new MultimodalFieldValue( + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), + "https://example.com/photo.jpg"))); Assertions.assertEquals( - ModalityType.PNG, multimodalFieldValue.getFieldSpec().getModalityType()); + ModalityType.PNG, + multimodalFieldValue.getSrcFields().get(0).getFieldSpec().getModalityType()); ObjectNode result = model.multimodalBody(multimodalFieldValue); ObjectNode inputNode = (ObjectNode) result.get("input").get(0); Assertions.assertEquals("image_url", inputNode.get("type").asText()); - - model.close(); } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/FieldSpecTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/FieldSpecTest.java deleted file mode 100644 index c97372f8fe2f..000000000000 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/FieldSpecTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.transform.embedding; - -import org.apache.seatunnel.transform.nlpmodel.embedding.FieldSpec; -import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.ModalityType; -import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.PayloadFormat; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.AbstractMap; -import java.util.HashMap; -import java.util.Map; - -public class FieldSpecTest { - - @Test - void testMapEntryConstructorWithStringValue() { - Map.Entry entry = - new AbstractMap.SimpleEntry<>("book_intro_vector", "book_intro"); - FieldSpec fieldSpec = new FieldSpec(entry); - Assertions.assertEquals("book_intro", fieldSpec.getFieldName()); - Assertions.assertEquals(ModalityType.TEXT, fieldSpec.getModalityType()); - Assertions.assertEquals(PayloadFormat.TEXT, fieldSpec.getPayloadFormat()); - Assertions.assertFalse(fieldSpec.isMultimodalField()); - Assertions.assertFalse(fieldSpec.isBinary()); - } - - @Test - void testMapEntryConstructorWithStringValueTrimming() { - Map.Entry entry = - new AbstractMap.SimpleEntry<>("book_intro_vector", " book_intro "); - FieldSpec fieldSpec = new FieldSpec(entry); - Assertions.assertEquals("book_intro", fieldSpec.getFieldName()); - Assertions.assertEquals(ModalityType.TEXT, fieldSpec.getModalityType()); - Assertions.assertEquals(PayloadFormat.TEXT, fieldSpec.getPayloadFormat()); - } - - @Test - void testMapEntryConstructorWithNullKey() { - Map.Entry entry = new AbstractMap.SimpleEntry<>(null, "book_intro"); - IllegalArgumentException exception = - Assertions.assertThrows(IllegalArgumentException.class, () -> new FieldSpec(entry)); - Assertions.assertTrue(exception.getMessage().contains("Field spec cannot be null")); - } - - @Test - void testMapEntryConstructorWithEmpty() { - Map.Entry entry = new AbstractMap.SimpleEntry<>("book_intro_vector", null); - IllegalArgumentException exception = - Assertions.assertThrows(IllegalArgumentException.class, () -> new FieldSpec(entry)); - Assertions.assertTrue( - exception.getMessage().contains("Invalid field spec for output field")); - - Map.Entry entry2 = new AbstractMap.SimpleEntry<>("book_intro_vector", ""); - exception = - Assertions.assertThrows( - IllegalArgumentException.class, () -> new FieldSpec(entry2)); - Assertions.assertTrue( - exception.getMessage().contains("Invalid field spec for output field")); - } - - @Test - void testMapEntryConstructorWithMapValue() { - - Map fieldConfig = new HashMap<>(); - fieldConfig.put("field", "book_image"); - fieldConfig.put("modality", "jpeg"); - fieldConfig.put("format", "binary"); - - Map.Entry entry = new AbstractMap.SimpleEntry<>("book_field", fieldConfig); - - FieldSpec fieldSpec = new FieldSpec(entry); - - Assertions.assertEquals("book_image", fieldSpec.getFieldName()); - Assertions.assertEquals(ModalityType.JPEG, fieldSpec.getModalityType()); - Assertions.assertEquals(PayloadFormat.BINARY, fieldSpec.getPayloadFormat()); - Assertions.assertTrue(fieldSpec.isMultimodalField()); - Assertions.assertTrue(fieldSpec.isBinary()); - } - - @Test - void testMapEntryConstructorWithMapValueNoModality() { - Map fieldConfig = new HashMap<>(); - fieldConfig.put("field", "book_intro"); - fieldConfig.put("modality", "text"); - fieldConfig.put("format", "text"); - - Map.Entry entry = new AbstractMap.SimpleEntry<>("book_field", fieldConfig); - - FieldSpec fieldSpec = new FieldSpec(entry); - - Assertions.assertEquals("book_intro", fieldSpec.getFieldName()); - Assertions.assertEquals(ModalityType.TEXT, fieldSpec.getModalityType()); - Assertions.assertEquals(PayloadFormat.TEXT, fieldSpec.getPayloadFormat()); - Assertions.assertFalse(fieldSpec.isMultimodalField()); - } -} diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/MultimodalConfigTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/MultimodalConfigTest.java index ba5eae1f716c..aecba8948966 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/MultimodalConfigTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/MultimodalConfigTest.java @@ -35,7 +35,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; public class MultimodalConfigTest { @@ -44,7 +47,10 @@ private CatalogTable createTestCatalogTable() { PhysicalColumn.of("text_field", BasicType.STRING_TYPE, 255L, true, null, ""), PhysicalColumn.of("image_field", BasicType.STRING_TYPE, 255L, true, null, ""), PhysicalColumn.of("video_field", BasicType.STRING_TYPE, 255L, true, null, ""), - PhysicalColumn.of("mixed_field", BasicType.STRING_TYPE, 255L, true, null, "") + PhysicalColumn.of("mixed_field", BasicType.STRING_TYPE, 255L, true, null, ""), + PhysicalColumn.of("text_field_2", BasicType.STRING_TYPE, 255L, true, null, ""), + PhysicalColumn.of("image_field_2", BasicType.STRING_TYPE, 255L, true, null, ""), + PhysicalColumn.of("video_field_2", BasicType.STRING_TYPE, 255L, true, null, ""), }; TableSchema tableSchema = TableSchema.builder().columns(Arrays.asList(columns)).build(); @@ -187,6 +193,110 @@ void testIsMultimodalFieldsDetectionWithMixedFields() { Assertions.assertTrue(transform.isMultimodalFields()); } + @Test + void testIsMultimodalFieldsDetectionWithMixedListFields() { + CatalogTable catalogTable = createTestCatalogTable(); + + Map configMap = new HashMap<>(); + configMap.put(ModelTransformConfig.MODEL_PROVIDER.key(), ModelProvider.DOUBAO.name()); + configMap.put(ModelTransformConfig.MODEL.key(), "doubao-embedding-vision"); + configMap.put(ModelTransformConfig.API_KEY.key(), "test-api-key"); + configMap.put(ModelTransformConfig.API_PATH.key(), "https://api.test.com/embeddings"); + + Map vectorizationFields = new HashMap<>(); + // Text type + List textFieldConfigList = Arrays.asList("text_field", "text_field_2"); + if (ThreadLocalRandom.current().nextBoolean()) { + textFieldConfigList = new ArrayList<>(); + Map textFieldConfig = new HashMap<>(); + textFieldConfig.put("field", "text_field"); + textFieldConfig.put("modality", "text"); + textFieldConfigList.add(textFieldConfig); + Map textFieldConfig2 = new HashMap<>(); + textFieldConfig2.put("field", "text_field_2"); + textFieldConfig2.put("modality", "text"); + textFieldConfigList.add(textFieldConfig2); + } + vectorizationFields.put("text_vector", textFieldConfigList); + + // Image type + List> imageFieldConfigList = new ArrayList<>(); + Map imageFieldConfig = new HashMap<>(); + imageFieldConfig.put("field", "image_field"); + imageFieldConfig.put("modality", "png"); + imageFieldConfigList.add(imageFieldConfig); + Map imageFieldConfig2 = new HashMap<>(); + imageFieldConfig2.put("field", "image_field_2"); + imageFieldConfig2.put("modality", "png"); + imageFieldConfigList.add(imageFieldConfig2); + vectorizationFields.put("image_vector", imageFieldConfigList); + + // Video type + List> videoFieldConfigList = new ArrayList<>(); + Map videoFieldConfig = new HashMap<>(); + videoFieldConfig.put("field", "video_field"); + videoFieldConfig.put("modality", "mp4"); + videoFieldConfig.put("format", "url"); + videoFieldConfigList.add(videoFieldConfig); + Map videoFieldConfig2 = new HashMap<>(); + videoFieldConfig2.put("field", "video_field_2"); + videoFieldConfig2.put("modality", "mp4"); + videoFieldConfig2.put("format", "url"); + videoFieldConfigList.add(videoFieldConfig2); + vectorizationFields.put("video_vector", videoFieldConfigList); + + configMap.put(EmbeddingTransformConfig.VECTORIZATION_FIELDS.key(), vectorizationFields); + + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + EmbeddingTransform transform = new EmbeddingTransform(config, catalogTable); + Assertions.assertNotNull(transform); + Assertions.assertTrue(transform.isMultimodalFields()); + } + + @Test + void testIsMultimodalFieldsDetectionWithMixedTypeFields() { + CatalogTable catalogTable = createTestCatalogTable(); + + Map configMap = new HashMap<>(); + configMap.put(ModelTransformConfig.MODEL_PROVIDER.key(), ModelProvider.DOUBAO.name()); + configMap.put(ModelTransformConfig.MODEL.key(), "doubao-embedding-vision"); + configMap.put(ModelTransformConfig.API_KEY.key(), "test-api-key"); + configMap.put(ModelTransformConfig.API_PATH.key(), "https://api.test.com/embeddings"); + + Map vectorizationFields = new LinkedHashMap<>(); + // Video type + Map videoFieldConfig = new HashMap<>(); + videoFieldConfig.put("field", "video_field"); + videoFieldConfig.put("modality", "mp4"); + videoFieldConfig.put("format", "url"); + vectorizationFields.put("video_vector", videoFieldConfig); + + // Image type + List> imageFieldConfigList = new ArrayList<>(); + Map imageFieldConfig = new HashMap<>(); + imageFieldConfig.put("field", "image_field"); + imageFieldConfig.put("modality", "png"); + imageFieldConfigList.add(imageFieldConfig); + Map imageFieldConfig2 = new HashMap<>(); + imageFieldConfig2.put("field", "image_field_2"); + imageFieldConfig2.put("modality", "png"); + imageFieldConfigList.add(imageFieldConfig2); + vectorizationFields.put("image_vector", imageFieldConfigList); + + // Text type + Object textFieldConfig = "text_field"; + vectorizationFields.put("text_vector", textFieldConfig); + + configMap.put(EmbeddingTransformConfig.VECTORIZATION_FIELDS.key(), vectorizationFields); + + ReadonlyConfig config = ReadonlyConfig.fromMap(configMap); + + EmbeddingTransform transform = new EmbeddingTransform(config, catalogTable); + Assertions.assertNotNull(transform); + Assertions.assertTrue(transform.isMultimodalFields()); + } + @Test void testMultimodalModelValidationFailure() { CatalogTable catalogTable = createTestCatalogTable(); diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/VectorFieldSpecTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/VectorFieldSpecTest.java new file mode 100644 index 000000000000..42677ead4675 --- /dev/null +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/VectorFieldSpecTest.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.embedding; + +import org.apache.seatunnel.transform.nlpmodel.embedding.SrcFieldSpec; +import org.apache.seatunnel.transform.nlpmodel.embedding.VectorFieldSpec; +import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.ModalityType; +import org.apache.seatunnel.transform.nlpmodel.embedding.multimodal.PayloadFormat; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class VectorFieldSpecTest { + + @Test + void testMapEntryConstructorWithStringValue() { + Map.Entry entry = + new AbstractMap.SimpleEntry<>("book_intro_vector", "book_intro"); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + Assertions.assertEquals("book_intro_vector", vectorFieldSpec.getFieldName()); + SrcFieldSpec srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(0); + Assertions.assertEquals("book_intro", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.TEXT, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.TEXT, srcFieldSpec.getPayloadFormat()); + Assertions.assertFalse(vectorFieldSpec.isMultimodalField()); + Assertions.assertFalse(srcFieldSpec.isBinary()); + } + + @Test + void testMapEntryConstructorWithStringValueTrimming() { + Map.Entry entry = + new AbstractMap.SimpleEntry<>("book_intro_vector", " book_intro "); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + SrcFieldSpec srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(0); + Assertions.assertEquals("book_intro", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.TEXT, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.TEXT, srcFieldSpec.getPayloadFormat()); + } + + @Test + void testMapEntryConstructorWithNullKey() { + Map.Entry entry = new AbstractMap.SimpleEntry<>(null, "book_intro"); + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, () -> new VectorFieldSpec(entry)); + Assertions.assertTrue( + exception.getMessage().contains("Field config name cannot be null or empty")); + } + + @Test + void testMapEntryConstructorWithEmpty() { + Map.Entry entry = new AbstractMap.SimpleEntry<>("book_intro_vector", null); + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, () -> new VectorFieldSpec(entry)); + Assertions.assertTrue(exception.getMessage().contains("Field config value cannot be null")); + + Map.Entry entry2 = new AbstractMap.SimpleEntry<>("book_intro_vector", ""); + exception = + Assertions.assertThrows( + IllegalArgumentException.class, () -> new VectorFieldSpec(entry2)); + Assertions.assertTrue( + exception.getMessage().contains("Invalid field spec for output field")); + } + + @Test + void testMapEntryConstructorWithMapValue() { + Map fieldConfig = new HashMap<>(); + fieldConfig.put("field", "book_image"); + fieldConfig.put("modality", "jpeg"); + fieldConfig.put("format", "binary"); + + Map.Entry entry = new AbstractMap.SimpleEntry<>("book_field", fieldConfig); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + SrcFieldSpec srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(0); + + Assertions.assertEquals("book_image", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.JPEG, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.BINARY, srcFieldSpec.getPayloadFormat()); + Assertions.assertTrue(vectorFieldSpec.isMultimodalField()); + Assertions.assertTrue(srcFieldSpec.isBinary()); + } + + @Test + void testMapEntryConstructorWithMapValueNoModality() { + Map fieldConfig = new HashMap<>(); + fieldConfig.put("field", "book_intro"); + fieldConfig.put("modality", "text"); + fieldConfig.put("format", "text"); + + Map.Entry entry = new AbstractMap.SimpleEntry<>("book_field", fieldConfig); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + SrcFieldSpec srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(0); + + Assertions.assertEquals("book_intro", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.TEXT, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.TEXT, srcFieldSpec.getPayloadFormat()); + Assertions.assertFalse(vectorFieldSpec.isMultimodalField()); + } + + @Test + void testMapEntryConstructorWithInvalidListValue() { + List textFieldConfig = Arrays.asList("text_field_1", "text_field_2"); + Map imageFieldConfig = new HashMap<>(); + imageFieldConfig.put("field", "image_field"); + imageFieldConfig.put("modality", "jpeg"); + imageFieldConfig.put("format", "url"); + + Map.Entry entry = + new AbstractMap.SimpleEntry<>( + "vector_field", Arrays.asList(textFieldConfig, imageFieldConfig)); + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, () -> new VectorFieldSpec(entry)); + Assertions.assertTrue( + exception.getMessage().contains("Invalid field spec for output field")); + } + + @Test + void testMapEntryConstructorWithSameModalityListValue() { + Map.Entry entry = + new AbstractMap.SimpleEntry<>( + "vector_field", Arrays.asList("text_field_1", "text_field_2")); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + Assertions.assertEquals("vector_field", vectorFieldSpec.getFieldName()); + Assertions.assertTrue(vectorFieldSpec.isMultimodalField()); + + SrcFieldSpec srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(0); + Assertions.assertEquals("text_field_1", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.TEXT, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.TEXT, srcFieldSpec.getPayloadFormat()); + Assertions.assertFalse(srcFieldSpec.isBinary()); + + srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(1); + Assertions.assertEquals("text_field_2", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.TEXT, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.TEXT, srcFieldSpec.getPayloadFormat()); + Assertions.assertFalse(srcFieldSpec.isBinary()); + } + + @Test + void testMapEntryConstructorWithDifferentModalityListValue() { + Map imageFieldConfig = new HashMap<>(); + imageFieldConfig.put("field", "image_field"); + imageFieldConfig.put("modality", "jpeg"); + imageFieldConfig.put("format", "url"); + + Map videoFieldConfig = new HashMap<>(); + videoFieldConfig.put("field", "video_field"); + videoFieldConfig.put("modality", "mp4"); + videoFieldConfig.put("format", "url"); + + Map.Entry entry = + new AbstractMap.SimpleEntry<>( + "vector_field", + Arrays.asList("text_field", imageFieldConfig, videoFieldConfig)); + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + Assertions.assertEquals("vector_field", vectorFieldSpec.getFieldName()); + Assertions.assertTrue(vectorFieldSpec.isMultimodalField()); + + SrcFieldSpec srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(0); + Assertions.assertEquals("text_field", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.TEXT, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.TEXT, srcFieldSpec.getPayloadFormat()); + Assertions.assertFalse(srcFieldSpec.isBinary()); + + srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(1); + Assertions.assertEquals("image_field", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.JPEG, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.URL, srcFieldSpec.getPayloadFormat()); + Assertions.assertFalse(srcFieldSpec.isBinary()); + + srcFieldSpec = vectorFieldSpec.getSrcFieldSpecs().get(2); + Assertions.assertEquals("video_field", srcFieldSpec.getFieldName()); + Assertions.assertEquals(ModalityType.MP4, srcFieldSpec.getModalityType()); + Assertions.assertEquals(PayloadFormat.URL, srcFieldSpec.getPayloadFormat()); + Assertions.assertFalse(srcFieldSpec.isBinary()); + } +} From 52c3d53c855fb5a472b05a668da2045a3502da23 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 29 Jun 2026 22:27:11 +0800 Subject: [PATCH 079/375] [Test][E2E] Retry SQL Server XA setup until login is ready (#11213) --- .../jdbc/SqlServerSchemaChangeIT.java | 89 ++++++++++++------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/SqlServerSchemaChangeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/SqlServerSchemaChangeIT.java index 5fdc075f954a..7f52dfcdcde7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/SqlServerSchemaChangeIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/SqlServerSchemaChangeIT.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.shade.com.google.common.collect.Lists; +import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; @@ -44,6 +45,8 @@ public class SqlServerSchemaChangeIT extends AbstractSchemaChangeBaseIT { private static final String SQLSERVER_PASSWORD = "paanssy1234$"; private static final int SQLSERVER_PORT = 1433; private static final int SQLSERVER_XA_PORT = 5022; + private static final Duration SQLSERVER_SQLCMD_READY_TIMEOUT = Duration.ofMinutes(2); + private static final Duration SQLSERVER_SQLCMD_RETRY_INTERVAL = Duration.ofSeconds(2); private final String SQLSERVER_JDBC_URL = "jdbc:sqlserver://%s:%s;databaseName=%s;" + "useBulkCopyForBatchInsert=true;delayLoadingLobs=true;useFmtOnly=false;" @@ -107,45 +110,24 @@ protected GenericContainer initSinkContainer() { try { // This set of commands prepares for the subsequent enabling of the external user // enabled configuration (for XA transaction support) - container.execInContainer( - "/opt/mssql-tools18/bin/sqlcmd", - "-S", - "localhost", - "-U", - SQLSERVER_USER, - "-P", - SQLSERVER_PASSWORD, - "-Q", - "EXEC sp_configure 'show advanced options', 1; RECONFIGURE;", - "-C"); + execSqlcmdWithRetry( + container, + "configure advanced options", + "EXEC sp_configure 'show advanced options', 1; RECONFIGURE;"); // Enable external user access permissions, which is a requirement for SQL Server to // support XA distributed transactions. - container.execInContainer( - "/opt/mssql-tools18/bin/sqlcmd", - "-S", - "localhost", - "-U", - SQLSERVER_USER, - "-P", - SQLSERVER_PASSWORD, - "-Q", - "EXEC sp_configure 'external user enabled', 1; RECONFIGURE;", - "-C"); + execSqlcmdWithRetry( + container, + "enable external user access", + "EXEC sp_configure 'external user enabled', 1; RECONFIGURE;"); log.info("Installing stored procedures sp_sqljdbc_xa_install."); - container.execInContainer( - "/opt/mssql-tools18/bin/sqlcmd", - "-S", - "localhost", - "-U", - SQLSERVER_USER, - "-P", - SQLSERVER_PASSWORD, - "-Q", + execSqlcmdWithRetry( + container, + "install SQL Server XA stored procedures", "IF NOT EXISTS (SELECT * FROM sys.objects WHERE name = 'xp_sqljdbc_xa_init_ex') " - + "EXEC sp_sqljdbc_xa_install", - "-C"); + + "EXEC sp_sqljdbc_xa_install"); } catch (IOException | InterruptedException e) { log.error("XA procedure installation failed: ", e); throw new RuntimeException(e); @@ -157,4 +139,45 @@ protected GenericContainer initSinkContainer() { protected String sinkDatabaseType() { return DATABASE_TYPE; } + + /** + * SQL Server 2022 can emit the generic ready log before system database upgrades finish, so + * retry sqlcmd setup steps until login and execution both succeed. + */ + private void execSqlcmdWithRetry(GenericContainer container, String description, String sql) + throws IOException, InterruptedException { + long deadline = System.nanoTime() + SQLSERVER_SQLCMD_READY_TIMEOUT.toNanos(); + Container.ExecResult lastResult = null; + while (System.nanoTime() < deadline) { + lastResult = + container.execInContainer( + "/opt/mssql-tools18/bin/sqlcmd", + "-S", + "localhost", + "-U", + SQLSERVER_USER, + "-P", + SQLSERVER_PASSWORD, + "-Q", + sql, + "-C"); + if (lastResult.getExitCode() == 0) { + return; + } + log.info( + "sqlcmd step [{}] is not ready yet, exitCode={}, stdout={}, stderr={}", + description, + lastResult.getExitCode(), + lastResult.getStdout(), + lastResult.getStderr()); + Thread.sleep(SQLSERVER_SQLCMD_RETRY_INTERVAL.toMillis()); + } + throw new IllegalStateException( + String.format( + "Timed out waiting for sqlcmd step [%s] to succeed, last exitCode=%s, stdout=%s, stderr=%s", + description, + lastResult == null ? null : lastResult.getExitCode(), + lastResult == null ? null : lastResult.getStdout(), + lastResult == null ? null : lastResult.getStderr())); + } } From 24ad326a1f57b32516c6ae1a6f1047ee39dde38f Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 29 Jun 2026 22:29:18 +0800 Subject: [PATCH 080/375] [Test][E2E] Reduce Databend CDC parallelism for CI stability (#11210) --- .../src/test/resources/databend/fake_to_databend_cdc.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/resources/databend/fake_to_databend_cdc.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/resources/databend/fake_to_databend_cdc.conf index 9a9eed6f9f72..60345c256357 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/resources/databend/fake_to_databend_cdc.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/src/test/resources/databend/fake_to_databend_cdc.conf @@ -16,7 +16,9 @@ # env { - execution.parallelism = 2 + # Keep this CDC E2E single-parallel to avoid slot starvation on the + # tight Flink CI matrix while still covering CDC row-kind merge semantics. + execution.parallelism = 1 job.mode = "BATCH" checkpoint.interval = 1000 } @@ -89,4 +91,4 @@ sink { conflict_key = "id" enable_delete = true } -} \ No newline at end of file +} From 7be1b0e4643451c448ca0a909c1ada8a118824d7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 29 Jun 2026 22:35:37 +0800 Subject: [PATCH 081/375] [Test][E2E] Stabilize Kafka exactly-once record reads (#11205) --- .../e2e/connector/kafka/KafkaIT.java | 81 ++++++++++++++----- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kafka/KafkaIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kafka/KafkaIT.java index f91c1827a188..72ac59dc22a9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kafka/KafkaIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kafka/KafkaIT.java @@ -1689,6 +1689,7 @@ public void testKafkaToKafkaExactlyOnceOnStreaming(TestContainer container) { createKafkaTopic(producerTopic); createKafkaTopic(consumerTopic); String sourceData = "Seatunnel Exactly Once Example"; + String keepAliveData = sourceData + "-keepalive-" + resourceSuffix; long sinkStartOffset = endOffsetOnP0(consumerTopic); for (int i = 0; i < 10; i++) { ProducerRecord record = @@ -1716,9 +1717,24 @@ public void testKafkaToKafkaExactlyOnceOnStreaming(TestContainer container) { .await() .atMost(5, MINUTES) .untilAsserted( - () -> - Assertions.assertTrue( - checkData(consumerTopic, sinkStartOffset, 10, sourceData))); + () -> { + // Keep the streaming source active so the last exactly-once transaction + // is forced through a later checkpoint on slow Flink CI axes. + ProducerRecord keepAliveRecord = + new ProducerRecord<>( + producerTopic, + null, + keepAliveData.getBytes(StandardCharsets.UTF_8)); + producer.send(keepAliveRecord); + producer.flush(); + Assertions.assertTrue( + checkData( + consumerTopic, + sinkStartOffset, + 10, + sourceData, + Collections.singletonList(keepAliveData))); + }); } @TestTemplate @@ -1754,21 +1770,40 @@ public void testKafkaToKafkaExactlyOnceOnBatch(TestContainer container) // Compare the values of data fields obtained from consumers private boolean checkData(String topicName, long startOffset, long expectedCount, String data) { - List listData = getKafkaConsumerListData(topicName, startOffset, expectedCount); - if (listData.isEmpty() || listData.size() != expectedCount) { + return checkData(topicName, startOffset, expectedCount, data, Collections.emptyList()); + } + + private boolean checkData( + String topicName, + long startOffset, + long expectedCount, + String data, + List ignoredValues) { + List listData = getKafkaConsumerListData(topicName, startOffset); + List matchedData = new ArrayList<>(); + for (String value : listData) { + if (data.equals(value)) { + matchedData.add(value); + continue; + } + if (ignoredValues.contains(value)) { + continue; + } log.error( - "testKafkaToKafkaExactlyOnce get data size is not expect,get consumer data size {},start offset {},expected count {}", + "testKafkaToKafkaExactlyOnce get unexpected data value {}, start offset {}", + value, + startOffset); + return false; + } + if (matchedData.isEmpty() || matchedData.size() != expectedCount) { + log.error( + "testKafkaToKafkaExactlyOnce get data size is not expect,get matched data size {},visible data size {},start offset {},expected count {}", + matchedData.size(), listData.size(), startOffset, expectedCount); return false; } - for (String value : listData) { - if (!data.equals(value)) { - log.error("testKafkaToKafkaExactlyOnce get data value is not expect"); - return false; - } - } return true; } @@ -2305,8 +2340,7 @@ private List getKafkaConsumerListData(String topicName) { } } - private List getKafkaConsumerListData( - String topicName, long startOffset, long expectedCount) { + private List getKafkaConsumerListData(String topicName, long startOffset) { KafkaConsumer consumer = null; try { List data = new ArrayList<>(); @@ -2314,17 +2348,28 @@ private List getKafkaConsumerListData( TopicPartition topicPartition = new TopicPartition(topicName, 0); consumer.assign(Collections.singletonList(topicPartition)); consumer.seek(topicPartition, startOffset); - long targetOffsetExclusive = startOffset + expectedCount; + // READ_COMMITTED consumers may skip aborted transactional offsets, so N committed + // records are not guaranteed to occupy N contiguous offsets after startOffset. + long visibleEndOffsetExclusive = + consumer.endOffsets(Collections.singletonList(topicPartition)) + .get(topicPartition); Long lastProcessedOffset = startOffset - 1; + int consecutiveEmptyPolls = 0; do { ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord record : records) { - if (record.offset() >= startOffset && record.offset() < targetOffsetExclusive) { + if (records.isEmpty()) { + consecutiveEmptyPolls++; + continue; + } + consecutiveEmptyPolls = 0; + for (ConsumerRecord record : records.records(topicPartition)) { + if (record.offset() >= startOffset && record.offset() > lastProcessedOffset) { data.add(record.value()); } lastProcessedOffset = record.offset(); } - } while (lastProcessedOffset < targetOffsetExclusive - 1); + } while (lastProcessedOffset < visibleEndOffsetExclusive - 1 + && consecutiveEmptyPolls < 20); return data; } finally { closeKafkaConsumer(consumer); From a1324acdae34bdb6bf8e524afb40a5c19691001d Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 29 Jun 2026 22:45:04 +0800 Subject: [PATCH 082/375] [Fix][Connector-V2] Avoid Xugu pooled connection isValid checks (#11190) --- .../JdbcConnectionValidationUtils.java | 78 +++++++++++++++++++ ...SimpleJdbcConnectionPoolProviderProxy.java | 5 +- .../SimpleJdbcConnectionProvider.java | 3 +- .../seatunnel/jdbc/sink/JdbcSinkWriter.java | 13 ++++ .../JdbcConnectionValidationUtilsTest.java | 78 +++++++++++++++++++ .../jdbc/sink/JdbcSinkWriterTest.java | 64 +++++++++++++++ 6 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/JdbcConnectionValidationUtils.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/JdbcConnectionValidationUtilsTest.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriterTest.java diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/JdbcConnectionValidationUtils.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/JdbcConnectionValidationUtils.java new file mode 100644 index 000000000000..15736a8a25c7 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/JdbcConnectionValidationUtils.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.connection; + +import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcConnectionConfig; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Optional; + +/** Utility methods for JDBC driver-specific connection validation hooks. */ +public final class JdbcConnectionValidationUtils { + + /** Xugu JDBC driver class name used by connector-jdbc ITs. */ + public static final String XUGU_DRIVER = "com.xugu.cloudjdbc.Driver"; + + /** Validation query used when the Xugu driver cannot answer Connection.isValid(timeout). */ + public static final String XUGU_VALIDATION_QUERY = "SELECT 1 FROM DUAL"; + + private JdbcConnectionValidationUtils() {} + + /** + * Xugu's driver throws during Connection.isValid(timeout), so pooled connections need a SQL + * probe instead of the JDBC driver validation hook. + */ + public static boolean isConnectionValid(Connection connection, JdbcConnectionConfig jdbcConfig) + throws SQLException { + if (connection == null) { + return false; + } + + Optional validationQuery = getConnectionValidationQuery(jdbcConfig); + if (!validationQuery.isPresent()) { + return connection.isValid(jdbcConfig.getConnectionCheckTimeoutSeconds()); + } + + try (PreparedStatement preparedStatement = + connection.prepareStatement(validationQuery.get()); + ResultSet resultSet = preparedStatement.executeQuery()) { + return resultSet.next(); + } + } + + /** + * Returns an optional validation query for drivers that need SQL-based liveness checks instead + * of {@link Connection#isValid(int)}. + */ + public static Optional getConnectionValidationQuery(JdbcConnectionConfig jdbcConfig) { + if (jdbcConfig == null) { + return Optional.empty(); + } + + String driverName = jdbcConfig.getDriverName(); + String url = jdbcConfig.getUrl(); + if (XUGU_DRIVER.equals(driverName) || (url != null && url.startsWith("jdbc:xugu:"))) { + return Optional.of(XUGU_VALIDATION_QUERY); + } + + return Optional.empty(); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/SimpleJdbcConnectionPoolProviderProxy.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/SimpleJdbcConnectionPoolProviderProxy.java index 6bca2db726cf..6a3f024e5641 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/SimpleJdbcConnectionPoolProviderProxy.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/SimpleJdbcConnectionPoolProviderProxy.java @@ -47,9 +47,8 @@ public Connection getConnection() { @Override public boolean isConnectionValid() throws SQLException { return poolManager.containsConnection(queueIndex) - && poolManager - .getConnection(queueIndex) - .isValid(jdbcConfig.getConnectionCheckTimeoutSeconds()); + && JdbcConnectionValidationUtils.isConnectionValid( + poolManager.getConnection(queueIndex), jdbcConfig); } @Override diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/SimpleJdbcConnectionProvider.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/SimpleJdbcConnectionProvider.java index f9f36325af3b..267b26c7134d 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/SimpleJdbcConnectionProvider.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/SimpleJdbcConnectionProvider.java @@ -59,8 +59,7 @@ public Connection getConnection() { @Override public boolean isConnectionValid() throws SQLException { - return connection != null - && connection.isValid(jdbcConfig.getConnectionCheckTimeoutSeconds()); + return JdbcConnectionValidationUtils.isConnectionValid(connection, jdbcConfig); } private static Driver loadDriver(String driverName) throws ClassNotFoundException { diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriter.java index 3518015b1a65..083a2ee85af7 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriter.java @@ -24,10 +24,12 @@ import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; +import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcConnectionConfig; import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcSinkConfig; import org.apache.seatunnel.connectors.seatunnel.jdbc.exception.JdbcConnectorErrorCode; import org.apache.seatunnel.connectors.seatunnel.jdbc.exception.JdbcConnectorException; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.JdbcOutputFormatBuilder; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.connection.JdbcConnectionValidationUtils; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.connection.SimpleJdbcConnectionPoolProviderProxy; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; @@ -95,10 +97,21 @@ public MultiTableResourceManager initMultiTableResourceMa ds.setPassword(jdbcSinkConfig.getJdbcConnectionConfig().getPassword().get()); } ds.setAutoCommit(jdbcSinkConfig.getJdbcConnectionConfig().isAutoCommit()); + applyConnectionValidation(ds, jdbcSinkConfig.getJdbcConnectionConfig()); jdbcSinkConfig.getJdbcConnectionConfig().getProperties().forEach(ds::addDataSourceProperty); return new JdbcMultiTableResourceManager(new ConnectionPoolManager(ds)); } + /** + * Configures pool-level validation for JDBC drivers that cannot pass Hikari's default + * Connection.isValid(timeout) probe. + */ + static void applyConnectionValidation( + HikariDataSource dataSource, JdbcConnectionConfig jdbcConnectionConfig) { + JdbcConnectionValidationUtils.getConnectionValidationQuery(jdbcConnectionConfig) + .ifPresent(dataSource::setConnectionTestQuery); + } + @Override public void setMultiTableResourceManager( MultiTableResourceManager multiTableResourceManager, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/JdbcConnectionValidationUtilsTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/JdbcConnectionValidationUtilsTest.java new file mode 100644 index 000000000000..7946142b991a --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/connection/JdbcConnectionValidationUtilsTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.connection; + +import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcConnectionConfig; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests driver-specific JDBC connection validation fallbacks. */ +class JdbcConnectionValidationUtilsTest { + + /** Verifies that Xugu uses an explicit SQL probe instead of Connection.isValid(timeout). */ + @Test + void testXuguValidationUsesSqlProbe() throws SQLException { + JdbcConnectionConfig jdbcConnectionConfig = + JdbcConnectionConfig.builder() + .driverName(JdbcConnectionValidationUtils.XUGU_DRIVER) + .url("jdbc:xugu://localhost:5138/SYSTEM") + .build(); + Connection connection = mock(Connection.class); + PreparedStatement preparedStatement = mock(PreparedStatement.class); + ResultSet resultSet = mock(ResultSet.class); + when(connection.prepareStatement(JdbcConnectionValidationUtils.XUGU_VALIDATION_QUERY)) + .thenReturn(preparedStatement); + when(preparedStatement.executeQuery()).thenReturn(resultSet); + when(resultSet.next()).thenReturn(true); + + Assertions.assertTrue( + JdbcConnectionValidationUtils.isConnectionValid(connection, jdbcConnectionConfig)); + verify(connection, never()).isValid(anyInt()); + } + + /** Verifies that default drivers still use the standard JDBC validation hook. */ + @Test + void testDefaultValidationFallsBackToJdbcIsValid() throws SQLException { + JdbcConnectionConfig jdbcConnectionConfig = + JdbcConnectionConfig.builder() + .driverName("org.postgresql.Driver") + .url("jdbc:postgresql://localhost:5432/test") + .connectionCheckTimeoutSeconds(12) + .build(); + Connection connection = mock(Connection.class); + when(connection.isValid(12)).thenReturn(true); + + Assertions.assertTrue( + JdbcConnectionValidationUtils.isConnectionValid(connection, jdbcConnectionConfig)); + verify(connection).isValid(12); + verify(connection, never()) + .prepareStatement(JdbcConnectionValidationUtils.XUGU_VALIDATION_QUERY); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriterTest.java new file mode 100644 index 000000000000..442b9f949304 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriterTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.sink; + +import org.apache.seatunnel.shade.com.zaxxer.hikari.HikariDataSource; + +import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcConnectionConfig; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.connection.JdbcConnectionValidationUtils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Tests JDBC sink connection pool validation query customization. */ +class JdbcSinkWriterTest { + + /** Verifies that Xugu pools use a validation query compatible with the driver. */ + @Test + void testApplyConnectionValidationSetsXuguValidationQuery() { + HikariDataSource dataSource = new HikariDataSource(); + JdbcConnectionConfig jdbcConnectionConfig = + JdbcConnectionConfig.builder() + .driverName(JdbcConnectionValidationUtils.XUGU_DRIVER) + .url("jdbc:xugu://localhost:5138/SYSTEM") + .build(); + + JdbcSinkWriter.applyConnectionValidation(dataSource, jdbcConnectionConfig); + + Assertions.assertEquals( + JdbcConnectionValidationUtils.XUGU_VALIDATION_QUERY, + dataSource.getConnectionTestQuery()); + dataSource.close(); + } + + /** Verifies that other drivers keep Hikari's default validation behavior. */ + @Test + void testApplyConnectionValidationKeepsDefaultDriverValidation() { + HikariDataSource dataSource = new HikariDataSource(); + JdbcConnectionConfig jdbcConnectionConfig = + JdbcConnectionConfig.builder() + .driverName("org.postgresql.Driver") + .url("jdbc:postgresql://localhost:5432/test") + .build(); + + JdbcSinkWriter.applyConnectionValidation(dataSource, jdbcConnectionConfig); + + Assertions.assertNull(dataSource.getConnectionTestQuery()); + dataSource.close(); + } +} From e8846b10112849a5e350ddbcecf9654c4a3f7099 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Mon, 29 Jun 2026 22:49:33 +0800 Subject: [PATCH 083/375] [Fix][API] Fix validateSingleChoice and extension exceptions bypassing error aggregation (#11121) --- .../util/ConditionExtension.java | 13 +- .../configuration/util/ConfigValidator.java | 99 +++++++++----- .../api/configuration/util/OptionUtil.java | 13 +- .../util/ConfigValidatorTest.java | 127 +++++++++++++++++- 4 files changed, 207 insertions(+), 45 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionExtension.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionExtension.java index 204805967852..df27000637f0 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionExtension.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionExtension.java @@ -50,14 +50,19 @@ public interface ConditionExtension { * Evaluates whether {@code value} passes this validation rule. * *

Return {@code false} for simple failure — the framework composes the error from {@link - * #description()} automatically. Throw {@link OptionValidationException} when a richer, - * context-specific message is needed. Avoid other unchecked exceptions — they propagate - * unwrapped. + * #description()} automatically and continues collecting other validation errors. + * + *

Throw {@link OptionValidationException} when a richer, context-specific message is needed. + * The framework catches the exception, extracts its message, and adds it to the aggregated + * error list — subsequent validations still run. Use this when you need a more descriptive + * error message than {@link #description()} alone provides. + * + *

Avoid other unchecked exceptions — they propagate unwrapped. * * @param config full configuration context (read-only), available for cross-field checks * @param value the resolved option value; may be {@code null} * @return {@code true} if valid - * @throws OptionValidationException for detailed error reporting + * @throws OptionValidationException for detailed, context-specific error reporting */ boolean evaluate(ReadonlyConfig config, T value) throws OptionValidationException; } diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java index ae7e4450f90f..42d1b1c7b953 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java @@ -30,18 +30,28 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.apache.seatunnel.api.configuration.util.OptionUtil.formatError; +import static org.apache.seatunnel.api.configuration.util.OptionUtil.formatOptionsError; import static org.apache.seatunnel.api.configuration.util.OptionUtil.getOptionKeys; public class ConfigValidator { private final ReadonlyConfig config; + /** Closed set of validation error categories used in formatted error messages. */ + private static final String TYPE_REQUIRED = "required"; + + private static final String TYPE_VALUE = "value"; + private static final String TYPE_BUNDLED = "bundled"; + private static final String TYPE_EXCLUSIVE = "exclusive"; + private static final String TYPE_CONDITIONAL = "conditional"; + private static final String TYPE_SINGLE_CHOICE = "singleChoice"; + private static final Set COMMON_KEYS = new HashSet<>(); static { @@ -191,14 +201,14 @@ private void collectErrors(OptionRule rule, Expression expression, List (RequiredOption.ConditionalRequiredOptions) requiredOption)) { continue; } - validateSingleChoice(option); + validateSingleChoice(option, errors); } } } for (Option option : rule.getOptionalOptions()) { if (SingleChoiceOption.class.isAssignableFrom(option.getClass())) { - validateSingleChoice(option); + validateSingleChoice(option, errors); } } @@ -212,11 +222,20 @@ private void collectErrors(OptionRule rule, Expression expression, List if (structurallyAbsentKeys.contains(constraint.getOption().key())) { continue; } - if (isConstraintApplicable(constraint, rule) && !validate(constraint)) { + if (!isConstraintApplicable(constraint, rule)) { + continue; + } + try { + if (!validate(constraint)) { + errors.add( + formatError( + constraint.getOption().key(), + TYPE_VALUE, + constraint.toString())); + } + } catch (OptionValidationException e) { errors.add( - String.format( - "option: %s\n type: value\n constraint: %s", - constraint.getOption().key(), constraint.toString())); + formatError(constraint.getOption().key(), TYPE_VALUE, e.getRawMessage())); } } } @@ -308,29 +327,32 @@ private boolean anyOrSegmentFullyPresent(Condition condition) { return false; } - void validateSingleChoice(Option option) { + void validateSingleChoice(Option option, List errors) { SingleChoiceOption singleChoiceOption = (SingleChoiceOption) option; List optionValues = singleChoiceOption.getOptionValues(); if (CollectionUtils.isEmpty(optionValues)) { - throw new OptionValidationException( - "These options(%s) are SingleChoiceOption, the optionValues must not be null.", - getOptionKeys(Collections.singletonList(singleChoiceOption))); + errors.add( + formatError( + option.key(), TYPE_SINGLE_CHOICE, "optionValues must not be empty")); + return; } Object o = singleChoiceOption.defaultValue(); if (o != null && !optionValues.contains(o)) { - throw new OptionValidationException( - "These options(%s) are SingleChoiceOption, the defaultValue(%s) must be one of the optionValues(%s).", - getOptionKeys(Collections.singletonList(singleChoiceOption)), o, optionValues); + errors.add( + formatError( + option.key(), + TYPE_SINGLE_CHOICE, + String.format("defaultValue(%s) must be one of %s", o, optionValues))); } Object value = config.get(option); if (value != null && !optionValues.contains(value)) { - throw new OptionValidationException( - "These options(%s) are SingleChoiceOption, the value(%s) must be one of the optionValues(%s).", - getOptionKeys(Collections.singletonList(singleChoiceOption)), - value, - optionValues); + errors.add( + formatError( + option.key(), + TYPE_SINGLE_CHOICE, + String.format("value(%s) must be one of %s", value, optionValues))); } } @@ -371,9 +393,10 @@ String checkAbsolutelyRequired( return null; } String hint = expression == null ? "" : " when [" + expression + "]"; - return String.format( - "option: %s\n type: required\n constraint: required option is not configured%s", - getOptionKeys(absentOptions), hint); + return formatError( + getOptionKeys(absentOptions), + TYPE_REQUIRED, + "required option is not configured" + hint); } boolean hasOption(Option option) { @@ -394,9 +417,12 @@ String checkBundled(RequiredOption.BundledRequiredOptions bundledRequiredOptions if (present.size() == bundledOptions.size() || absent.size() == bundledOptions.size()) { return null; } - return String.format( - "options: %s\n type: bundled\n constraint: bundled options must be present or absent together (present: [%s], absent: [%s])", - getOptionKeys(bundledOptions), getOptionKeys(present), getOptionKeys(absent)); + return formatOptionsError( + getOptionKeys(bundledOptions), + TYPE_BUNDLED, + String.format( + "bundled options must be present or absent together (present: [%s], absent: [%s])", + getOptionKeys(present), getOptionKeys(absent))); } String checkExclusive(RequiredOption.ExclusiveRequiredOptions exclusiveRequiredOptions) { @@ -411,14 +437,17 @@ String checkExclusive(RequiredOption.ExclusiveRequiredOptions exclusiveRequiredO return null; } if (count == 0) { - return String.format( - "options: %s\n type: exclusive\n constraint: exactly one option must be set, but none are configured", - getOptionKeys(exclusiveRequiredOptions.getExclusiveOptions())); + return formatOptionsError( + getOptionKeys(exclusiveRequiredOptions.getExclusiveOptions()), + TYPE_EXCLUSIVE, + "exactly one option must be set, but none are configured"); } - return String.format( - "options: %s\n type: exclusive\n constraint: mutually exclusive, but multiple are set: [%s]", + return formatOptionsError( getOptionKeys(exclusiveRequiredOptions.getExclusiveOptions()), - getOptionKeys(presentOptions)); + TYPE_EXCLUSIVE, + String.format( + "mutually exclusive, but multiple are set: [%s]", + getOptionKeys(presentOptions))); } String checkConditional(RequiredOption.ConditionalRequiredOptions conditionalRequiredOptions) { @@ -431,10 +460,12 @@ String checkConditional(RequiredOption.ConditionalRequiredOptions conditionalReq if (absentOptions.isEmpty()) { return null; } - return String.format( - "option: %s\n type: conditional\n constraint: required because [%s] is true", + return formatError( getOptionKeys(absentOptions), - conditionalRequiredOptions.getExpression().toString()); + TYPE_CONDITIONAL, + String.format( + "required because [%s] is true", + conditionalRequiredOptions.getExpression().toString())); } private boolean validate(Expression expression) { diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/OptionUtil.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/OptionUtil.java index 7a4a838e00e5..128d59a091fd 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/OptionUtil.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/OptionUtil.java @@ -22,15 +22,18 @@ import org.apache.seatunnel.api.configuration.Option; +import lombok.experimental.UtilityClass; + import java.lang.reflect.Field; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Collections; import java.util.List; +@UtilityClass public class OptionUtil { - private OptionUtil() {} + private static final String ERROR_TEMPLATE = "%s: %s\n type: %s\n constraint: %s"; public static String getOptionKeys(List> options) { StringBuilder builder = new StringBuilder(); @@ -93,6 +96,14 @@ public Type getType() { return options; } + public static String formatError(String optionKey, String type, String constraint) { + return String.format(ERROR_TEMPLATE, "option", optionKey, type, constraint); + } + + public static String formatOptionsError(String optionKeys, String type, String constraint) { + return String.format(ERROR_TEMPLATE, "options", optionKeys, type, constraint); + } + private static String formatUnderScoreCase(String camel) { StringBuilder underScore = new StringBuilder(String.valueOf(Character.toLowerCase(camel.charAt(0)))); diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java index 2b859bb05981..a0bcd46ff1b9 100644 --- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java +++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java @@ -296,9 +296,13 @@ public void testSingleChoiceOptionDefaultValueValidator() { Map config = new HashMap<>(); config.put(SINGLE_CHOICE_TEST.key(), "A"); Executable executable = () -> validate(config, optionRule); - assertEquals( - "ErrorCode:[API-02], ErrorDescription:[Option item validate failed] - These options('single_choice_test') are SingleChoiceOption, the defaultValue(M) must be one of the optionValues([A, B, C]).", - assertThrows(OptionValidationException.class, executable).getMessage()); + OptionValidationException ex = assertThrows(OptionValidationException.class, executable); + String msg = ex.getMessage(); + Assertions.assertTrue( + msg.contains("single_choice_test"), "Should mention option key: " + msg); + Assertions.assertTrue( + msg.contains("defaultValue(M) must be one of"), + "Should mention invalid defaultValue: " + msg); } @Test @@ -311,9 +315,12 @@ public void testSingleChoiceOptionValueValidator() { config.put(SINGLE_CHOICE_VALUE_TEST.key(), "N"); executable = () -> validate(config, optionRule); - assertEquals( - "ErrorCode:[API-02], ErrorDescription:[Option item validate failed] - These options('single_choice_test') are SingleChoiceOption, the value(N) must be one of the optionValues([A, B, C]).", - assertThrows(OptionValidationException.class, executable).getMessage()); + OptionValidationException ex = assertThrows(OptionValidationException.class, executable); + String msg = ex.getMessage(); + Assertions.assertTrue( + msg.contains("single_choice_test"), "Should mention option key: " + msg); + Assertions.assertTrue( + msg.contains("value(N) must be one of"), "Should mention invalid value: " + msg); } @Test @@ -3483,4 +3490,112 @@ public boolean evaluate(ReadonlyConfig config, String value) { config5.put(TEST_TOPIC.key(), Collections.emptyList()); assertThrows(OptionValidationException.class, () -> validate(config5, rule)); } + + // ==================== collectErrors contract tests ==================== + + @Test + public void testMultipleSingleChoiceErrorsCollected() { + Option choice1 = + Options.key("mode1") + .singleChoice(String.class, Arrays.asList("A", "B", "C")) + .defaultValue("A") + .withDescription("mode1"); + Option choice2 = + Options.key("mode2") + .singleChoice(String.class, Arrays.asList("X", "Y", "Z")) + .defaultValue("X") + .withDescription("mode2"); + + OptionRule rule = OptionRule.builder().required(choice1, choice2).build(); + Map config = new HashMap<>(); + config.put("mode1", "INVALID1"); + config.put("mode2", "INVALID2"); + + OptionValidationException ex = + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + String msg = ex.getMessage(); + Assertions.assertTrue(msg.contains("mode1"), "Should report mode1 error: " + msg); + Assertions.assertTrue(msg.contains("mode2"), "Should report mode2 error: " + msg); + Assertions.assertTrue( + msg.contains("INVALID1") && msg.contains("INVALID2"), + "Should report both invalid values: " + msg); + } + + @Test + public void testMixedErrorTypesAllCollected() { + Option choice = + Options.key("format") + .singleChoice(String.class, Arrays.asList("json", "csv", "avro")) + .defaultValue("json") + .withDescription("format"); + Option host = + Options.key("host").stringType().noDefaultValue().withDescription("host"); + Option port = + Options.key("port").intType().noDefaultValue().withDescription("port"); + + OptionRule rule = + OptionRule.builder() + .required(choice, host) + .required(port, greaterOrEqual(port, 1)) + .build(); + + Map config = new HashMap<>(); + config.put("format", "xml"); + config.put("port", 0); + + OptionValidationException ex = + assertThrows(OptionValidationException.class, () -> validate(config, rule)); + String msg = ex.getMessage(); + Assertions.assertTrue(msg.contains("host"), "Should report missing host: " + msg); + Assertions.assertTrue( + msg.contains("format") && msg.contains("singleChoice"), + "Should report single_choice error: " + msg); + Assertions.assertTrue( + msg.contains("port") && msg.contains("value"), + "Should report value constraint error: " + msg); + } + + @Test + public void testExtensionExceptionIsAggregatedNotFailFast() { + ConditionExtension throwingExtension = + new ConditionExtension() { + @Override + public String description() { + return "must be positive"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) + throws OptionValidationException { + if (value != null && value <= 0) { + throw new OptionValidationException( + "port value %d is not positive", value); + } + return true; + } + }; + + OptionRule rule = + OptionRule.builder() + .required(HOST, notBlank(HOST)) + .required(PORT, Conditions.extension(PORT, throwingExtension)) + .build(); + + Map config = new HashMap<>(); + config.put(HOST.key(), ""); + config.put(PORT.key(), -1); + + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + String msg = ex.getMessage(); + Assertions.assertTrue(msg.contains("2 errors"), "both errors should be aggregated: " + msg); + Assertions.assertTrue( + msg.contains("host") && msg.contains("is not blank"), + "host notBlank error should appear: " + msg); + Assertions.assertTrue( + msg.contains("port value -1 is not positive"), + "extension exception message should be preserved: " + msg); + } } From b253248ab7ba8985ccb9b7f750c01888fb614bfb Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 30 Jun 2026 20:06:52 +0800 Subject: [PATCH 084/375] [Fix][Connector-V2] Clean up stale SFTP pooled sessions (#11180) --- .../file/sftp/system/SFTPConnectionPool.java | 127 ++++++++---- .../file/sftp/system/SFTPFileSystem.java | 18 +- .../sftp/system/SFTPConnectionPoolTest.java | 183 ++++++++++++++++++ 3 files changed, 283 insertions(+), 45 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-file/connector-file-sftp/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPConnectionPoolTest.java diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPConnectionPool.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPConnectionPool.java index 4906c8061ab4..677d56c9e924 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPConnectionPool.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPConnectionPool.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.Map; import java.util.Set; public class SFTPConnectionPool { @@ -53,6 +54,9 @@ public class SFTPConnectionPool { } synchronized ChannelSftp getFromPool(ConnectionInfo info) throws IOException { + if (con2infoMap == null) { + throw new IOException("SFTP connection pool has been closed."); + } Set cons = idleConnections.get(info); ChannelSftp channel; @@ -60,7 +64,10 @@ synchronized ChannelSftp getFromPool(ConnectionInfo info) throws IOException { Iterator it = cons.iterator(); if (it.hasNext()) { channel = it.next(); - idleConnections.remove(info); + it.remove(); + if (cons.isEmpty()) { + idleConnections.remove(info); + } return channel; } else { throw new IOException("Connection pool error."); @@ -69,42 +76,50 @@ synchronized ChannelSftp getFromPool(ConnectionInfo info) throws IOException { return null; } - synchronized void returnToPool(ChannelSftp channel) { + synchronized boolean returnToPool(ChannelSftp channel) { + if (con2infoMap == null) { + return false; + } ConnectionInfo info = con2infoMap.get(channel); + if (info == null) { + return false; + } HashSet cons = idleConnections.get(info); if (cons == null) { cons = new HashSet(); idleConnections.put(info, cons); } cons.add(channel); + return true; } /** Shutdown the connection pool and close all open connections. */ - synchronized void shutdown() { - if (this.con2infoMap == null) { - return; // already shutdown in case it is called + void shutdown() { + Map connectionsToClose; + synchronized (this) { + if (this.con2infoMap == null) { + return; // already shutdown in case it is called + } + LOG.info("Inside shutdown, con2infoMap size=" + con2infoMap.size()); + + // Shutdown must close every tracked connection regardless of live-count drift. + connectionsToClose = new HashMap(con2infoMap); + this.maxConnection = 0; + this.liveConnectionCount = 0; + this.idleConnections = null; + this.con2infoMap = null; } - LOG.info("Inside shutdown, con2infoMap size=" + con2infoMap.size()); - this.maxConnection = 0; - Set cons = con2infoMap.keySet(); - if (cons != null && cons.size() > 0) { - // make a copy since we need to modify the underlying Map - Set copy = new HashSet(cons); - // Initiate disconnect from all outstanding connections - for (ChannelSftp con : copy) { - try { - disconnect(con); - } catch (IOException ioe) { - ConnectionInfo info = con2infoMap.get(con); - LOG.error( - "Error encountered while closing connection to " + info.getHost(), ioe); - } + for (Map.Entry entry : connectionsToClose.entrySet()) { + try { + closeChannel(entry.getKey()); + } catch (IOException ioe) { + LOG.error( + "Error encountered while closing connection to " + + entry.getValue().getHost(), + ioe); } } - // make sure no further connections can be returned. - this.idleConnections = null; - this.con2infoMap = null; } public synchronized int getMaxConnection() { @@ -125,11 +140,9 @@ public ChannelSftp connect(String host, int port, String user, String password, if (channel.isConnected()) { return channel; } else { + removeTrackedChannel(channel); + closeChannel(channel); channel = null; - synchronized (this) { - --liveConnectionCount; - con2infoMap.remove(channel); - } } } @@ -155,6 +168,9 @@ public ChannelSftp connect(String host, int port, String user, String password, session = jsch.getSession(user, host, port); } + // JSch creates a session reader thread; make it daemon so leaked sessions cannot keep + // Spark local-mode JVMs alive after a batch job has already finished. + session.setDaemonThread(true); session.setPassword(password); java.util.Properties config = new java.util.Properties(); @@ -182,31 +198,62 @@ void disconnect(ChannelSftp channel) throws IOException { // close connection if too many active connections boolean closeConnection = false; synchronized (this) { - if (liveConnectionCount > maxConnection) { + if (con2infoMap == null || !con2infoMap.containsKey(channel)) { + closeConnection = true; + } else if (liveConnectionCount > maxConnection) { --liveConnectionCount; con2infoMap.remove(channel); closeConnection = true; } } if (closeConnection) { - if (channel.isConnected()) { - try { - Session session = channel.getSession(); - channel.disconnect(); - session.disconnect(); - } catch (JSchException e) { - throw new IOException(StringUtils.stringifyException(e)); - } + closeChannel(channel); + } else if (!returnToPool(channel)) { + closeChannel(channel); + } + } + } + + /** + * Remove bookkeeping for a channel that can no longer be reused before we create a replacement + * connection. + */ + synchronized void removeTrackedChannel(ChannelSftp channel) { + if (con2infoMap == null) { + return; + } + ConnectionInfo info = con2infoMap.remove(channel); + if (info != null) { + Set cons = idleConnections.get(info); + if (cons != null) { + cons.remove(channel); + if (cons.isEmpty()) { + idleConnections.remove(info); } + } + } + if (liveConnectionCount > 0) { + liveConnectionCount--; + } + } - } else { - returnToPool(channel); + /** Close both the SFTP channel and its backing SSH session when they are still open. */ + private void closeChannel(ChannelSftp channel) throws IOException { + try { + Session session = channel.getSession(); + if (channel.isConnected()) { + channel.disconnect(); } + if (session != null && session.isConnected()) { + session.disconnect(); + } + } catch (JSchException e) { + throw new IOException(StringUtils.stringifyException(e)); } } public int getIdleCount() { - return this.idleConnections.size(); + return this.idleConnections == null ? 0 : this.idleConnections.size(); } public int getLiveConnCount() { @@ -214,7 +261,7 @@ public int getLiveConnCount() { } public int getConnPoolSize() { - return this.con2infoMap.size(); + return this.con2infoMap == null ? 0 : this.con2infoMap.size(); } /** diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java index 99bf41776390..0990726aa065 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java @@ -115,7 +115,7 @@ private void setConfigurationFromURI(URI uriInfo, Configuration conf) throws IOE } int connectionMax = conf.getInt(FS_SFTP_CONNECTION_MAX, DEFAULT_MAX_CONNECTION); - connectionPool = new SFTPConnectionPool(connectionMax, connectionMax); + connectionPool = new SFTPConnectionPool(connectionMax, 0); } private ChannelSftp connect() throws IOException { @@ -548,8 +548,11 @@ public FSDataOutputStream create( new FSDataOutputStream(os, statistics) { @Override public void close() throws IOException { - super.close(); - disconnect(client); + try { + super.close(); + } finally { + disconnect(client); + } } }; @@ -651,7 +654,12 @@ public FileStatus getFileStatus(Path f) throws IOException { @Override public void close() throws IOException { - super.close(); - connectionPool.shutdown(); + try { + super.close(); + } finally { + if (connectionPool != null) { + connectionPool.shutdown(); + } + } } } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPConnectionPoolTest.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPConnectionPoolTest.java new file mode 100644 index 000000000000..cf26cbfbe579 --- /dev/null +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPConnectionPoolTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.sftp.system; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; + +import java.lang.reflect.Field; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Tests the SFTP connection pool bookkeeping that continuous discovery relies on during cleanup. + */ +class SFTPConnectionPoolTest { + + /** + * Keep sibling idle channels tracked after one channel is borrowed from the shared pool key. + */ + @Test + void getFromPoolShouldKeepOtherIdleChannelsTracked() throws Exception { + SFTPConnectionPool connectionPool = new SFTPConnectionPool(2, 0); + SFTPConnectionPool.ConnectionInfo connectionInfo = + new SFTPConnectionPool.ConnectionInfo("host", 22, "user"); + ChannelSftp firstChannel = new TestChannelSftp(true, Mockito.mock(Session.class)); + ChannelSftp secondChannel = new TestChannelSftp(true, Mockito.mock(Session.class)); + + HashSet idleChannels = new HashSet<>(); + idleChannels.add(firstChannel); + idleChannels.add(secondChannel); + idleConnections(connectionPool).put(connectionInfo, idleChannels); + trackedConnections(connectionPool).put(firstChannel, connectionInfo); + trackedConnections(connectionPool).put(secondChannel, connectionInfo); + + ChannelSftp borrowedChannel = connectionPool.getFromPool(connectionInfo); + + Assertions.assertTrue(borrowedChannel == firstChannel || borrowedChannel == secondChannel); + Set remainingIdleChannels = + idleConnections(connectionPool).get(connectionInfo); + Assertions.assertNotNull(remainingIdleChannels); + Assertions.assertEquals(1, remainingIdleChannels.size()); + Assertions.assertFalse(remainingIdleChannels.contains(borrowedChannel)); + } + + /** Always disconnect the SSH session when a stale channel is being permanently closed. */ + @Test + void disconnectShouldCloseSessionForDisconnectedChannel() throws Exception { + SFTPConnectionPool connectionPool = new SFTPConnectionPool(0, 1); + SFTPConnectionPool.ConnectionInfo connectionInfo = + new SFTPConnectionPool.ConnectionInfo("host", 22, "user"); + Session session = Mockito.mock(Session.class); + Mockito.when(session.isConnected()).thenReturn(true); + TestChannelSftp channel = new TestChannelSftp(false, session); + trackedConnections(connectionPool).put(channel, connectionInfo); + + connectionPool.disconnect(channel); + + Mockito.verify(session).disconnect(); + Assertions.assertFalse(channel.wasDisconnected()); + Assertions.assertEquals(0, connectionPool.getLiveConnCount()); + Assertions.assertEquals(0, connectionPool.getConnPoolSize()); + } + + /** Shutdown must close tracked sessions even when live-count bookkeeping has drifted. */ + @Test + void shutdownShouldCloseTrackedChannelsRegardlessOfLiveCount() throws Exception { + SFTPConnectionPool connectionPool = new SFTPConnectionPool(2, 0); + SFTPConnectionPool.ConnectionInfo connectionInfo = + new SFTPConnectionPool.ConnectionInfo("host", 22, "user"); + Session session = Mockito.mock(Session.class); + Mockito.when(session.isConnected()).thenReturn(true); + TestChannelSftp channel = new TestChannelSftp(true, session); + trackedConnections(connectionPool).put(channel, connectionInfo); + + connectionPool.shutdown(); + + Assertions.assertTrue(channel.wasDisconnected()); + Mockito.verify(session).disconnect(); + } + + /** Late disconnects after shutdown must close the channel instead of touching closed maps. */ + @Test + void disconnectAfterShutdownShouldCloseChannel() throws Exception { + SFTPConnectionPool connectionPool = new SFTPConnectionPool(1, 0); + Session session = Mockito.mock(Session.class); + Mockito.when(session.isConnected()).thenReturn(true); + TestChannelSftp channel = new TestChannelSftp(true, session); + + connectionPool.shutdown(); + connectionPool.disconnect(channel); + + Assertions.assertTrue(channel.wasDisconnected()); + Mockito.verify(session).disconnect(); + Assertions.assertEquals(0, connectionPool.getIdleCount()); + Assertions.assertEquals(0, connectionPool.getConnPoolSize()); + } + + /** Unknown channels are not reusable pool entries, so they must be closed immediately. */ + @Test + void disconnectShouldCloseUntrackedChannel() throws Exception { + SFTPConnectionPool connectionPool = new SFTPConnectionPool(1, 1); + Session session = Mockito.mock(Session.class); + Mockito.when(session.isConnected()).thenReturn(true); + TestChannelSftp channel = new TestChannelSftp(true, session); + + connectionPool.disconnect(channel); + + Assertions.assertTrue(channel.wasDisconnected()); + Mockito.verify(session).disconnect(); + Assertions.assertEquals(0, connectionPool.getIdleCount()); + } + + /** Small concrete ChannelSftp stub that keeps Mockito away from final JSch internals. */ + private static final class TestChannelSftp extends ChannelSftp { + private final boolean connected; + private final Session session; + private boolean disconnected; + + private TestChannelSftp(boolean connected, Session session) { + this.connected = connected; + this.session = session; + } + + @Override + public boolean isConnected() { + return connected; + } + + @Override + public void disconnect() { + disconnected = true; + } + + @Override + public Session getSession() throws JSchException { + return session; + } + + private boolean wasDisconnected() { + return disconnected; + } + } + + /** Read the private idle map to assert that the pool does not lose sibling channels. */ + @SuppressWarnings("unchecked") + private static Map> idleConnections( + SFTPConnectionPool connectionPool) throws Exception { + Field field = SFTPConnectionPool.class.getDeclaredField("idleConnections"); + field.setAccessible(true); + return (Map>) + field.get(connectionPool); + } + + /** Read the private tracked-channel map to seed deterministic pool state for unit tests. */ + @SuppressWarnings("unchecked") + private static Map trackedConnections( + SFTPConnectionPool connectionPool) throws Exception { + Field field = SFTPConnectionPool.class.getDeclaredField("con2infoMap"); + field.setAccessible(true); + return (Map) field.get(connectionPool); + } +} From c9716cc60b9540d5844d48cfe93e63919dc6c18a Mon Sep 17 00:00:00 2001 From: Jast Date: Tue, 30 Jun 2026 20:08:29 +0800 Subject: [PATCH 085/375] [Docs] Fix connector documentation links (#11217) --- docs/en/connectors/sink/Hudi.md | 3 +-- docs/en/connectors/sink/Kudu.md | 6 +++--- docs/en/connectors/sink/MongoDB.md | 3 +-- docs/en/connectors/sink/Pulsar.md | 4 ++-- docs/en/connectors/sink/Qdrant.md | 4 ++-- docs/en/connectors/sink/Socket.md | 3 +-- docs/zh/connectors/sink/AmazonSqs.md | 4 ++-- docs/zh/connectors/sink/Hudi.md | 4 ++-- docs/zh/connectors/sink/Kudu.md | 6 +++--- docs/zh/connectors/sink/ObsFile.md | 16 ++++++++-------- docs/zh/connectors/sink/OceanBase.md | 4 ++-- docs/zh/connectors/sink/OssFile.md | 2 +- docs/zh/connectors/sink/OssJindoFile.md | 4 ++-- docs/zh/connectors/sink/Pulsar.md | 4 ++-- docs/zh/connectors/sink/Qdrant.md | 4 ++-- docs/zh/connectors/sink/RocketMQ.md | 2 +- docs/zh/connectors/sink/S3-Redshift.md | 4 ++-- docs/zh/connectors/sink/SftpFile.md | 2 +- docs/zh/connectors/sink/Socket.md | 2 +- docs/zh/connectors/sink/Tablestore.md | 2 +- docs/zh/connectors/source/CosFile.md | 2 +- docs/zh/connectors/source/DB2.md | 4 ++-- docs/zh/connectors/source/Easysearch.md | 4 ++-- 23 files changed, 45 insertions(+), 48 deletions(-) diff --git a/docs/en/connectors/sink/Hudi.md b/docs/en/connectors/sink/Hudi.md index 44110a2cc6c0..8f604fb07c28 100644 --- a/docs/en/connectors/sink/Hudi.md +++ b/docs/en/connectors/sink/Hudi.md @@ -137,7 +137,7 @@ Option introduction： ### common options -Source plugin common parameters, please refer to [Source Common Options](../common-options/sink-common-options.md) for details. +Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. ## Examples @@ -214,4 +214,3 @@ sink { ## Changelog - diff --git a/docs/en/connectors/sink/Kudu.md b/docs/en/connectors/sink/Kudu.md index 1946a8388328..8879dbc326bd 100644 --- a/docs/en/connectors/sink/Kudu.md +++ b/docs/en/connectors/sink/Kudu.md @@ -48,13 +48,13 @@ import ChangeLog from '../changelog/connector-kudu.md'; | kerberos_principal | String | No | - | Kerberos principal. Note that all zeta nodes require have this file. | | kerberos_keytab | String | No | - | Kerberos keytab. Note that all zeta nodes require have this file. | | kerberos_krb5conf | String | No | - | Kerberos krb5 conf. Note that all zeta nodes require have this file. | -| save_mode | String | No | - | Storage mode, support `overwrite` and `append`. | +| save_mode | String | No | APPEND | Storage mode, support `overwrite` and `append`. | | session_flush_mode | String | No | AUTO_FLUSH_SYNC | Kudu flush mode. Default AUTO_FLUSH_SYNC. | -| batch_size | Int | No | 1024 | The flush max size (includes all append, upsert and delete records), over this number of records, will flush data. The default value is 100 | +| batch_size | Int | No | 1024 | The flush max size (includes all append, upsert and delete records), over this number of records, will flush data. The default value is 1024 | | buffer_flush_interval | Int | No | 10000 | The flush interval mills, over this time, asynchronous threads will flush data. | | ignore_not_found | Bool | No | false | If true, ignore all not found rows. | | ignore_not_duplicate | Bool | No | false | If true, ignore all dulicate rows. | -| common-options | | No | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/sink-common-options.md) for details. | +| common-options | | No | - | Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. | ## Task Example diff --git a/docs/en/connectors/sink/MongoDB.md b/docs/en/connectors/sink/MongoDB.md index 73fb23942965..c03d59a8c97a 100644 --- a/docs/en/connectors/sink/MongoDB.md +++ b/docs/en/connectors/sink/MongoDB.md @@ -75,7 +75,7 @@ The following table lists the field data type mapping from MongoDB BSON type to | upsert-enable | Boolean | No | false | Whether to write documents via upsert mode. | | primary-key | List | No | - | The primary keys for upsert/update. Keys are in `["id","name",...]` format for properties. | | transaction | Boolean | No | false | Whether to use transactions in MongoSink (requires MongoDB 4.2+). | -| common-options | | No | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/sink-common-options.md) for details | +| common-options | | No | - | Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details | | data_save_mode | String | No | APPEND_DATA | The data saving mode of mongodb，Option introduction,`DROP_DATA`:The collection will be cleared before inserting data;`APPEND_DATA`:Append data ;`ERROR_WHEN_DATA_EXISTS`:An error will be reported if there is data in the collection. | @@ -206,4 +206,3 @@ sink { ## Changelog - diff --git a/docs/en/connectors/sink/Pulsar.md b/docs/en/connectors/sink/Pulsar.md index 636bffcc844a..68d30a1670dc 100644 --- a/docs/en/connectors/sink/Pulsar.md +++ b/docs/en/connectors/sink/Pulsar.md @@ -40,7 +40,7 @@ Sink connector for Apache Pulsar. | pulsar.config | Map | No | - | In addition to the above parameters that must be specified by the Pulsar producer client. | | message.routing.mode | Enum | No | RoundRobinPartition | Default routing mode for messages to partition. | | partition_key_fields | array | No | - | Configure which fields are used as the key of the pulsar message. | -| common-options | config | no | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/sink-common-options.md) for details. | +| common-options | config | no | - | Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. | ## Parameter Interpretation @@ -137,7 +137,7 @@ The selected field must be an existing field in the upstream. ### common options -Source plugin common parameters, please refer to [Source Common Options](../common-options/sink-common-options.md) for details. +Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. ## Task Example diff --git a/docs/en/connectors/sink/Qdrant.md b/docs/en/connectors/sink/Qdrant.md index 3301d8670107..824a4165ac80 100644 --- a/docs/en/connectors/sink/Qdrant.md +++ b/docs/en/connectors/sink/Qdrant.md @@ -69,8 +69,8 @@ Whether to use TLS(SSL) connection. Required if using Qdrant cloud(https). ### common options -Sink plugin common parameters, please refer to [Source Common Options](../common-options/sink-common-options.md) for details. +Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. ## Changelog - \ No newline at end of file + diff --git a/docs/en/connectors/sink/Socket.md b/docs/en/connectors/sink/Socket.md index 019806a13e4b..53ac809291cc 100644 --- a/docs/en/connectors/sink/Socket.md +++ b/docs/en/connectors/sink/Socket.md @@ -27,7 +27,7 @@ Used to send data to Socket Server. Both support streaming and batch mode. | host | String | Yes | | socket server host | | port | Integer | Yes | | socket server port | | max_retries | Integer | No | 3 | The number of retries to send record failed | -| common-options | | No | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/sink-common-options.md) for details | +| common-options | | No | - | Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details | ## Task Example @@ -76,4 +76,3 @@ nc -l -v 9999 ## Changelog - diff --git a/docs/zh/connectors/sink/AmazonSqs.md b/docs/zh/connectors/sink/AmazonSqs.md index abdcb10354f5..76aff4863a8f 100644 --- a/docs/zh/connectors/sink/AmazonSqs.md +++ b/docs/zh/connectors/sink/AmazonSqs.md @@ -29,7 +29,7 @@ import ChangeLog from '../changelog/connector-amazonsqs.md'; |-------------------------|--------|--|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | url | String | 是 | - | 从Amazon SQS读取的队列URL. | | region | String | 否 | - | SQS服务的AWS区域 | -| format | String | 否 | json | 数据格式。默认格式为json。可选文本格式，canal json和debezium json。如果你使用json或文本格式。默认字段分隔符为“，”。如果自定义分隔符，请添加“field_delimiter”选项。如果您使用canal格式，请参阅[canal-json]（../formats/canal-json.md）了解详细信息。如果您使用debezium格式，请参阅[debezium json]（../formats/debezium json.md）了解详细信息. | +| format | String | 否 | json | 数据格式。默认格式为json。可选文本格式，canal json和debezium json。如果你使用json或文本格式。默认字段分隔符为“，”。如果自定义分隔符，请添加“field_delimiter”选项。如果您使用canal格式，请参阅[canal-json](../formats/canal-json.md)了解详细信息。如果您使用debezium格式，请参阅[debezium-json](../formats/debezium-json.md)了解详细信息. | | format_error_handle_way | String | 否 | fail | 数据格式错误的处理方法。默认值为fail，可选值为（fail，skip）。当选择失败时，数据格式错误将被阻止，并引发异常。当选择跳过时，数据格式错误将跳过此行数据. | | field_delimiter | String | 否 | , | 自定义数据格式的字段分隔符. | @@ -89,4 +89,4 @@ sink { ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/Hudi.md b/docs/zh/connectors/sink/Hudi.md index b8684ff37322..75024388f440 100644 --- a/docs/zh/connectors/sink/Hudi.md +++ b/docs/zh/connectors/sink/Hudi.md @@ -137,7 +137,7 @@ import ChangeLog from '../changelog/connector-hudi.md'; ### 通用选项 -数据源插件的通用参数，请参考 [Source Common Options](../common-options/sink-common-options.md) 了解详细信息。 +Sink插件通用参数，请参考 [Sink Common Options](../common-options/sink-common-options.md) 了解详细信息。 ## 示例 @@ -213,4 +213,4 @@ sink { ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/Kudu.md b/docs/zh/connectors/sink/Kudu.md index 64e51f838abc..5db7ffb262a8 100644 --- a/docs/zh/connectors/sink/Kudu.md +++ b/docs/zh/connectors/sink/Kudu.md @@ -48,13 +48,13 @@ import ChangeLog from '../changelog/connector-kudu.md'; | kerberos_principal | String | 否 | - | Kerberos主体。请注意，所有zeta节点都需要此文件。 | | kerberos_keytab | String | 否 | - | Kerberos密钥表。请注意，所有zeta节点都需要此文件。 | | kerberos_krb5conf | String | 否 | - | Kerberos krb5 conf.请注意，所有zeta节点都需要此文件。 | -| save_mode | String | 否 | - | 存储模式，支持 `overwrite` 和 `append`. | +| save_mode | String | 否 | APPEND | 存储模式，支持 `overwrite` 和 `append`. | | session_flush_mode | String | 否 | AUTO_FLUSH_SYNC | Kudu刷新模式。默认AUTO_FLUSH_SYNC。 | -| batch_size | Int | 否 | 1024 | 超过此记录数的刷新最大大小（包括所有追加、追加和删除记录）将刷新数据。默认值为100 | +| batch_size | Int | 否 | 1024 | 超过此记录数的刷新最大大小（包括所有追加、追加和删除记录）将刷新数据。默认值为1024 | | buffer_flush_interval | Int | 否 | 10000 | 刷新间隔期间，异步线程将刷新数据。 | | ignore_not_found | Bool | 否 | false | 如果为true，则忽略所有未找到的行。 | | ignore_not_duplicate | Bool | 否 | false | 如果为true，则忽略所有dulicate行。 | -| common-options | | 否 | - |源插件常用参数，详见[Source common Options]（../sink common-Options.md）。 | +| common-options | | 否 | - | Sink插件常用参数，详见[Sink common Options](../common-options/sink-common-options.md)。 | ## 任务示例 diff --git a/docs/zh/connectors/sink/ObsFile.md b/docs/zh/connectors/sink/ObsFile.md index 35ce836c4f60..70d604d73cbd 100644 --- a/docs/zh/connectors/sink/ObsFile.md +++ b/docs/zh/connectors/sink/ObsFile.md @@ -68,16 +68,16 @@ import ChangeLog from '../changelog/connector-file-obs.md'; | access_secret | string | 是 | - | obs文件系统的访问私钥。 | | endpoint | string | 是 | - | obs文件系统的终端。 | | custom_filename | boolean | 否 | false | 是否需要自定义文件名。 | -| file_name_expression | string | 否 | "${transactionId}" | 描述将在“路径”中创建的文件表达式。仅在custom_filename为true时使用。[提示]（#file_name_expression） | -| filename_time_format | string | 否 | "yyyy.MM.dd" | 指定“path”的时间格式。仅在custom_filename为true时使用。[提示]（#filename_time_format） | -| file_format_type | string | 否 | "csv" | 支持的文件类型。[提示]（#file_format_type） | +| file_name_expression | string | 否 | "${transactionId}" | 描述将在“路径”中创建的文件表达式。仅在custom_filename为true时使用。[提示](#file_name_expression) | +| filename_time_format | string | 否 | "yyyy.MM.dd" | 指定“path”的时间格式。仅在custom_filename为true时使用。[提示](#filename_time_format) | +| file_format_type | string | 否 | "csv" | 支持的文件类型。[提示](#file_format_type) | | field_delimiter | string | 否 | '\001' | 数据行中列之间的分隔符。仅在file_format为文本时使用。 | | row_delimiter | string | 否 | "\n" | 文件中行之间的分隔符。仅被 `text`、`csv`、`json` 文件格式需要。 | | have_partition | boolean | 否 | false | 是否需要处理分区。 | | partition_by | array | 否 | - | 根据所选字段对数据进行分区。只有在have_partition为true时才使用。 | -| partition_dir_expression | string | 否 | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | 只有在have_partition为真true时才使用。[提示]（#partition_dir_expression） | -| is_partition_field_write_in_file | boolean | 否 | false | 只有在have_partition为true时才使用。[提示]（#is_partition_field_write_in_file） | -| sink_columns | array | 否 | | 当此参数为空时，所有字段都是接收列。[提示]（#sink_columns） | +| partition_dir_expression | string | 否 | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | 只有在have_partition为真true时才使用。[提示](#partition_dir_expression) | +| is_partition_field_write_in_file | boolean | 否 | false | 只有在have_partition为true时才使用。[提示](#is_partition_field_write_in_file) | +| sink_columns | array | 否 | | 当此参数为空时，所有字段都是接收列。[提示](#sink_columns) | | is_enable_transaction | boolean | 否 | true | [提示](#is_enable_transaction) | | batch_size | int | 否 | 1000000 | [提示](#batch_size) | | single_file_mode | boolean | 否 | false | 每个并行处理只会输出一个文件。启用此参数后，batch_size将不会生效。输出文件名没有文件块后缀。 | @@ -176,7 +176,7 @@ import ChangeLog from '../changelog/connector-file-obs.md'; #### common options ->Sink插件常用参数，请参考[Sink common Options]（../common-options/sink-common-options.md）了解详细信息。 +>Sink插件常用参数，请参考[Sink common Options](../common-options/sink-common-options.md)了解详细信息。 ## 任务示例 @@ -329,4 +329,4 @@ LocalFile { ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/OceanBase.md b/docs/zh/connectors/sink/OceanBase.md index 392ff43b1a03..bafa55d0feb0 100644 --- a/docs/zh/connectors/sink/OceanBase.md +++ b/docs/zh/connectors/sink/OceanBase.md @@ -88,7 +88,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; | transaction_timeout_sec | Int | 否 | -1 | 事务打开后的超时，默认值为-1（永不超时）。请注意，设置超时可能会影响＜br/＞精确一次语义 | | auto_commit | Boolean | 否 | true | 默认情况下启用自动事务提交 | | properties | Map | 否 | - | 其他连接配置参数，当属性和URL具有相同的参数时，优先级由驱动程序的特定实现决定。例如，在MySQL中，属性优先于URL。 | -| common-options | | 否 | - | Sink插件常用参数，详见[Sink common Options]（../common-options/sink-common-options.md） | +| common-options | | 否 | - | Sink插件常用参数，详见[Sink common Options](../common-options/sink-common-options.md) | | enable_upsert | Boolean | 否 | true | 通过primary_keys存在启用upsert，如果任务没有键重复数据，将此参数设置为“false”可以加快数据导入 | ### 提示 @@ -187,4 +187,4 @@ sink { ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/OssFile.md b/docs/zh/connectors/sink/OssFile.md index 962533728ecb..e377655074f0 100644 --- a/docs/zh/connectors/sink/OssFile.md +++ b/docs/zh/connectors/sink/OssFile.md @@ -257,7 +257,7 @@ oss文件系统的endpoint端点。 ### 通用选项 -Sink插件常用参数，请参考[Sink common Options]（../Sink common Options.md）了解详细信息。 +Sink插件常用参数，请参考[Sink common Options](../common-options/sink-common-options.md)了解详细信息。 ### max_rows_in_memory [int] diff --git a/docs/zh/connectors/sink/OssJindoFile.md b/docs/zh/connectors/sink/OssJindoFile.md index 9d715056c1de..992d0b1f4903 100644 --- a/docs/zh/connectors/sink/OssJindoFile.md +++ b/docs/zh/connectors/sink/OssJindoFile.md @@ -203,7 +203,7 @@ oss文件系统的端点。 ### common options -Sink插件常用参数，请参考[Sink common Options]（../common-options/sink-common-options.md）了解详细信息。 +Sink插件常用参数，请参考[Sink common Options](../common-options/sink-common-options.md)了解详细信息。 ### max_rows_in_memory [int] @@ -345,4 +345,4 @@ LocalFile { ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/Pulsar.md b/docs/zh/connectors/sink/Pulsar.md index f58ba962ad39..ef0b820c0309 100644 --- a/docs/zh/connectors/sink/Pulsar.md +++ b/docs/zh/connectors/sink/Pulsar.md @@ -41,7 +41,7 @@ Apache Pulsar 的接收连接器。 | pulsar.config | Map | No | - | 除了上述必须由 Pulsar 生产者客户端指定的参数外. | | message.routing.mode | Enum | No | RoundRobinPartition | 要分区的消息的默认路由模式. | | partition_key_fields | array | No | - | 配置哪些字段用作 pulsar 消息的键. | -| common-options | config | no | - | 源插件常用参数，详见源码 [常用选项](../common-options/sink-common-options.md). | +| common-options | config | no | - | Sink插件常用参数，详见 [Sink通用选项](../common-options/sink-common-options.md). | ## 参数解释 @@ -120,7 +120,7 @@ Pulsar 服务的 Service URL 提供程序。要使用客户端库连接到 Pulsa ### 常见选项 -源插件常用参数，详见源码[常用选项](../common-options/sink-common-options.md) . +Sink插件常用参数，详见[Sink通用选项](../common-options/sink-common-options.md). ## 任务示例 diff --git a/docs/zh/connectors/sink/Qdrant.md b/docs/zh/connectors/sink/Qdrant.md index bb8e574ace45..eff0c8e397f4 100644 --- a/docs/zh/connectors/sink/Qdrant.md +++ b/docs/zh/connectors/sink/Qdrant.md @@ -67,8 +67,8 @@ Qdrant 实例的 gRPC 端口。 ### 通用选项 -接收插件的通用参数，请参考[源通用选项](../common-options/sink-common-options.md)了解详情。 +Sink插件通用参数，请参考[Sink通用选项](../common-options/sink-common-options.md)了解详情。 ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/RocketMQ.md b/docs/zh/connectors/sink/RocketMQ.md index df334e7c9c16..db23ad0c0892 100644 --- a/docs/zh/connectors/sink/RocketMQ.md +++ b/docs/zh/connectors/sink/RocketMQ.md @@ -43,7 +43,7 @@ import ChangeLog from '../changelog/connector-rocketmq.md'; | exactly.once | Boolean | 否 | false | 如果为 true，将发送事务消息。 | | max.message.size | int | 否 | 4194304 | 允许的最大消息体大小（字节）。 | | send.message.timeout | int | 否 | 3000 | 发送消息的超时时间（毫秒）。 | -| common-options | config | 否 | - | Sink插件常用参数，请参考[sink common options]（../common-options/sink-common-options.md）了解详细信息。 | +| common-options | config | 否 | - | Sink插件常用参数，请参考[sink common options](../common-options/sink-common-options.md)了解详细信息。 | ### partition.key.fields [array] diff --git a/docs/zh/connectors/sink/S3-Redshift.md b/docs/zh/connectors/sink/S3-Redshift.md index 0b80c5f8f2b6..a51e171e79a3 100644 --- a/docs/zh/connectors/sink/S3-Redshift.md +++ b/docs/zh/connectors/sink/S3-Redshift.md @@ -10,7 +10,7 @@ import ChangeLog from '../changelog/connector-s3-redshift.md'; >提示： ->我们基于[S3File]（S3File.md）来实现这个连接器。因此，您可以使用与S3File相同的配置。 +>我们基于[S3File](S3File.md)来实现这个连接器。因此，您可以使用与S3File相同的配置。 >为了支持更多的文件类型，我们进行了一些权衡，因此我们使用HDFS协议对S3进行内部访问，而这个连接器需要一些hadoop依赖。 >它只支持hadoop版本**2.6.5+**。 @@ -182,7 +182,7 @@ hadoop_s3_properties { ### common options -Sink插件常用参数，请参考[Sink Common Options]（../common-options/sink-common-options.md）了解详细信息。 +Sink插件常用参数，请参考[Sink Common Options](../common-options/sink-common-options.md)了解详细信息。 ## 示例 diff --git a/docs/zh/connectors/sink/SftpFile.md b/docs/zh/connectors/sink/SftpFile.md index 6bdea36121f6..8fad15ff9aa6 100644 --- a/docs/zh/connectors/sink/SftpFile.md +++ b/docs/zh/connectors/sink/SftpFile.md @@ -206,7 +206,7 @@ import ChangeLog from '../changelog/connector-file-sftp.md'; ### common options -Sink插件常用参数，请参考[Sink common Options]（../common-options/sink-common-options.md）了解详细信息。 +Sink插件常用参数，请参考[Sink common Options](../common-options/sink-common-options.md)了解详细信息。 ### max_rows_in_memory diff --git a/docs/zh/connectors/sink/Socket.md b/docs/zh/connectors/sink/Socket.md index 5f992e3b377f..097ce669f2a9 100644 --- a/docs/zh/connectors/sink/Socket.md +++ b/docs/zh/connectors/sink/Socket.md @@ -28,7 +28,7 @@ import ChangeLog from '../changelog/connector-socket.md'; | host | String | 是 | | socket 服务器主机 | | port | Integer | 是 | | socket 服务器端口 | | max_retries | Integer | 否 | 3 | 发送记录的重试失败次数 | -| common-options | | 否 | - | 源插件常用参数，详见[Source common Options]（../sink common-Options.md） | +| common-options | | 否 | - | Sink插件常用参数，详见[Sink common Options](../common-options/sink-common-options.md) | ## 任务示例 diff --git a/docs/zh/connectors/sink/Tablestore.md b/docs/zh/connectors/sink/Tablestore.md index 4fb0aa41f70f..4753407876fe 100644 --- a/docs/zh/connectors/sink/Tablestore.md +++ b/docs/zh/connectors/sink/Tablestore.md @@ -52,7 +52,7 @@ Tablestore 的主键。 ### common 选项 [ config ] -Sink插件常用参数，请参考[Sink common Options]（../common-options/sink-common-options.md）了解详细信息。 +Sink插件常用参数，请参考[Sink common Options](../common-options/sink-common-options.md)了解详细信息。 ## 示例 diff --git a/docs/zh/connectors/source/CosFile.md b/docs/zh/connectors/source/CosFile.md index 6841dcbd4ea7..6accdb86ce85 100644 --- a/docs/zh/connectors/source/CosFile.md +++ b/docs/zh/connectors/source/CosFile.md @@ -446,7 +446,7 @@ abc.* ### common options -源插件常用参数，详见[源端通用选项]（../common-options/source-common-options.md）。 +源插件常用参数，详见[源端通用选项](../common-options/source-common-options.md)。 ## 例如 diff --git a/docs/zh/connectors/source/DB2.md b/docs/zh/connectors/source/DB2.md index 091133dc1742..a2e60b6b8364 100644 --- a/docs/zh/connectors/source/DB2.md +++ b/docs/zh/connectors/source/DB2.md @@ -80,7 +80,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; | partition_num | Int | 否 | job parallelism | 分区计数的数量，只支持正整数。默认值是作业并行性 | | fetch_size | Int | 否 | 0 | 对于返回大量对象的查询，您可以配置查询中使用的行提取大小，通过减少满足选择条件所需的数据库请求次数来提高性能。0表示使用jdbc默认值。 | | properties | Map | 否 | - | 其他连接配置参数，当属性和URL具有相同的参数时，优先级由驱动程序的特定实现决定。例如，在MySQL中，属性优先于URL。 | -| common-options | | 否 | - | source插件常用参数，详见[Source common Options]（../common-options/source-common-options.md） | +| common-options | | 否 | - | source插件常用参数，详见[Source common Options](../common-options/source-common-options.md) | ### 小贴士 @@ -167,4 +167,4 @@ source { ## 变更日志 - \ No newline at end of file + diff --git a/docs/zh/connectors/source/Easysearch.md b/docs/zh/connectors/source/Easysearch.md index 238b27ea8649..9b182bfc873d 100644 --- a/docs/zh/connectors/source/Easysearch.md +++ b/docs/zh/connectors/source/Easysearch.md @@ -114,7 +114,7 @@ PEM或JKS信任存储的路径。运行SeaTunnel的操作系统用户必须能 ### common options -Source插件常用参数，详见[Source common Options]（../common-options/source-common-options.md） +Source插件常用参数，详见[Source common Options](../common-options/source-common-options.md) ## 示例 @@ -202,4 +202,4 @@ source { ## 变更日志 - \ No newline at end of file + From 1827600a2b9c99390d3fc13fc91babff537f6c1a Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 30 Jun 2026 23:02:56 +0800 Subject: [PATCH 086/375] [Test][E2E] Stabilize JDBC schema change assertions (#11200) --- .../jdbc/AbstractSchemaChangeBaseIT.java | 218 +++++++++++++----- .../src/test/resources/ddl/add_columns.sql | 17 +- 2 files changed, 168 insertions(+), 67 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java index 99d1826937a3..8a7dcd8a356a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/java/org/apache/seatunnel/connectors/jdbc/AbstractSchemaChangeBaseIT.java @@ -55,6 +55,7 @@ import java.sql.NClob; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -77,6 +78,23 @@ public abstract class AbstractSchemaChangeBaseIT extends TestSuiteBase implement private static final String SOURCE_DATABASE = "shop"; private static final String SOURCE_TABLE = "products"; + /** + * Deterministic source row used to prove the MySQL CDC reader is consuming binlog events before + * schema-change DDL is executed. + */ + private static final int STREAM_READY_MARKER_ID = 1000; + /** + * Marker value written to the source table so sink-side readiness polling can identify the + * probe row without depending on connector internals. + */ + private static final String STREAM_READY_MARKER_NAME = "__cdc_stream_ready__"; + /** + * Stable payload for the readiness probe row; keeping it constant makes repeated test attempts + * idempotent through the upsert statement. + */ + private static final String STREAM_READY_MARKER_DESCRIPTION = + "wait for binlog stream readiness"; + private static final String MYSQL_HOST = "mysql_cdc_e2e"; private static final String MYSQL_USER_NAME = "mysqluser"; private static final String MYSQL_USER_PASSWORD = "mysqlpw"; @@ -85,6 +103,7 @@ public abstract class AbstractSchemaChangeBaseIT extends TestSuiteBase implement private static final String QUERY = "select * from %s.%s"; private static final String PROJECTION_QUERY = "select id,name,description,weight,add_column1,add_column2,add_column3 from %s.%s"; + private static final String SOURCE_DESC_QUERY = "desc %s.%s"; private static final String SOURCE_QUERY_COLUMNS = "SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' ORDER by COLUMN_NAME"; @@ -294,18 +313,16 @@ public void testMysqlCdcWithSchemaEvolutionCaseExactlyOnce(TestContainer contain } private void assertSchemaEvolution(String sourceTable, String sinkTable) { - await().atMost(120, TimeUnit.SECONDS) + // The exactly-once path can report RUNNING before the sink finishes the first XA batch in + // slower CI environments, so reuse the longer schema assertion timeout for the initial + // data catch-up instead of failing on a transient empty sink table. + await().atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - querySource( - String.format(QUERY, SOURCE_DATABASE, sourceTable)), - querySink( - String.format( - QUERY, - schemaChangeCase.getSchemaName(), - sinkTable) - + ORDER_BY))); + assertTableDataEqualsBySourceColumnOrder( + sourceTable, sinkTable, null)); + + waitForStreamingReady(sourceTable, sinkTable); // case1 add columns with cdc data at same time sourceDatabase.setTemplateName("add_columns").createAndInitialize(); @@ -322,24 +339,19 @@ private void assertSchemaEvolution(String sourceTable, String sinkTable) { assertCaseByDdlName("modify_columns"); } - private void assertCaseByDdlName(String drop_columns) { - sourceDatabase.setTemplateName(drop_columns).createAndInitialize(); + private void assertCaseByDdlName(String ddlTemplateName) { + sourceDatabase.setTemplateName(ddlTemplateName).createAndInitialize(); assertTableStructureAndData(SOURCE_TABLE, schemaChangeCase.getSinkTable2()); } private void assertSchemaEvolutionForAddColumns(String sourceTable, String sinkTable) { - await().atMost(120, TimeUnit.SECONDS) + await().atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - querySource( - String.format(QUERY, SOURCE_DATABASE, sourceTable)), - querySink( - String.format( - QUERY, - schemaChangeCase.getSchemaName(), - sinkTable) - + ORDER_BY))); + assertTableDataEqualsBySourceColumnOrder( + sourceTable, sinkTable, null)); + + waitForStreamingReady(sourceTable, sinkTable); // case1 add columns with cdc data at same time sourceDatabase.setTemplateName("add_columns").createAndInitialize(); @@ -348,26 +360,51 @@ private void assertSchemaEvolutionForAddColumns(String sourceTable, String sinkT } /** - * Schema-change sinks can publish the new rows before the sink table metadata is fully updated. - * Waiting for the column list first avoids racing the add-columns data assertions. + * Snapshot convergence does not prove the MySQL CDC reader has entered steady-state binlog + * consumption. Emit one deterministic DML event and wait until the sink receives it before + * running schema-change DDL bursts. */ - private void waitForSinkColumnsCatchUp(String sourceTable, String sinkTable) { + private void waitForStreamingReady(String sourceTable, String sinkTable) { + executeSourceSql( + String.format( + "INSERT INTO %s.%s (id, name, description, weight) " + + "VALUES (%d, '%s', '%s', 0.0) " + + "ON DUPLICATE KEY UPDATE " + + "name = VALUES(name), description = VALUES(description), weight = VALUES(weight)", + SOURCE_DATABASE, + sourceTable, + STREAM_READY_MARKER_ID, + STREAM_READY_MARKER_NAME, + STREAM_READY_MARKER_DESCRIPTION)); + + String readyQuery = + String.format( + "select id,name,description,weight from %%s.%%s where id = %d order by id", + STREAM_READY_MARKER_ID); await().atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> Assertions.assertIterableEquals( querySource( String.format( - SOURCE_QUERY_COLUMNS, - SOURCE_DATABASE, - sourceTable)), + readyQuery, SOURCE_DATABASE, sourceTable)), querySink( String.format( - schemaChangeCase.getSinkQueryColumns(), + readyQuery, schemaChangeCase.getSchemaName(), sinkTable)))); } + /** + * Schema-change sinks can publish the new rows before the sink table metadata is fully updated. + * Waiting for the column list first avoids racing the add-columns data assertions. + */ + private void waitForSinkColumnsCatchUp(String sourceTable, String sinkTable) { + await().atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> assertColumnNamesEqualsIgnoringPhysicalOrder(sourceTable, sinkTable)); + } + /** * Validates both the new add-columns rows and the projected full-table view once schema * evolution has settled on the sink side. @@ -376,17 +413,8 @@ private void assertAddColumnsDataSynced(String sourceTable, String sinkTable) { await().atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> { - Assertions.assertIterableEquals( - querySource( - String.format(QUERY, SOURCE_DATABASE, sourceTable) - + " where id >= 128"), - querySink( - String.format( - QUERY, - schemaChangeCase.getSchemaName(), - sinkTable) - + " where id >= 128" - + ORDER_BY)); + assertTableDataEqualsBySourceColumnOrder( + sourceTable, sinkTable, "id >= 128"); Assertions.assertIterableEquals( querySource( @@ -408,30 +436,85 @@ private void assertTableStructureAndData(String sourceTable, String sinkTable) { .await() .atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( - () -> - Assertions.assertIterableEquals( - querySource( - String.format( - SOURCE_QUERY_COLUMNS, - SOURCE_DATABASE, - sourceTable)), - querySink( - String.format( - schemaChangeCase.getSinkQueryColumns(), - schemaChangeCase.getSchemaName(), - sinkTable)))); + () -> assertColumnNamesEqualsIgnoringPhysicalOrder(sourceTable, sinkTable)); await().atMost(SCHEMA_ASSERT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) .untilAsserted( () -> - Assertions.assertIterableEquals( - querySource( - String.format(QUERY, SOURCE_DATABASE, sourceTable)), - querySink( - String.format( - QUERY, - schemaChangeCase.getSchemaName(), - sinkTable) - + ORDER_BY))); + assertTableDataEqualsBySourceColumnOrder( + sourceTable, sinkTable, null)); + } + + /** + * JDBC schema evolution can keep the effective column set while materializing a different + * physical order in the sink, so schema assertions should compare normalized column names. + */ + private void assertColumnNamesEqualsIgnoringPhysicalOrder( + String sourceTable, String sinkTable) { + Assertions.assertIterableEquals( + normalizeColumnNames( + querySource( + String.format(SOURCE_QUERY_COLUMNS, SOURCE_DATABASE, sourceTable))), + normalizeColumnNames( + querySink( + String.format( + schemaChangeCase.getSinkQueryColumns(), + schemaChangeCase.getSchemaName(), + sinkTable)))); + } + + /** + * Projects sink data with the current source column order so row assertions stay stable when a + * JDBC sink reorders equivalent columns after applying schema changes. + */ + private void assertTableDataEqualsBySourceColumnOrder( + String sourceTable, String sinkTable, String whereClause) { + List sourceColumns = getSourceColumnNames(sourceTable); + Assertions.assertIterableEquals( + querySource( + buildProjectionQuery( + SOURCE_DATABASE, sourceTable, sourceColumns, whereClause)), + querySink( + buildProjectionQuery( + schemaChangeCase.getSchemaName(), + sinkTable, + sourceColumns, + whereClause))); + } + + /** Reads the current MySQL source schema order that downstream row assertions should follow. */ + private List getSourceColumnNames(String sourceTable) { + List sourceColumns = new ArrayList<>(); + for (List row : + querySource(String.format(SOURCE_DESC_QUERY, SOURCE_DATABASE, sourceTable))) { + sourceColumns.add(String.valueOf(row.get(0))); + } + return sourceColumns; + } + + /** Builds a deterministic projection query without relying on sink-specific physical order. */ + private String buildProjectionQuery( + String database, String table, List columns, String whereClause) { + StringBuilder queryBuilder = + new StringBuilder("select ") + .append(String.join(",", columns)) + .append(" from ") + .append(database) + .append(".") + .append(table); + if (StringUtils.isNotBlank(whereClause)) { + queryBuilder.append(" where ").append(whereClause); + } + return queryBuilder.append(ORDER_BY).toString(); + } + + /** Sorts schema query output by column name so assertions ignore placement-only differences. */ + private List normalizeColumnNames(List> rows) { + List normalizedColumnNames = new ArrayList<>(); + for (List row : rows) { + normalizedColumnNames.add(String.valueOf(row.get(0))); + } + normalizedColumnNames.sort(String::compareTo); + return normalizedColumnNames; } private Connection getJdbcConnection(String connectionType) throws SQLException { @@ -470,6 +553,19 @@ private List> querySource(String sql) { } } + /** + * Executes a source-side DML statement directly against MySQL to produce a CDC event that the + * running SeaTunnel job must consume. + */ + private void executeSourceSql(String sql) { + try (Connection connection = getJdbcConnection("source"); + Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + private List> querySink(String sql) { try (Connection connection = getJdbcConnection("sink")) { ResultSet resultSet = connection.createStatement().executeQuery(sql); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/ddl/add_columns.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/ddl/add_columns.sql index dc334b7b73d4..a213c2073050 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/ddl/add_columns.sql +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/ddl/add_columns.sql @@ -33,9 +33,12 @@ VALUES (110,"scooter","Small 2-wheel scooter",3.14), update products set name = 'hawk9821' where id = 101; delete from products where id = 102; -alter table products ADD COLUMN add_column1 varchar(64) not null default 'yy',ADD COLUMN add_column2 int not null default 1; --- Let the source observe the DDL event before the first row uses the new columns. -DO SLEEP(5); +alter table products ADD COLUMN add_column1 varchar(64) not null default 'yy'; +-- Debezium updates schema history asynchronously, so keep adjacent DDLs in separate binlog batches. +DO SLEEP(10); +alter table products ADD COLUMN add_column2 int not null default 1; +-- Let the source observe both DDL events before the first row uses the new columns. +DO SLEEP(15); update products set name = 'hawk9821' where id = 110; insert into products @@ -51,10 +54,12 @@ values (119,"scooter","Small 2-wheel scooter",3.14,'xx',1), delete from products where id = 118; alter table products ADD COLUMN add_column3 float not null default 1.1; +-- Keep adjacent add-column DDLs in separate schema-history batches. +DO SLEEP(10); ## timestamp is not supported as a cross-database default values for DDL statements alter table products ADD COLUMN add_column4 timestamp; --- The second add-columns batch also needs a short gap before the new-column DML arrives. -DO SLEEP(5); +-- The second add-columns batch also needs a longer gap before the new-column DML arrives. +DO SLEEP(15); delete from products where id = 113; insert into products @@ -71,7 +76,7 @@ update products set name = 'hawk9821' where id = 135; alter table products ADD COLUMN add_column6 varchar(64) not null default 'ff'; -- Keep the final add-column DDL and the follow-up DML in separate CDC batches. -DO SLEEP(5); +DO SLEEP(15); delete from products where id = 115; insert into products values (173,"scooter","Small 2-wheel scooter",3.14,'xx',1,1.1,'2023-02-02 09:09:09','tt'), From e880195fcabfdd0cbd029928cd8d11d0de69e816 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 30 Jun 2026 23:20:59 +0800 Subject: [PATCH 087/375] [Fix][E2E] Wait for Postgres CDC WAL stream readiness (#11202) --- .../seatunnel/cdc/postgres/PostgresCDCIT.java | 239 ++++++++++++++++-- ...to_pg_with_multi_table_mode_one_table.conf | 3 +- ...to_pg_with_multi_table_mode_two_table.conf | 3 +- .../postgrescdc_to_metadata_trans.conf | 1 + .../resources/postgrescdc_to_postgres.conf | 3 +- ...ostgrescdc_to_postgres_test_add_Filed.conf | 1 + ...c_to_postgres_with_custom_primary_key.conf | 3 +- ...dc_to_postgres_with_debezium_to_kafka.conf | 3 +- ...ostgrescdc_to_postgres_with_heartbeat.conf | 3 +- ...c_to_postgres_with_interval_data_type.conf | 1 + ...o_postgres_with_network_address_types.conf | 1 + ...escdc_to_postgres_with_no_primary_key.conf | 3 +- 12 files changed, 232 insertions(+), 32 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/PostgresCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/PostgresCDCIT.java index d4446f1ac9a2..652b3438ce9b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/PostgresCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/PostgresCDCIT.java @@ -44,8 +44,10 @@ import org.apache.kafka.common.TopicPartition; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestTemplate; import org.slf4j.Logger; @@ -125,6 +127,12 @@ public class PostgresCDCIT extends TestSuiteBase implements TestResource { "full_types_no_primary_key_with_debezium"; private static final String SOURCE_SQL_TEMPLATE = "select * from %s.%s order by id"; + private static final String GENERATED_SLOT_PREFIX = "seatunnel_"; + /** + * Debezium JSON change events can lag under CI load, so snapshot and DML record waits use the + * same timeout budget. + */ + private static final long DEBEZIUM_JSON_RECORD_WAIT_TIMEOUT_SECONDS = 180L; // kafka container private static final String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:7.0.9"; @@ -251,6 +259,28 @@ private Properties kafkaConsumerConfig() { return props; } + /** + * Replication slots are shared inside the reused Postgres test container, so each CDC job needs + * an isolated slot to avoid cross-test collisions when streaming jobs overlap. + */ + private String createSlotName() { + return GENERATED_SLOT_PREFIX + Long.toHexString(JobIdGenerator.newJobId()); + } + + private String toSlotVariable(String slotName) { + return "slot_name=" + slotName; + } + + @BeforeEach + public void beforeEach() { + cleanupGeneratedReplicationSlots(); + } + + @AfterEach + public void afterEach() { + cleanupGeneratedReplicationSlots(); + } + private List getKafkaData() { long endOffset; long lastProcessedOffset = -1L; @@ -281,6 +311,8 @@ private List getKafkaData() { type = {EngineType.SPARK, EngineType.FLINK}, disabledReason = "Currently Only support Zeta engine") public void testPostgresCdcWithDebeziumJsonFormat(TestContainer container) { + String slotName = createSlotName(); + String slotVariable = toSlotVariable(slotName); try { log.info( @@ -289,14 +321,15 @@ public void testPostgresCdcWithDebeziumJsonFormat(TestContainer container) { query(getQuerySQL(POSTGRESQL_SCHEMA, SOURCE_TABLE_NO_PRIMARY_KEY_DEBEZIUM))); Properties props = kafkaConsumerConfig(); - props.put(ConsumerConfig.GROUP_ID_CONFIG, "group-debezium-json-format"); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "group-" + slotName); kafkaConsumer = new KafkaConsumer<>(props); CompletableFuture.supplyAsync( () -> { try { container.executeJob( - "/postgrescdc_to_postgres_with_debezium_to_kafka.conf"); + "/postgrescdc_to_postgres_with_debezium_to_kafka.conf", + Collections.singletonList(slotVariable)); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -305,35 +338,46 @@ public void testPostgresCdcWithDebeziumJsonFormat(TestContainer container) { }); AtomicReference dataSize = new AtomicReference<>(0); - await().atMost(1000 * 60 * 3, TimeUnit.MILLISECONDS) + await().atMost(DEBEZIUM_JSON_RECORD_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS) .untilAsserted( () -> { dataSize.updateAndGet(v -> v + getKafkaData().size()); Assertions.assertEquals(1, dataSize.get()); }); - // insert update delete - upsertDeleteSourceTable(POSTGRESQL_SCHEMA, SOURCE_TABLE_NO_PRIMARY_KEY_DEBEZIUM); - - await().atMost(1000 * 60 * 3, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - dataSize.updateAndGet(v -> v + getKafkaData().size()); - Assertions.assertEquals(5, dataSize.get()); - }); + // The snapshot row can reach Kafka before the WAL stream is fully attached. + // Wait for the replication slot to become active so the following DML is emitted as + // incremental change events instead of being skipped during the snapshot handoff. + waitForReplicationSlotActive(slotName); + // Keep each row-level mutation isolated so the exactly-once Kafka sink does not + // collapse same-key changes into only the final visible state under CI pressure. + insertSourceTableRow(POSTGRESQL_SCHEMA, SOURCE_TABLE_NO_PRIMARY_KEY_DEBEZIUM, 2); + awaitKafkaRecordCount(dataSize, 2); + insertSourceTableRow(POSTGRESQL_SCHEMA, SOURCE_TABLE_NO_PRIMARY_KEY_DEBEZIUM, 3); + awaitKafkaRecordCount(dataSize, 3); + deleteSourceTableRow(POSTGRESQL_SCHEMA, SOURCE_TABLE_NO_PRIMARY_KEY_DEBEZIUM, 2); + awaitKafkaRecordCount(dataSize, 4); + updateSourceTableBigField( + POSTGRESQL_SCHEMA, SOURCE_TABLE_NO_PRIMARY_KEY_DEBEZIUM, 3, 10000); + awaitKafkaRecordCount(dataSize, 5); } finally { clearTable(POSTGRESQL_SCHEMA, SOURCE_TABLE_NO_PRIMARY_KEY_DEBEZIUM); - kafkaConsumer.close(); + if (kafkaConsumer != null) { + kafkaConsumer.close(); + } } } @TestTemplate public void testMPostgresCdcCheckDataE2e(TestContainer container) { + String slotVariable = toSlotVariable(createSlotName()); try { CompletableFuture.supplyAsync( () -> { try { - container.executeJob("/postgrescdc_to_postgres.conf"); + container.executeJob( + "/postgrescdc_to_postgres.conf", + Collections.singletonList(slotVariable)); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -373,6 +417,7 @@ public void testMPostgresCdcCheckDataE2e(TestContainer container) { disabledReason = "Heartbeat action query is currently only supported by the zeta engine.") public void testMPostgresCdcCheckDataE2eWithHeartbeat(TestContainer container) { + String slotVariable = toSlotVariable(createSlotName()); executeSql( "CREATE TABLE IF NOT EXISTS " + POSTGRESQL_SCHEMA @@ -385,7 +430,9 @@ public void testMPostgresCdcCheckDataE2eWithHeartbeat(TestContainer container) { CompletableFuture.supplyAsync( () -> { try { - container.executeJob("/postgrescdc_to_postgres_with_heartbeat.conf"); + container.executeJob( + "/postgrescdc_to_postgres_with_heartbeat.conf", + Collections.singletonList(slotVariable)); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -435,11 +482,14 @@ public void testMPostgresCdcCheckDataE2eWithHeartbeat(TestContainer container) { public void testMPostgresCdcMetadataTrans(TestContainer container) throws InterruptedException { Long jobId = JobIdGenerator.newJobId(); + String slotVariable = toSlotVariable(createSlotName()); CompletableFuture.runAsync( () -> { try { container.executeJob( - "/postgrescdc_to_postgres.conf", String.valueOf(jobId)); + "/postgrescdc_to_postgres.conf", + String.valueOf(jobId), + slotVariable); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -475,13 +525,15 @@ public void testMPostgresCdcMetadataTrans(TestContainer container) throws Interr type = {EngineType.SPARK}, disabledReason = "Currently SPARK do not support cdc") public void testPostgresCdcMultiTableE2e(TestContainer container) { + String slotVariable = toSlotVariable(createSlotName()); try { CompletableFuture.supplyAsync( () -> { try { container.executeJob( - "/pgcdc_to_pg_with_multi_table_mode_two_table.conf"); + "/pgcdc_to_pg_with_multi_table_mode_two_table.conf", + Collections.singletonList(slotVariable)); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -561,13 +613,15 @@ public void testPostgresCdcMultiTableE2e(TestContainer container) { public void testMultiTableWithRestore(TestContainer container) throws IOException, InterruptedException { Long jobId = JobIdGenerator.newJobId(); + String slotVariable = toSlotVariable(createSlotName()); try { CompletableFuture.supplyAsync( () -> { try { return container.executeJob( "/pgcdc_to_pg_with_multi_table_mode_one_table.conf", - String.valueOf(jobId)); + String.valueOf(jobId), + slotVariable); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -601,7 +655,8 @@ public void testMultiTableWithRestore(TestContainer container) try { container.restoreJob( "/pgcdc_to_pg_with_multi_table_mode_two_table.conf", - String.valueOf(jobId)); + String.valueOf(jobId), + slotVariable); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -660,13 +715,15 @@ public void testMultiTableWithRestore(TestContainer container) public void testAddFieldWithRestore(TestContainer container) throws IOException, InterruptedException { Long jobId = JobIdGenerator.newJobId(); + String slotVariable = toSlotVariable(createSlotName()); try { CompletableFuture.supplyAsync( () -> { try { return container.executeJob( "/postgrescdc_to_postgres_test_add_Filed.conf", - String.valueOf(jobId)); + String.valueOf(jobId), + slotVariable); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -702,7 +759,8 @@ public void testAddFieldWithRestore(TestContainer container) try { container.restoreJob( "/postgrescdc_to_postgres_test_add_Filed.conf", - String.valueOf(jobId)); + String.valueOf(jobId), + slotVariable); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -741,13 +799,15 @@ public void testAddFieldWithRestore(TestContainer container) @TestTemplate public void testPostgresCdcCheckDataWithNoPrimaryKey(TestContainer container) throws Exception { + String slotVariable = toSlotVariable(createSlotName()); try { CompletableFuture.supplyAsync( () -> { try { container.executeJob( - "/postgrescdc_to_postgres_with_no_primary_key.conf"); + "/postgrescdc_to_postgres_with_no_primary_key.conf", + Collections.singletonList(slotVariable)); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -789,13 +849,15 @@ public void testPostgresCdcCheckDataWithNoPrimaryKey(TestContainer container) th @TestTemplate public void testPostgresCdcCheckDataWithCustomPrimaryKey(TestContainer container) { + String slotVariable = toSlotVariable(createSlotName()); try { CompletableFuture.supplyAsync( () -> { try { container.executeJob( - "/postgrescdc_to_postgres_with_custom_primary_key.conf"); + "/postgrescdc_to_postgres_with_custom_primary_key.conf", + Collections.singletonList(slotVariable)); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -838,13 +900,15 @@ public void testPostgresCdcCheckDataWithCustomPrimaryKey(TestContainer container @TestTemplate public void testPostgresCdcCheckDataWithIntervalDataType(TestContainer container) throws Exception { + String slotVariable = toSlotVariable(createSlotName()); try { CompletableFuture.supplyAsync( () -> { try { container.executeJob( - "/postgrescdc_to_postgres_with_interval_data_type.conf"); + "/postgrescdc_to_postgres_with_interval_data_type.conf", + Collections.singletonList(slotVariable)); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -868,12 +932,14 @@ public void testPostgresCdcCheckDataWithIntervalDataType(TestContainer container @TestTemplate public void testPostgresCdcCheckDataWithNetworkAddressTypes(TestContainer container) { + String slotVariable = toSlotVariable(createSlotName()); try { CompletableFuture.supplyAsync( () -> { try { container.executeJob( - "/postgrescdc_to_postgres_with_network_address_types.conf"); + "/postgrescdc_to_postgres_with_network_address_types.conf", + Collections.singletonList(slotVariable)); } catch (Exception e) { log.error("Commit task exception :" + e.getMessage()); throw new RuntimeException(e); @@ -928,6 +994,84 @@ private Connection getJdbcConnection() throws SQLException { POSTGRES_CONTAINER.getPassword()); } + /** + * The Postgres container is shared across all test methods in this class, so stale generated + * replication slots must be dropped before the next CDC job starts. + */ + private void cleanupGeneratedReplicationSlots() { + for (String slotName : listGeneratedReplicationSlots()) { + await().ignoreExceptions() + .atMost(30, TimeUnit.SECONDS) + .untilAsserted(() -> dropReplicationSlot(slotName)); + } + } + + /** + * The Debezium JSON Kafka test verifies every row-level change event, so it must wait until the + * WAL stream is active before mutating the source table. + */ + private void waitForReplicationSlotActive(String slotName) { + await().ignoreExceptions() + .atMost(30, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assertions.assertTrue( + isReplicationSlotActive(slotName), + "Replication slot is not active yet: " + slotName)); + } + + private List listGeneratedReplicationSlots() { + List slotNames = new ArrayList<>(); + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = + statement.executeQuery( + "SELECT slot_name FROM pg_replication_slots WHERE slot_name LIKE '" + + GENERATED_SLOT_PREFIX + + "%'")) { + while (resultSet.next()) { + slotNames.add(resultSet.getString("slot_name")); + } + return slotNames; + } catch (SQLException e) { + throw new RuntimeException("Failed to query generated replication slots", e); + } + } + + private boolean isReplicationSlotActive(String slotName) { + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = + statement.executeQuery( + "SELECT active FROM pg_replication_slots WHERE slot_name = '" + + slotName + + "'")) { + return resultSet.next() && resultSet.getBoolean("active"); + } catch (SQLException e) { + throw new RuntimeException("Failed to query replication slot activity: " + slotName, e); + } + } + + private void dropReplicationSlot(String slotName) { + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = + statement.executeQuery( + "SELECT active FROM pg_replication_slots WHERE slot_name = '" + + slotName + + "'")) { + if (!resultSet.next()) { + return; + } + Assertions.assertFalse( + resultSet.getBoolean("active"), + "Replication slot is still active: " + slotName); + statement.execute("SELECT pg_drop_replication_slot('" + slotName + "')"); + } catch (SQLException e) { + throw new RuntimeException("Failed to drop replication slot: " + slotName, e); + } + } + protected void initializePostgresTable(PostgreSQLContainer container, String sqlFile) { final String ddlFile = String.format("ddl/%s.sql", sqlFile); final URL ddlTestFile = PostgresCDCIT.class.getClassLoader().getResource(ddlFile); @@ -1006,6 +1150,51 @@ private void insertSourceTableForAddFields(String database, String tableName) { + " VALUES (2, '2', 32767, 65535, 2147483647);"); } + /** Wait until the Debezium JSON test observes the expected number of Kafka change records. */ + private void awaitKafkaRecordCount(AtomicReference dataSize, int expectedCount) { + await().atMost(DEBEZIUM_JSON_RECORD_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> { + dataSize.updateAndGet(v -> v + getKafkaData().size()); + Assertions.assertEquals(expectedCount, dataSize.get()); + }); + } + + /** + * Insert one row so the CDC test can assert the emitted Kafka record before the next mutation. + */ + private void insertSourceTableRow(String database, String tableName, int id) { + executeSql( + "INSERT INTO " + + database + + "." + + tableName + + " VALUES (" + + id + + ", '2', 32767, 65535, 2147483647, 5.5, 6.6, 123.12345, 404.4443, true,\n" + + " 'Hello World', 'a', 'abc', 'abcd..xyz', '2020-07-17 18:00:22.123', '2020-07-17 18:00:22.123456',\n" + + " '2020-07-17', '18:00:22', 500, 88, '192.168.1.1');"); + } + + /** Delete one row after its insert event is already visible in Kafka. */ + private void deleteSourceTableRow(String database, String tableName, int id) { + executeSql("DELETE FROM " + database + "." + tableName + " where id = " + id + ";"); + } + + /** Update the inserted row in a separate step so CI can assert the incremental change event. */ + private void updateSourceTableBigField(String database, String tableName, int id, int value) { + executeSql( + "UPDATE " + + database + + "." + + tableName + + " SET f_big = " + + value + + " where id = " + + id + + ";"); + } + private void upsertDeleteSourceTable(String database, String tableName) { executeSql( diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_one_table.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_one_table.conf index e29e41e18300..8a09f4bf7202 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_one_table.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_one_table.conf @@ -37,6 +37,7 @@ source { table-names = ["postgres_cdc.inventory.postgres_cdc_table_1"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" + slot.name = ${slot_name} } } @@ -59,4 +60,4 @@ sink { tablePrefix = "sink_" primary_keys = ["id"] } -} \ No newline at end of file +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_two_table.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_two_table.conf index 0f4f516152f7..cf7112ec735f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_two_table.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_two_table.conf @@ -37,6 +37,7 @@ source { table-names = ["postgres_cdc.inventory.postgres_cdc_table_1","postgres_cdc.inventory.postgres_cdc_table_2"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" + slot.name = ${slot_name} } } @@ -59,4 +60,4 @@ sink { tablePrefix = "sink_" primary_keys = ["id"] } -} \ No newline at end of file +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_metadata_trans.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_metadata_trans.conf index 1cc7b87d88b3..09265a3589d8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_metadata_trans.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_metadata_trans.conf @@ -37,6 +37,7 @@ source { table-names = ["postgres_cdc.inventory.postgres_cdc_table_1"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" + slot.name = ${slot_name} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres.conf index 5e36bffe8517..41979ea299fb 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres.conf @@ -37,6 +37,7 @@ source { table-names = ["postgres_cdc.inventory.postgres_cdc_table_1"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" + slot.name = ${slot_name} } } @@ -58,4 +59,4 @@ sink { table = inventory.sink_postgres_cdc_table_1 primary_keys = ["id"] } -} \ No newline at end of file +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_test_add_Filed.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_test_add_Filed.conf index cc0d114a9257..b5aaad0076aa 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_test_add_Filed.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_test_add_Filed.conf @@ -37,6 +37,7 @@ source { table-names = ["postgres_cdc.inventory.postgres_cdc_table_3"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" + slot.name = ${slot_name} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_custom_primary_key.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_custom_primary_key.conf index 79e99b48ac46..91797b6a5d4b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_custom_primary_key.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_custom_primary_key.conf @@ -38,6 +38,7 @@ source { url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" exactly_once = true + slot.name = ${slot_name} table-names-config = [ { table = "postgres_cdc.inventory.full_types_no_primary_key" @@ -65,4 +66,4 @@ sink { table = inventory.sink_postgres_cdc_table_1 primary_keys = ["id"] } -} \ No newline at end of file +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_debezium_to_kafka.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_debezium_to_kafka.conf index ee9b99942233..09d6abed2c81 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_debezium_to_kafka.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_debezium_to_kafka.conf @@ -36,6 +36,7 @@ source { url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" exactly_once = true + slot.name = ${slot_name} table-names-config = [ { table = "postgres_cdc.inventory.full_types_no_primary_key_with_debezium" @@ -64,4 +65,4 @@ sink { "value.converter.schemas.enable": false } } -} \ No newline at end of file +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_heartbeat.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_heartbeat.conf index 047ec2b0e4e0..93c64d0769da 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_heartbeat.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_heartbeat.conf @@ -37,6 +37,7 @@ source { table-names = ["postgres_cdc.inventory.postgres_cdc_table_1"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" + slot.name = ${slot_name} debezium { heartbeat.interval.ms = 100 heartbeat.action.query = "INSERT INTO inventory.heartbeat (ts) VALUES (NOW())" @@ -62,4 +63,4 @@ sink { table = inventory.sink_postgres_cdc_table_1 primary_keys = ["id"] } -} \ No newline at end of file +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_interval_data_type.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_interval_data_type.conf index 64cd3de34ae4..8d85875582e3 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_interval_data_type.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_interval_data_type.conf @@ -36,6 +36,7 @@ source { schema-names = ["inventory"] table-names = ["postgres_cdc.inventory.postgres_cdc_table_4"] base-url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" + slot.name = ${slot_name} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_network_address_types.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_network_address_types.conf index 6c096bca14e3..b7bdfdc34f2d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_network_address_types.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_network_address_types.conf @@ -36,6 +36,7 @@ source { schema-names = ["inventory"] table-names = ["postgres_cdc.inventory.postgres_cdc_table_5"] base-url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" + slot.name = ${slot_name} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_no_primary_key.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_no_primary_key.conf index baefbec3e54f..5eb34e4643fe 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_no_primary_key.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_no_primary_key.conf @@ -38,6 +38,7 @@ source { decoding.plugin.name = "decoderbufs" table-names = ["postgres_cdc.inventory.full_types_no_primary_key"] exactly_once = false + slot.name = ${slot_name} } } @@ -59,4 +60,4 @@ sink { table = inventory.sink_postgres_cdc_table_1 primary_keys = ["id"] } -} \ No newline at end of file +} From b60eb4c56258babc91b1b9149a392be65c253718 Mon Sep 17 00:00:00 2001 From: luxiaolong Date: Wed, 1 Jul 2026 11:30:28 +0800 Subject: [PATCH 088/375] [Feature][Connector-V2] PR1: Pass sink table-options into auto-created MySQL target tables (#11101) Co-authored-by: luxiaolong-ct <294671909+luxiaolong-ct@users.noreply.github.com> Co-authored-by: det101 --- docs/en/connectors/sink/Jdbc.md | 39 ++++ docs/zh/connectors/sink/Jdbc.md | 39 ++++ .../options/SinkConnectorCommonOptions.java | 14 ++ .../jdbc/internal/dialect/JdbcDialect.java | 19 ++ .../internal/dialect/mysql/MysqlDialect.java | 27 +++ .../seatunnel/jdbc/sink/JdbcSinkFactory.java | 7 + .../JdbcTableOptionsConditionExtension.java | 57 +++++ .../jdbc/sink/JdbcTableOptionsValidator.java | 57 +++++ .../mysql/MysqlCreateTableSqlBuilderTest.java | 32 +++ .../dialect/mysql/MysqlDialectTest.java | 69 ++++++ ...dbcTableOptionsConditionExtensionTest.java | 97 +++++++++ .../jdbc/JdbcMysqlTableOptionsIT.java | 203 ++++++++++++++++++ .../jdbc_mysql_sink_with_table_options.conf | 54 +++++ 13 files changed, 714 insertions(+) create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsConditionExtension.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsValidator.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsConditionExtensionTest.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlTableOptionsIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/resources/jdbc_mysql_sink_with_table_options.conf diff --git a/docs/en/connectors/sink/Jdbc.md b/docs/en/connectors/sink/Jdbc.md index c5ee8ccd798f..dfb1b77a4334 100644 --- a/docs/en/connectors/sink/Jdbc.md +++ b/docs/en/connectors/sink/Jdbc.md @@ -60,6 +60,7 @@ support `Xa transactions`. You can set `is_exactly_once=true` to enable it. | data_save_mode | Enum | No | APPEND_DATA | | custom_sql | String | No | - | | enable_upsert | Boolean | No | true | +| table_options | Map | No | - | | use_copy_statement | Boolean | No | false | | oracle_insert_mode | Enum | No | CONVENTIONAL | | create_index | Boolean | No | true | @@ -230,6 +231,44 @@ When data_save_mode selects CUSTOM_PROCESSING, you should fill in the CUSTOM_SQL Note: in sink `query` mode, `custom_sql` is not executed. This behavior is a current limitation of JDBC sink. +### table_options [Map] + +Sink-specific table options applied when SaveMode creates the target table (DDL phase). They take effect only when `schema_save_mode` triggers table creation, such as `CREATE_SCHEMA_WHEN_NOT_EXIST` or `RECREATE_SCHEMA`. They do **not** affect INSERT/UPSERT at runtime and do **not** run `ALTER TABLE` on existing tables. + +Current support: + +| Dialect | Supported | Allowed keys | +|---------|-----------|--------------| +| MySQL | Yes | `engine`, `charset`, `collate` | +| Other JDBC dialects | No | Non-empty `table_options` fails validation at job submission | + +Invalid or unsupported keys are validated early via `JdbcSinkFactory` option rules (`--check` and job submission), not only at runtime DDL. + +Example (MySQL auto-create with engine and charset): + +```hocon +sink { + Jdbc { + url = "jdbc:mysql://localhost:3307/mydb" + driver = "com.mysql.cj.jdbc.Driver" + username = "root" + password = "password" + database = "mydb" + table = "orders" + generate_sink_sql = true + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + primary_keys = ["id"] + table_options = { + "engine" = "InnoDB" + "charset" = "utf8mb4" + "collate" = "utf8mb4_general_ci" + } + } +} +``` + +The generated `CREATE TABLE` statement appends `ENGINE`, `DEFAULT CHARSET`, and `COLLATE` clauses. Keys outside the dialect whitelist (for example `bucket_num`) fail during job submission. + ### enable_upsert [boolean] Enable upsert by primary_keys exist, If the task has no key duplicate data, setting this parameter to `false` can speed up data import diff --git a/docs/zh/connectors/sink/Jdbc.md b/docs/zh/connectors/sink/Jdbc.md index 732eaa8df6c8..17783d2eaa7f 100644 --- a/docs/zh/connectors/sink/Jdbc.md +++ b/docs/zh/connectors/sink/Jdbc.md @@ -58,6 +58,7 @@ import ChangeLog from '../changelog/connector-jdbc.md'; | data_save_mode | Enum | 否 | APPEND_DATA | | custom_sql | String | 否 | - | | enable_upsert | Boolean | 否 | true | +| table_options | Map | 否 | - | | use_copy_statement | Boolean | 否 | false | | oracle_insert_mode | Enum | 否 | CONVENTIONAL | | access_key_id | String | 否 | | @@ -218,6 +219,44 @@ Sink插件常用参数，请参考 [Sink常用选项](../common-options/sink-com `CUSTOM_PROCESSING`：允许用户自定义数据处理方式
`ERROR_WHEN_DATA_EXISTS`：当有数据时抛出错误
+### table_options [Map] + +Sink 在自动建表（SaveMode DDL）时附加的表级选项。仅在 `schema_save_mode` 触发建表时生效，例如 `CREATE_SCHEMA_WHEN_NOT_EXIST`、`RECREATE_SCHEMA`；**不影响**数据写入阶段的 INSERT/UPSERT，也**不会**对已存在表执行 `ALTER TABLE`。 + +当前支持情况： + +| 方言 | 是否支持 | 可用 key | +|------|----------|----------| +| MySQL | 是 | `engine`、`charset`、`collate` | +| 其他 JDBC 方言 | 否 | 配置非空 `table_options` 时任务启动即校验失败 | + +非法或不支持的 key 会在 `JdbcSinkFactory` 的 option 规则阶段提前校验（`--check` 与作业提交），而非仅在运行时 DDL 阶段失败。 + +示例（MySQL 自动建表时指定存储引擎与字符集）： + +```hocon +sink { + Jdbc { + url = "jdbc:mysql://localhost:3307/mydb" + driver = "com.mysql.cj.jdbc.Driver" + username = "root" + password = "password" + database = "mydb" + table = "orders" + generate_sink_sql = true + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + primary_keys = ["id"] + table_options = { + "engine" = "InnoDB" + "charset" = "utf8mb4" + "collate" = "utf8mb4_general_ci" + } + } +} +``` + +生成的 DDL 会追加 `ENGINE`、`DEFAULT CHARSET`、`COLLATE` 子句。未在白名单内的 key（如 `bucket_num`）会在作业提交阶段报错。 + ### custom_sql [String] 当`data_save_mode`选择`CUSTOM_PROCESSING`时，需要填写`CUSTOM_SQL`参数。该参数通常填写一条可以执行的SQL。SQL将在同步任务之前执行 diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/SinkConnectorCommonOptions.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/SinkConnectorCommonOptions.java index c245d908dcaf..1808aad5069c 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/SinkConnectorCommonOptions.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/SinkConnectorCommonOptions.java @@ -21,6 +21,9 @@ import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.Options; +import java.util.HashMap; +import java.util.Map; + public class SinkConnectorCommonOptions extends ConnectorCommonOptions { @Experimental @@ -29,4 +32,15 @@ public class SinkConnectorCommonOptions extends ConnectorCommonOptions { .intType() .defaultValue(1) .withDescription("The replica number of multi table sink writer"); + + @Experimental + public static Option> TABLE_OPTIONS = + Options.key("table_options") + .mapType() + .defaultValue(new HashMap<>()) + .withDescription( + "Experimental sink-specific table options applied when auto-creating " + + "target tables during SaveMode. Allowed keys and semantics are " + + "defined and validated by each sink connector and/or database " + + "dialect; see the connector documentation for supported options."); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialect.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialect.java index d943e22d8ada..95055a7d7095 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialect.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/JdbcDialect.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; +import org.apache.seatunnel.api.common.SeaTunnelAPIErrorCode; import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.converter.BasicTypeDefine; @@ -32,6 +33,7 @@ import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent; import org.apache.seatunnel.api.table.type.SqlType; import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcConnectionConfig; +import org.apache.seatunnel.connectors.seatunnel.jdbc.exception.JdbcConnectorException; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.connection.JdbcConnectionProvider; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.connection.SimpleJdbcConnectionProvider; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.JdbcRowConverter; @@ -904,4 +906,21 @@ default String getCollateSql(String collate) { default String dualTable() { return ""; } + + /** + * Validate sink table options for auto-create mode. + * + *

Default behavior is fail-fast for any non-empty table options. Dialects should override + * this when they support sink-specific table options. + */ + default void validateTableOptions(Map tableOptions) { + if (tableOptions == null || tableOptions.isEmpty()) { + return; + } + throw new JdbcConnectorException( + SeaTunnelAPIErrorCode.CONFIG_VALIDATION_FAILED, + String.format( + "JDBC table_options are not supported for dialect '%s' yet.", + dialectName())); + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialect.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialect.java index c63cbc409794..09fefdd56413 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialect.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialect.java @@ -19,9 +19,11 @@ import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; +import org.apache.seatunnel.api.common.SeaTunnelAPIErrorCode; import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.api.table.converter.BasicTypeDefine; import org.apache.seatunnel.api.table.converter.TypeConverter; +import org.apache.seatunnel.connectors.seatunnel.jdbc.exception.JdbcConnectorException; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.JdbcRowConverter; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; @@ -41,11 +43,14 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; @Slf4j @@ -53,6 +58,9 @@ public class MysqlDialect implements JdbcDialect { private static final List NOT_SUPPORTED_DEFAULT_VALUES = Arrays.asList(MysqlType.BLOB, MysqlType.TEXT, MysqlType.JSON, MysqlType.GEOMETRY); + private static final Set SUPPORTED_TABLE_OPTIONS = + Collections.unmodifiableSet( + new LinkedHashSet<>(Arrays.asList("engine", "charset", "collate"))); public String fieldIde = FieldIdeEnum.ORIGINAL.getValue(); @@ -389,4 +397,23 @@ public boolean needsQuotesWithDefaultValue(BasicTypeDefine columnDefine) { return false; } } + + @Override + public void validateTableOptions(Map tableOptions) { + if (tableOptions == null || tableOptions.isEmpty()) { + return; + } + + Set unsupportedOptions = new LinkedHashSet<>(tableOptions.keySet()); + unsupportedOptions.removeAll(SUPPORTED_TABLE_OPTIONS); + if (!unsupportedOptions.isEmpty()) { + throw new JdbcConnectorException( + SeaTunnelAPIErrorCode.CONFIG_VALIDATION_FAILED, + String.format( + "Unsupported JDBC table_options for dialect '%s': %s. Supported keys: %s", + dialectName(), + String.join(", ", unsupportedOptions), + String.join(", ", SUPPORTED_TABLE_OPTIONS))); + } + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java index bf035c2c0b72..33cb078fc12a 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkFactory.java @@ -73,6 +73,7 @@ private ReadonlyConfig getCatalogOptions(TableSinkFactoryContext context) { @Override public TableSink createSink(TableSinkFactoryContext context) { ReadonlyConfig config = context.getOptions(); + Map sinkTableOptions = config.get(SinkConnectorCommonOptions.TABLE_OPTIONS); CatalogTable catalogTable = context.getCatalogTable(); ReadonlyConfig catalogOptions = getCatalogOptions(context); Optional optionalTable = config.getOptional(JdbcSinkOptions.TABLE); @@ -182,6 +183,7 @@ public TableSink createSink(TableSinkFactoryContext context) { final ReadonlyConfig options = config; JdbcSinkConfig sinkConfig = JdbcSinkConfig.of(config); FieldIdeEnum fieldIdeEnum = config.get(JdbcSinkOptions.FIELD_IDE); + catalogTable.getOptions().putAll(sinkTableOptions); catalogTable .getOptions() .put("fieldIde", fieldIdeEnum == null ? null : fieldIdeEnum.getValue()); @@ -246,6 +248,11 @@ public OptionRule optionRule() { JdbcSinkOptions.TABLE_SUFFIX, SinkConnectorCommonOptions.MULTI_TABLE_SINK_REPLICA, JdbcSinkOptions.DIALECT) + .optional( + SinkConnectorCommonOptions.TABLE_OPTIONS, + Conditions.extension( + SinkConnectorCommonOptions.TABLE_OPTIONS, + JdbcTableOptionsConditionExtension.INSTANCE)) .conditional( JdbcSinkOptions.IS_EXACTLY_ONCE, true, diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsConditionExtension.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsConditionExtension.java new file mode 100644 index 000000000000..dc74d956c88d --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsConditionExtension.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.connectors.seatunnel.jdbc.exception.JdbcConnectorException; + +import java.util.Map; + +/** + * Early validation for JDBC sink {@code table_options}. Delegates to {@link + * JdbcTableOptionsValidator} so dialect-specific rules are defined on {@link + * org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect}. + */ +public class JdbcTableOptionsConditionExtension implements ConditionExtension> { + + public static final JdbcTableOptionsConditionExtension INSTANCE = + new JdbcTableOptionsConditionExtension(); + + private JdbcTableOptionsConditionExtension() {} + + @Override + public String description() { + return "must use dialect-specific keys supported by the JDBC sink (see JDBC connector docs)"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Map value) + throws OptionValidationException { + if (value == null || value.isEmpty()) { + return true; + } + try { + JdbcTableOptionsValidator.validate(config, value); + return true; + } catch (JdbcConnectorException e) { + throw new OptionValidationException(e.getMessage()); + } + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsValidator.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsValidator.java new file mode 100644 index 000000000000..00d6b69c846a --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsValidator.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.options.SinkConnectorCommonOptions; +import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcSinkConfig; +import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcSinkOptions; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectLoader; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.dialectenum.FieldIdeEnum; + +import java.util.Collections; +import java.util.Map; + +/** Validates sink {@code table_options} via the resolved {@link JdbcDialect}. */ +public final class JdbcTableOptionsValidator { + + private JdbcTableOptionsValidator() {} + + public static void validate(ReadonlyConfig config, Map tableOptions) { + if (tableOptions == null || tableOptions.isEmpty()) { + return; + } + JdbcSinkConfig sinkConfig = JdbcSinkConfig.of(config); + FieldIdeEnum fieldIdeEnum = config.get(JdbcSinkOptions.FIELD_IDE); + JdbcDialect dialect = + JdbcDialectLoader.load( + sinkConfig.getJdbcConnectionConfig().getUrl(), + sinkConfig.getJdbcConnectionConfig().getCompatibleMode(), + sinkConfig.getJdbcConnectionConfig().getDialect(), + fieldIdeEnum == null ? null : fieldIdeEnum.getValue()); + dialect.validateTableOptions(tableOptions); + } + + public static void validate(ReadonlyConfig config) { + validate( + config, + config.getOptional(SinkConnectorCommonOptions.TABLE_OPTIONS) + .orElse(Collections.emptyMap())); + } +} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MysqlCreateTableSqlBuilderTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MysqlCreateTableSqlBuilderTest.java index 0f62186e20e1..21740a282f34 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MysqlCreateTableSqlBuilderTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MysqlCreateTableSqlBuilderTest.java @@ -40,7 +40,9 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.Map; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -150,6 +152,36 @@ public void testBuild() { Assertions.assertEquals(expectSkipIndex, createTableSqlSkipIndex); } + @Test + public void testBuildCreateTableSqlWithTableOptions() { + TablePath tablePath = TablePath.of("test_db", "test_table"); + TableSchema tableSchema = + TableSchema.builder() + .column(PhysicalColumn.of("id", BasicType.LONG_TYPE, 0, false, null, "id")) + .primaryKey(PrimaryKey.of("id", Lists.newArrayList("id"))) + .build(); + Map options = new HashMap<>(); + options.put(MySqlCatalog.TABLE_OPTION_ENGINE, "InnoDB"); + options.put(MySqlCatalog.TABLE_OPTION_CHARSET, "utf8mb4"); + options.put(MySqlCatalog.TABLE_OPTION_COLLATE, "utf8mb4_unicode_ci"); + CatalogTable catalogTable = + CatalogTable.of( + TableIdentifier.of("test_catalog", "test_db", "test_table"), + tableSchema, + options, + Collections.emptyList(), + "table with options"); + + String createTableSql = + MysqlCreateTableSqlBuilder.builder( + tablePath, catalogTable, MySqlTypeConverter.DEFAULT_INSTANCE, true) + .build(DatabaseIdentifier.MYSQL); + + Assertions.assertTrue(createTableSql.contains("ENGINE = InnoDB")); + Assertions.assertTrue(createTableSql.contains("DEFAULT CHARSET = utf8mb4")); + Assertions.assertTrue(createTableSql.contains("COLLATE = utf8mb4_unicode_ci")); + } + @Test public void testColumnSinkType() { MysqlCreateTableSqlBuilder sqlBuilder = mock(MysqlCreateTableSqlBuilder.class); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialectTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialectTest.java index 122cec9e289b..1c9925eb9cee 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialectTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialectTest.java @@ -18,6 +18,11 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.mysql; import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.connectors.seatunnel.jdbc.exception.JdbcConnectorException; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.JdbcRowConverter; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect; +import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper; import org.apache.seatunnel.connectors.seatunnel.jdbc.source.JdbcSourceTable; import org.apache.seatunnel.connectors.seatunnel.jdbc.source.StringRangeSplitDecision; @@ -33,15 +38,79 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.zip.CRC32; @Slf4j public class MysqlDialectTest { + @Test + public void testValidateTableOptionsForMysql() { + MysqlDialect dialect = new MysqlDialect(); + Map tableOptions = new HashMap<>(); + tableOptions.put("engine", "InnoDB"); + tableOptions.put("charset", "utf8mb4"); + tableOptions.put("collate", "utf8mb4_unicode_ci"); + + Assertions.assertDoesNotThrow(() -> dialect.validateTableOptions(tableOptions)); + } + + @Test + public void testValidateTableOptionsForMysqlWithUnknownOption() { + MysqlDialect dialect = new MysqlDialect(); + Map tableOptions = new HashMap<>(); + tableOptions.put("bucket_num", "3"); + + JdbcConnectorException exception = + Assertions.assertThrows( + JdbcConnectorException.class, + () -> dialect.validateTableOptions(tableOptions)); + Assertions.assertTrue(exception.getMessage().contains("Unsupported JDBC table_options")); + } + + @Test + public void testValidateTableOptionsForUnsupportedDialect() { + JdbcDialect unsupportedDialect = + new JdbcDialect() { + @Override + public String dialectName() { + return DatabaseIdentifier.POSTGRESQL; + } + + @Override + public JdbcRowConverter getRowConverter() { + return null; + } + + @Override + public JdbcDialectTypeMapper getJdbcDialectTypeMapper() { + return null; + } + + @Override + public Optional getUpsertStatement( + String database, + String tableName, + String[] fieldNames, + String[] pkNames) { + return Optional.empty(); + } + }; + + JdbcConnectorException exception = + Assertions.assertThrows( + JdbcConnectorException.class, + () -> + unsupportedDialect.validateTableOptions( + Collections.singletonMap("engine", "InnoDB"))); + Assertions.assertTrue(exception.getMessage().contains("not supported")); + } + @Test public void testValidateStringRangeSplitAcceptsPrintableAsciiPunctuation() throws Exception { MysqlDialect dialect = new MysqlDialect(); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsConditionExtensionTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsConditionExtensionTest.java new file mode 100644 index 000000000000..dd8e99697c84 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcTableOptionsConditionExtensionTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc.sink; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.api.options.SinkConnectorCommonOptions; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * Verifies {@code table_options} early validation is wired through {@link + * JdbcSinkFactory#optionRule()} and {@link JdbcTableOptionsConditionExtension}. Dialect-specific + * allowlists are covered in {@code *DialectTest} classes. + */ +class JdbcTableOptionsConditionExtensionTest { + + @Test + void testMysqlTableOptionsPassViaOptionRule() { + Map config = mysqlSinkConfig(); + Map tableOptions = new HashMap<>(); + tableOptions.put("engine", "InnoDB"); + tableOptions.put("charset", "utf8mb4"); + tableOptions.put("collate", "utf8mb4_unicode_ci"); + config.put(SinkConnectorCommonOptions.TABLE_OPTIONS.key(), tableOptions); + + Assertions.assertDoesNotThrow(() -> validateSinkOptionRule(config)); + } + + @Test + void testPostgresRejectsNonEmptyTableOptionsViaOptionRule() { + Map config = postgresSinkConfig(); + Map tableOptions = new HashMap<>(); + tableOptions.put("fillfactor", "70"); + config.put(SinkConnectorCommonOptions.TABLE_OPTIONS.key(), tableOptions); + + OptionValidationException exception = + Assertions.assertThrows( + OptionValidationException.class, () -> validateSinkOptionRule(config)); + Assertions.assertTrue( + exception.getMessage().contains("not supported for dialect 'Postgres'")); + } + + @Test + void testAbsentTableOptionsSkipsExtension() { + Assertions.assertDoesNotThrow(() -> validateSinkOptionRule(mysqlSinkConfig())); + } + + @Test + void testEmptyTableOptionsSkipsExtension() { + Map config = mysqlSinkConfig(); + config.put(SinkConnectorCommonOptions.TABLE_OPTIONS.key(), new HashMap<>()); + + Assertions.assertDoesNotThrow(() -> validateSinkOptionRule(config)); + } + + private static void validateSinkOptionRule(Map config) { + ConfigValidator.of(ReadonlyConfig.fromMap(config)) + .validate(new JdbcSinkFactory().optionRule()); + } + + private static Map mysqlSinkConfig() { + Map config = new HashMap<>(); + config.put("url", "jdbc:mysql://127.0.0.1:3306/test"); + config.put("driver", "com.mysql.cj.jdbc.Driver"); + config.put("query", "INSERT INTO test_table VALUES (?)"); + return config; + } + + private static Map postgresSinkConfig() { + Map config = new HashMap<>(); + config.put("url", "jdbc:postgresql://127.0.0.1:5432/test"); + config.put("driver", "org.postgresql.Driver"); + config.put("query", "INSERT INTO test_table VALUES (?)"); + return config; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlTableOptionsIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlTableOptionsIT.java new file mode 100644 index 000000000000..0d3a755fecae --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlTableOptionsIT.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.mysql.MySqlCatalog; +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.PullPolicy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.stream.Stream; + +@Slf4j +public class JdbcMysqlTableOptionsIT extends TestSuiteBase implements TestResource { + + private static final String MYSQL_DRIVER_JAR = + "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; + + private static final String MYSQL_IMAGE = "mysql:8.0.43"; + private static final String MYSQL_CONTAINER_HOST = "mysql-e2e-table-options"; + private static final String MYSQL_DATABASE = "seatunnel"; + private static final String MYSQL_SOURCE = "source"; + private static final String MYSQL_SINK = "sink_table_options"; + + private static final String MYSQL_USERNAME = "root"; + private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; + + private static final String CONFIG_FILE = "/jdbc_mysql_sink_with_table_options.conf"; + + private static final String CREATE_SOURCE_TABLE_SQL = + "CREATE TABLE IF NOT EXISTS `" + + MYSQL_SOURCE + + "` (\n" + + " `id` BIGINT NOT NULL,\n" + + " `name` VARCHAR(255) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`)\n" + + ");"; + + private static final String INSERT_SOURCE_SQL = + "INSERT INTO `" + + MYSQL_SOURCE + + "` (`id`, `name`) VALUES (1, 'name_1'), (2, 'name_2'), (3, 'name_3');"; + + // MySQL 8.0.43 cold start may exceed the default 120s JDBC wait in Testcontainers. + private static final int MYSQL_STARTUP_TIMEOUT_SECONDS = + (int) Duration.ofMinutes(10).getSeconds(); + + private MySQLContainer mysqlContainer; + + @TestContainerExtension + protected final ContainerExtendedFactory extendedFactory = + container -> { + Container.ExecResult extraCommands = + container.execInContainer( + "bash", + "-c", + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " + + MYSQL_DRIVER_JAR); + Assertions.assertEquals(0, extraCommands.getExitCode(), extraCommands.getStderr()); + }; + + void initContainer() { + DockerImageName imageName = DockerImageName.parse(MYSQL_IMAGE); + mysqlContainer = + new MySQLContainer<>(imageName) + .withImagePullPolicy(PullPolicy.ageBased(Duration.ofDays(7))) + .withUsername(MYSQL_USERNAME) + .withPassword(MYSQL_PASSWORD) + .withDatabaseName(MYSQL_DATABASE) + .withNetwork(NETWORK) + .withNetworkAliases(MYSQL_CONTAINER_HOST) + .withStartupTimeoutSeconds(MYSQL_STARTUP_TIMEOUT_SECONDS) + .waitingFor(Wait.forHealthcheck()) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); + + Startables.deepStart(Stream.of(mysqlContainer)).join(); + } + + @Override + @BeforeAll + public void startUp() throws Exception { + initContainer(); + initializeJdbcTable(); + } + + @Override + @AfterAll + public void tearDown() { + if (mysqlContainer != null) { + mysqlContainer.close(); + } + } + + @TestTemplate + public void testTableOptionsSink(TestContainer container) + throws IOException, InterruptedException, SQLException { + try { + Container.ExecResult execResult = container.executeJob(CONFIG_FILE); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + assertSinkTableOptions(); + } finally { + clearSinkTable(); + } + } + + private void assertSinkTableOptions() throws SQLException { + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + ResultSet createTableResult = + statement.executeQuery( + String.format( + "SHOW CREATE TABLE `%s`.`%s`", MYSQL_DATABASE, MYSQL_SINK)); + Assertions.assertTrue(createTableResult.next()); + String createTableSql = createTableResult.getString(2).toLowerCase(); + Assertions.assertTrue( + createTableSql.contains( + MySqlCatalog.TABLE_OPTION_ENGINE.toLowerCase() + "=innodb"), + createTableSql); + Assertions.assertTrue( + createTableSql.contains( + MySqlCatalog.TABLE_OPTION_CHARSET.toLowerCase() + "=utf8mb4"), + createTableSql); + Assertions.assertTrue( + createTableSql.contains( + MySqlCatalog.TABLE_OPTION_COLLATE.toLowerCase() + + "=utf8mb4_unicode_ci"), + createTableSql); + + ResultSet countResult = + statement.executeQuery( + String.format( + "SELECT COUNT(*) FROM `%s`.`%s`", MYSQL_DATABASE, MYSQL_SINK)); + Assertions.assertTrue(countResult.next()); + Assertions.assertEquals(3, countResult.getInt(1)); + } + } + + private Connection getJdbcConnection() throws SQLException { + return DriverManager.getConnection( + mysqlContainer.getJdbcUrl(), + mysqlContainer.getUsername(), + mysqlContainer.getPassword()); + } + + private void initializeJdbcTable() { + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + statement.execute(CREATE_SOURCE_TABLE_SQL); + statement.execute(INSERT_SOURCE_SQL); + } catch (SQLException e) { + throw new RuntimeException("Initializing MySQL table failed!", e); + } + } + + private void clearSinkTable() { + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + String.format("DROP TABLE IF EXISTS `%s`.`%s`", MYSQL_DATABASE, MYSQL_SINK)); + } catch (SQLException e) { + throw new RuntimeException("Clearing sink table failed!", e); + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/resources/jdbc_mysql_sink_with_table_options.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/resources/jdbc_mysql_sink_with_table_options.conf new file mode 100644 index 000000000000..99b09f3f6edf --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/resources/jdbc_mysql_sink_with_table_options.conf @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + jdbc { + url = "jdbc:mysql://mysql-e2e-table-options:3306/seatunnel?useSSL=false" + driver = "com.mysql.cj.jdbc.Driver" + connection_check_timeout_sec = 100 + username = "root" + password = "Abc!@#135_seatunnel" + query = "select * from source;" + } +} + +sink { + jdbc { + url = "jdbc:mysql://mysql-e2e-table-options:3306/seatunnel?useSSL=false" + driver = "com.mysql.cj.jdbc.Driver" + username = "root" + password = "Abc!@#135_seatunnel" + + generate_sink_sql = true + database = "seatunnel" + table = "sink_table_options" + primary_keys = ["id"] + + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + data_save_mode = "APPEND_DATA" + table_options = { + "engine" = "InnoDB" + "charset" = "utf8mb4" + "collate" = "utf8mb4_unicode_ci" + } + } +} From 05978c6f5b1d4ae1238b892a5a71b205c511f096 Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Wed, 1 Jul 2026 13:42:16 +0800 Subject: [PATCH 089/375] [Docs][Connector-V2] Improve Redis source examples (#11240) Co-authored-by: DanielCarter-stack --- docs/en/connectors/source/Redis.md | 44 ++++++++++++++-- docs/zh/connectors/source/Redis.md | 84 ++++++++++++++++++++++++++++-- 2 files changed, 120 insertions(+), 8 deletions(-) diff --git a/docs/en/connectors/source/Redis.md b/docs/en/connectors/source/Redis.md index f5e8c4dbaa5a..5277fb7a61ec 100644 --- a/docs/en/connectors/source/Redis.md +++ b/docs/en/connectors/source/Redis.md @@ -39,7 +39,7 @@ When using `tables_configs` to read multiple key patterns, each table configurat | name | type | required | default value | description | |---------------------|--------|----------|---------------|-------------| | keys | string | yes | - | Redis key pattern to scan | -| data_type | string | yes | - | Redis data type: `key`, `hash`, `list`, `set`, `zset` | +| data_type | string | yes | - | Redis data type: `key`, `string`, `hash`, `list`, `set`, `zset` | | batch_size | int | no | 10 | Batch size for SCAN operations | | format | string | no | json | Data format: `json` or `text` | | schema | config | no | - | Schema configuration for this table | @@ -185,9 +185,9 @@ indicates the number of keys to attempt to return per iteration,default 10 ### data_type [string] -redis data types, support `key` `hash` `list` `set` `zset` +redis data types, support `key` `string` `hash` `list` `set` `zset` -- key +- key/string > The value of each key will be sent downstream as a single row of data. > For example, the value of key is `SeaTunnel test message`, the data received downstream is `SeaTunnel test message` and only one message will be received. @@ -391,6 +391,42 @@ sink { } ``` +read string type keys together with their Redis keys + +```hocon +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "string_test*" + data_type = string + batch_size = 33 + read_key_enabled = true + key_field_name = custom_key + single_field_name = custom_value + format = json + schema = { + table = "RedisDatabase.RedisTable" + columns = [ + { + name = "custom_key" + type = "string" + }, + { + name = "custom_value" + type = "string" + } + ] + } + } +} + +sink { + Console {} +} +``` + ### Multiple Table Mode **Example 1: Reading multiple key patterns with different data types** @@ -498,4 +534,4 @@ sink { ``` ## Changelog - \ No newline at end of file + diff --git a/docs/zh/connectors/source/Redis.md b/docs/zh/connectors/source/Redis.md index f7d32d27c951..39416dac71ad 100644 --- a/docs/zh/connectors/source/Redis.md +++ b/docs/zh/connectors/source/Redis.md @@ -39,7 +39,7 @@ import ChangeLog from '../changelog/connector-redis.md'; | 名称 | 类型 | 是否必须 | 默认值 | 描述 | |---------------------|---------|--------|-------|--------------------------------------------| | keys | string | 是 | - | 要扫描的 Redis key pattern | -| data_type | string | 是 | - | Redis 数据类型：`key`、`hash`、`list`、`set`、`zset` | +| data_type | string | 是 | - | Redis 数据类型：`key`、`string`、`hash`、`list`、`set`、`zset` | | batch_size | int | 否 | 10 | SCAN 操作的批量大小 | | format | string | 否 | json | 数据格式：`json` 或 `text` | | schema | config | 否 | - | Schema 配置 | @@ -131,6 +131,46 @@ hash key 中的每个 kv 将会被视为一行并被发送给上游。 keys 模式 +### read_key_enabled [boolean] + +配置为 `true` 时，Redis source 会把 Redis key 和 value 一起读出。 + +默认值为 `false`，也就是只读取 value。 + +如果读取的是 `string`、`list`、`set`、`zset` 这类单值类型，并且开启了 `read_key_enabled`，需要同时配置： + +- `key_field_name`：Redis key 写入哪一列。 +- `single_field_name`：Redis value 写入哪一列。 + +示例： + +```hocon +read_key_enabled = true +key_field_name = key +single_field_name = value +schema { + fields { + key = string + value = string + } +} +``` + +### key_field_name [string] + +指定 Redis key 输出到哪一个字段。 + +- 当 `read_key_enabled = true` 时，如果不配置，默认字段名是 `key`。 +- 当 `data_type = hash` 且不配置该选项时，默认字段名是 `hash_key`。 + +当默认字段名和已有字段冲突，或者想使用更明确的字段名时，可以配置该选项。 + +### single_field_name [string] + +读取单值类型并且 `read_key_enabled = true` 时，指定 Redis value 输出到哪一个字段。 + +该选项主要用于 `string`、`list`、`set`、`zset` 这类不能直接映射成多列对象的数据。 + ### batch_size [int] 表示每次迭代尝试返回的键的数量，默认值为 10。 @@ -139,9 +179,9 @@ keys 模式 ### data_type [string] -redis 数据类型, 支持 `key` `hash` `list` `set` `zset`。 +redis 数据类型, 支持 `key` `string` `hash` `list` `set` `zset`。 -- key +- key/string > 将每个 key 的值将作为单行数据发送给下游。 > 例如，key 对应的值为 `SeaTunnel test message`，则下游接收到的数据为 `SeaTunnel test message`，并且仅会收到一条信息。 @@ -326,6 +366,42 @@ sink { } ``` +读取 string 类型时同时读取 Redis key： + +```hocon +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "string_test*" + data_type = string + batch_size = 33 + read_key_enabled = true + key_field_name = custom_key + single_field_name = custom_value + format = json + schema = { + table = "RedisDatabase.RedisTable" + columns = [ + { + name = "custom_key" + type = "string" + }, + { + name = "custom_value" + type = "string" + } + ] + } + } +} + +sink { + Console {} +} +``` + ### 多表模式 **示例 1：读取具有不同数据类型的多个 key pattern** @@ -435,4 +511,4 @@ sink { ## 变更日志 - \ No newline at end of file + From 90d5b59bbf7798383665841941946620291c3a03 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 1 Jul 2026 15:58:03 +0800 Subject: [PATCH 090/375] [Test][E2E] Retry RocketMQ restore offset checks (#11234) --- .../e2e/connector/rocketmq/RocketMqIT.java | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rocketmq/RocketMqIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rocketmq/RocketMqIT.java index 9d04f31c8a5f..67616f44403a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rocketmq/RocketMqIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rocketmq/RocketMqIT.java @@ -88,6 +88,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import static org.apache.seatunnel.e2e.connector.rocketmq.RocketMqContainer.NAMESRV_PORT; @@ -653,9 +654,8 @@ public void testSourceRocketMqRestore(TestContainer container) .atMost(5, TimeUnit.MINUTES) .until(() -> getTopicMaxOffset(sinkTopic) >= expectedSinkAfterFirstRun + 15); - Thread.sleep(5000); - long finalSinkOffset = getTopicMaxOffset(sinkTopic); long expectedTotal = expectedSinkAfterFirstRun + 15; + long finalSinkOffset = awaitTopicMaxOffset(sinkTopic, expectedTotal, Duration.ofMinutes(1)); Assertions.assertEquals( expectedTotal, finalSinkOffset, @@ -722,11 +722,41 @@ private List pollMessagesFromOffset(String topicName, long fromOffset) { return result; } + /** + * Waits for RocketMQ admin offset visibility and returns the successful observed offset. + * + *

This keeps the final restore assertion from depending on a single broker metadata read. + */ + private long awaitTopicMaxOffset(String topicName, long expectedOffset, Duration timeout) { + AtomicLong observedOffset = new AtomicLong(); + Awaitility.await() + .pollInterval(2, TimeUnit.SECONDS) + .atMost(timeout) + .until( + () -> { + long current = getTopicMaxOffset(topicName); + observedOffset.set(current); + return current >= expectedOffset; + }); + return observedOffset.get(); + } + + /** + * Reads topic max offsets with retries because RocketMQ admin queries can temporarily fail + * during restore and broker channel transitions. + */ private long getTopicMaxOffset(String topicName) { try { List> offsetTopics = - RocketMqAdminUtil.offsetTopics( - newConfiguration(), Lists.newArrayList(topicName)); + RetryUtils.retryWithException( + () -> + RocketMqAdminUtil.offsetTopics( + newConfiguration(), Lists.newArrayList(topicName)), + new RetryUtils.RetryMaterial( + Constant.OPERATION_RETRY_TIME, + true, + exception -> true, + Constant.OPERATION_RETRY_SLEEP)); if (offsetTopics.isEmpty()) { return 0; } From a546d86e7027c44041e0b1885387a0bba4d31cff Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 1 Jul 2026 15:59:31 +0800 Subject: [PATCH 091/375] [Chore] Enable PR branch update suggestions (#11247) Co-authored-by: DanielLeens --- .asf.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index 4445166e3c5d..f2733c8c98e0 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -48,6 +48,8 @@ github: squash: true merge: false rebase: false + pull_requests: + allow_update_branch: true protected_branches: dev: required_status_checks: From 18b6cd876ca23d8298c0b0926a094a67eab0f8f1 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Wed, 1 Jul 2026 16:41:58 +0800 Subject: [PATCH 092/375] [Fix][API] Preserve extension exception context in conditional validation (#11228) --- .../util/ConditionEvaluators.java | 13 ++--- .../configuration/util/ConfigValidator.java | 13 ++++- .../util/ConfigValidatorTest.java | 54 +++++++++++++++++++ 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionEvaluators.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionEvaluators.java index eacb9566e64a..68db4568a07c 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionEvaluators.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConditionEvaluators.java @@ -46,15 +46,10 @@ static boolean evaluate(Condition condition, ReadonlyConfig config) { throw new OptionValidationException( "Condition for option '%s' has a null operator", condition.getOption().key()); } - try { - Object value = config.get(condition.getOption()); - Evaluator evaluator = REGISTRY.get(operator); - return evaluator.evaluate(value, condition, config); - } catch (OptionValidationException e) { - throw new OptionValidationException( - "Failed to evaluate constraint '%s' on option '%s': %s", - condition.toString(), condition.getOption().key(), e.getRawMessage()); - } + + Object value = config.get(condition.getOption()); + Evaluator evaluator = REGISTRY.get(operator); + return evaluator.evaluate(value, condition, config); } @SuppressWarnings({"rawtypes"}) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java index 42d1b1c7b953..ed5a2b356fef 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java @@ -213,8 +213,17 @@ private void collectErrors(OptionRule rule, Expression expression, List } for (ConditionRule conditionRule : rule.getConditionRules()) { - if (validate(conditionRule.getExpression())) { - collectErrors(conditionRule.getOptionRule(), conditionRule.getExpression(), errors); + try { + if (validate(conditionRule.getExpression())) { + collectErrors( + conditionRule.getOptionRule(), conditionRule.getExpression(), errors); + } + } catch (OptionValidationException e) { + errors.add( + formatError( + conditionRule.getExpression().toString(), + TYPE_CONDITIONAL, + e.getRawMessage())); } } diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java index a0bcd46ff1b9..4e1f4090109c 100644 --- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java +++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/util/ConfigValidatorTest.java @@ -3598,4 +3598,58 @@ public boolean evaluate(ReadonlyConfig config, Integer value) msg.contains("port value -1 is not positive"), "extension exception message should be preserved: " + msg); } + + @Test + public void testConditionalExpressionExceptionUsesWholeExpressionContext() { + ConditionExtension throwingExtension = + new ConditionExtension() { + @Override + public String description() { + return "must be positive"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Integer value) + throws OptionValidationException { + if (value != null && value <= 0) { + throw new OptionValidationException( + "port value %d is not positive", value); + } + return true; + } + }; + + Expression expression = + Expression.of( + Condition.of(MODE, "stream") + .and(Conditions.extension(PORT, throwingExtension))); + OptionRule rule = + new OptionRule( + Arrays.asList(MODE, PORT), + Collections.emptyList(), + Collections.singletonList( + new ConditionRule( + expression, + new OptionRule( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList()))), + Collections.emptyList()); + + Map config = new HashMap<>(); + config.put(MODE.key(), "stream"); + config.put(PORT.key(), -1); + + OptionValidationException ex = + Assertions.assertThrows( + OptionValidationException.class, + () -> ConfigValidator.of(ReadonlyConfig.fromMap(config)).validate(rule)); + String msg = ex.getMessage(); + Assertions.assertTrue(msg.contains("Option validation failed (1 error):"), msg); + Assertions.assertTrue( + msg.contains("[1] option: ('mode' == stream && 'port' must be positive)"), msg); + Assertions.assertTrue(msg.contains("type: conditional"), msg); + Assertions.assertTrue(msg.contains("constraint: port value -1 is not positive"), msg); + } } From f0f50be87d125c1483b3074bfdb9632e1e0f320c Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 1 Jul 2026 17:06:54 +0800 Subject: [PATCH 093/375] [Test][E2E] Avoid SQLServer CDC driver download in container (#11233) --- .../cdc/sqlserver/SqlServerCDCIT.java | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java index f45cc8f7e35d..2c0a780efbcf 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java @@ -45,7 +45,9 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; +import com.microsoft.sqlserver.jdbc.SQLServerDriver; import io.debezium.jdbc.JdbcConnection; import io.debezium.relational.TableId; import lombok.extern.slf4j.Slf4j; @@ -53,6 +55,7 @@ import java.io.IOException; import java.net.URL; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; @@ -195,20 +198,39 @@ public class SqlServerCDCIT extends TestSuiteBase implements TestResource { new Slf4jLogConsumer( DockerLoggerFactory.getLogger("sqlserver-docker-image"))); - private String driverUrl() { - return "https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/9.4.1.jre8/mssql-jdbc-9.4.1.jre8.jar"; + /** + * Resolve the SQLServer JDBC test dependency from the active test classpath instead of + * hard-coding a Maven local repository path. + */ + private Path driverJarPath() { + try { + return Paths.get( + SQLServerDriver.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI()); + } catch (Exception e) { + throw new SeaTunnelException( + "Failed to resolve SQLServer JDBC driver jar from the test classpath", e); + } } @TestContainerExtension protected final ContainerExtendedFactory extendedFactory = container -> { + Path driverJarPath = driverJarPath(); + Assertions.assertTrue( + Files.isRegularFile(driverJarPath), + "SQLServer JDBC driver should be resolved from the test classpath before E2E runs: " + + driverJarPath); Container.ExecResult extraCommands = container.execInContainer( - "bash", - "-c", - "mkdir -p /tmp/seatunnel/plugins/SqlServer-CDC/lib && cd /tmp/seatunnel/plugins/SqlServer-CDC/lib && wget " - + driverUrl()); + "bash", "-c", "mkdir -p /tmp/seatunnel/plugins/SqlServer-CDC/lib"); Assertions.assertEquals(0, extraCommands.getExitCode(), extraCommands.getStderr()); + container.copyFileToContainer( + MountableFile.forHostPath(driverJarPath), + "/tmp/seatunnel/plugins/SqlServer-CDC/lib/mssql-jdbc-9.4.1.jre8.jar"); }; @Override From ad00a4bbe9d71f7dc4a9d6ecd27bcb8387c1c88c Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 1 Jul 2026 17:08:21 +0800 Subject: [PATCH 094/375] [Fix][E2E] Fix DM schema change JDBC URLs (#11179) --- .../src/test/resources/mysqlcdc_to_dm_with_schema_change.conf | 2 +- .../mysqlcdc_to_dm_with_schema_change_exactly_once.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/mysqlcdc_to_dm_with_schema_change.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/mysqlcdc_to_dm_with_schema_change.conf index 662ca2c2f954..7532789bd26b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/mysqlcdc_to_dm_with_schema_change.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/mysqlcdc_to_dm_with_schema_change.conf @@ -41,7 +41,7 @@ source { sink { jdbc { - url = "jdbc:dm://e2e_dmdb:5236" + url = "jdbc:dm://e2e_dmdb:5236/SYSDBA" driver = "dm.jdbc.driver.DmDriver" connection_check_timeout_sec = 1000 username = "SYSDBA" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/mysqlcdc_to_dm_with_schema_change_exactly_once.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/mysqlcdc_to_dm_with_schema_change_exactly_once.conf index ec26f0457a3b..66e21512b573 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/mysqlcdc_to_dm_with_schema_change_exactly_once.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-ddl/src/test/resources/mysqlcdc_to_dm_with_schema_change_exactly_once.conf @@ -41,7 +41,7 @@ source { sink { jdbc { - url = "jdbc:dm://e2e_dmdb:5236" + url = "jdbc:dm://e2e_dmdb:5236/SYSDBA" driver = "dm.jdbc.driver.DmDriver" connection_check_timeout_sec = 1000 username = "SYSDBA" From e3f58eb0be8cff56d7e434bd55f0facf14866fd9 Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Wed, 1 Jul 2026 17:47:25 +0800 Subject: [PATCH 095/375] [Docs] Improve Email connector examples (#11245) Co-authored-by: DanielCarter-stack --- docs/en/connectors/sink/Email.md | 134 ++++++++++++++++++++++++++----- docs/zh/connectors/sink/Email.md | 131 +++++++++++++++++++++++++----- 2 files changed, 227 insertions(+), 38 deletions(-) diff --git a/docs/en/connectors/sink/Email.md b/docs/en/connectors/sink/Email.md index b422c1df471f..a1418e468dde 100644 --- a/docs/en/connectors/sink/Email.md +++ b/docs/en/connectors/sink/Email.md @@ -6,7 +6,7 @@ import ChangeLog from '../changelog/connector-email.md'; ## Description -Send the data as a file to email. +Send the received rows as an attachment file to one or more email addresses. The tested email version is 1.5.6. @@ -24,7 +24,7 @@ The tested email version is 1.5.6. | email_transport_protocol | string | yes | - | | email_smtp_auth | boolean | yes | - | | email_smtp_port | int | no | 465 | -| email_authorization_code | string | no | - | +| email_authorization_code | string | yes | - | | email_message_headline | string | yes | - | | email_message_content | string | yes | - | | email_attachment_name | string | no | emailsink.csv | @@ -39,6 +39,8 @@ Sender Email Address. Address to receive mail, Support multiple email addresses, separated by commas (,). +Example: `receiver-1@example.com,receiver-2@example.com`. + ### email_host [string] SMTP server to connect to. @@ -57,7 +59,10 @@ Select port for authentication. ### email_authorization_code [string] -authorization code,You can obtain the authorization code from the mailbox Settings. +Authorization code or password. You can obtain the authorization code from the mailbox settings. + +This option is required by the connector configuration. When `email_smtp_auth = false`, it can be +set to an empty string. ### email_message_headline [string] @@ -69,36 +74,127 @@ The body of the entire message. ### email_attachment_name [string] -The name of the email attachment file. Default is `emailsink.csv`. +The name of the email attachment file. Default is `emailsink.csv`. The connector writes the rows to +this local file before sending the email. ### email_field_delimiter [string] The delimiter used to separate fields in the attachment file. Default is comma `,`. +The attachment has no header row. Field values are written in the upstream schema order. `null` +values are written as empty strings. + ### common options Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. ## Example -```bash - - EmailSink { - email_from_address = "xxxxxx@qq.com" - email_to_address = "xxxxxx@163.com" - email_host="smtp.qq.com" - email_transport_protocol="smtp" - email_smtp_auth="true" - email_authorization_code="" - email_message_headline="" - email_message_content="" - email_attachment_name="report.csv" # Optional, default is emailsink.csv - email_field_delimiter="|" # Optional, default is , - } +### Send one table to multiple recipients + +This example follows the Email e2e job. It uses a test SMTP server without authentication and sends +one email to each address in `email_to_address`. + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + tables_configs = [ + { + row.num = 100 + schema = { + table = "test.table1" + columns = [ + { + name = "id" + type = "bigint" + }, + { + name = "name" + type = "string" + }, + { + name = "age" + type = "int" + } + ] + } + } + ] + } +} + +sink { + EmailSink { + email_from_address = "sender@example.com" + email_to_address = "receiver-1@example.com,receiver-2@example.com" + email_host = "email-e2e" + email_transport_protocol = "smtp" + email_smtp_auth = false + email_smtp_port = 3025 + email_authorization_code = "" + email_message_headline = "test-title" + email_message_content = "test-content" + email_attachment_name = "report.csv" + email_field_delimiter = "|" + } +} +``` +### Send multiple tables + +Email sink supports multi-table input. In the e2e job, two upstream tables create two emails for +each recipient. + +```hocon +source { + FakeSource { + tables_configs = [ + { + row.num = 100 + schema { + table = "test.table1" + fields { + id = bigint + name = string + age = int + } + } + }, + { + row.num = 100 + schema { + table = "test.table2" + fields { + id = bigint + name = string + age = int + } + } + } + ] + } +} + +sink { + EmailSink { + email_from_address = "sender@example.com" + email_to_address = "receiver-3@example.com,receiver-4@example.com" + email_host = "email-e2e" + email_transport_protocol = "smtp" + email_smtp_auth = false + email_smtp_port = 3025 + email_authorization_code = "" + email_message_headline = "test-title" + email_message_content = "test-content" + } +} ``` ## Changelog - diff --git a/docs/zh/connectors/sink/Email.md b/docs/zh/connectors/sink/Email.md index 3f2ca57e2b09..ec90dffaa2f9 100644 --- a/docs/zh/connectors/sink/Email.md +++ b/docs/zh/connectors/sink/Email.md @@ -6,7 +6,7 @@ import ChangeLog from '../changelog/connector-email.md'; ## 描述 -将接收的数据作为文件发送到电子邮件 +将接收到的数据写成附件文件，并发送到一个或多个邮箱地址。 ## 支持版本 @@ -27,7 +27,7 @@ import ChangeLog from '../changelog/connector-email.md'; | email_transport_protocol | string | 是 | - | | email_smtp_auth | boolean | 是 | - | | email_smtp_port | int | 否 | 465 | -| email_authorization_code | string | 否 | - | +| email_authorization_code | string | 是 | - | | email_message_headline | string | 是 | - | | email_message_content | string | 是 | - | | email_attachment_name | string | 否 | emailsink.csv | @@ -42,6 +42,8 @@ import ChangeLog from '../changelog/connector-email.md'; 接收邮件的地址，支持多个邮箱地址，以逗号（,）分隔。 +示例：`receiver-1@example.com,receiver-2@example.com`。 + ### email_host [string] 连接的SMTP服务器地址 @@ -60,7 +62,9 @@ import ChangeLog from '../changelog/connector-email.md'; ### email_authorization_code [string] -授权码,您可以从邮箱设置中获取授权码 +授权码或密码，可以从邮箱设置中获取。 + +连接器要求必须配置该项。当 `email_smtp_auth = false` 时，可以配置为空字符串。 ### email_message_headline [string] @@ -72,35 +76,124 @@ import ChangeLog from '../changelog/connector-email.md'; ### email_attachment_name [string] -邮件附件的文件名。默认为 `emailsink.csv`。 +邮件附件的文件名。默认为 `emailsink.csv`。连接器会先把数据写到本地这个文件里，再作为附件发送。 ### email_field_delimiter [string] 附件文件中用于分隔字段的分隔符。默认为逗号 `,`。 +附件不包含表头。字段会按上游 schema 的顺序写入，`null` 值会写成空字符串。 + ### common options Sink插件常用参数，请参考 [Sink常用选项](../common-options/sink-common-options.md) 了解详情. ## 示例 -```bash - - EmailSink { - email_from_address = "xxxxxx@qq.com" - email_to_address = "xxxxxx@163.com" - email_host="smtp.qq.com" - email_transport_protocol="smtp" - email_smtp_auth="true" - email_authorization_code="" - email_message_headline="" - email_message_content="" - email_attachment_name="report.csv" # 可选，默认为 emailsink.csv - email_field_delimiter="|" # 可选，默认为 , - } +### 发送单表数据到多个收件人 + +这个示例来自 Email e2e 任务。示例使用不需要认证的测试 SMTP 服务，并给 `email_to_address` +中的每个邮箱各发送一封邮件。 + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + tables_configs = [ + { + row.num = 100 + schema = { + table = "test.table1" + columns = [ + { + name = "id" + type = "bigint" + }, + { + name = "name" + type = "string" + }, + { + name = "age" + type = "int" + } + ] + } + } + ] + } +} + +sink { + EmailSink { + email_from_address = "sender@example.com" + email_to_address = "receiver-1@example.com,receiver-2@example.com" + email_host = "email-e2e" + email_transport_protocol = "smtp" + email_smtp_auth = false + email_smtp_port = 3025 + email_authorization_code = "" + email_message_headline = "test-title" + email_message_content = "test-content" + email_attachment_name = "report.csv" + email_field_delimiter = "|" + } +} +``` +### 发送多表数据 + +Email sink 支持多表输入。在 e2e 任务中，两个上游表会让每个收件人收到两封邮件。 + +```hocon +source { + FakeSource { + tables_configs = [ + { + row.num = 100 + schema { + table = "test.table1" + fields { + id = bigint + name = string + age = int + } + } + }, + { + row.num = 100 + schema { + table = "test.table2" + fields { + id = bigint + name = string + age = int + } + } + } + ] + } +} + +sink { + EmailSink { + email_from_address = "sender@example.com" + email_to_address = "receiver-3@example.com,receiver-4@example.com" + email_host = "email-e2e" + email_transport_protocol = "smtp" + email_smtp_auth = false + email_smtp_port = 3025 + email_authorization_code = "" + email_message_headline = "test-title" + email_message_content = "test-content" + } +} ``` ## 变更日志 - \ No newline at end of file + From 67f873a61a211ea869b5badbe2a3fbbac2c14061 Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Wed, 1 Jul 2026 19:54:13 +0800 Subject: [PATCH 096/375] [Docs][Connector-V2] Improve Qdrant connector docs (#11252) Co-authored-by: DanielCarter-stack --- docs/en/connectors/sink/Qdrant.md | 54 +++++++++++++++++++++++++---- docs/en/connectors/source/Qdrant.md | 51 +++++++++++++++++++++++++-- docs/zh/connectors/sink/Qdrant.md | 52 +++++++++++++++++++++++---- docs/zh/connectors/source/Qdrant.md | 51 +++++++++++++++++++++++++-- 4 files changed, 189 insertions(+), 19 deletions(-) diff --git a/docs/en/connectors/sink/Qdrant.md b/docs/en/connectors/sink/Qdrant.md index 824a4165ac80..6f4cf19bfc60 100644 --- a/docs/en/connectors/sink/Qdrant.md +++ b/docs/en/connectors/sink/Qdrant.md @@ -10,6 +10,8 @@ import ChangeLog from '../changelog/connector-qdrant.md'; This connector can be used to write data into a Qdrant collection. +The target collection must already exist before the job starts. Vector field names and dimensions in Qdrant must match the vector columns in the SeaTunnel row. + ## Data Type Mapping | SeaTunnel Data Type | Qdrant Data Type | @@ -36,20 +38,15 @@ The value of the primary key column will be used as point ID in Qdrant. If no pr | name | type | required | default value | |-----------------|--------|----------|---------------| | collection_name | string | yes | - | -| batch_size | int | no | 64 | | host | string | no | localhost | | port | int | no | 6334 | | api_key | string | no | - | -| use_tls | int | no | false | +| use_tls | bool | no | false | | common-options | | no | - | ### collection_name [string] -The name of the Qdrant collection to read data from. - -### batch_size [int] - -The batch size of each upsert request to Qdrant. +The name of the Qdrant collection to write data to. ### host [string] @@ -71,6 +68,49 @@ Whether to use TLS(SSL) connection. Required if using Qdrant cloud(https). Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. +## Task Example + +The following example writes records from a Qdrant source collection to another Qdrant collection. Payload fields such as `file_name` and `file_size` are written as point payloads, and `my_vector` is written as a named vector. + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Qdrant { + collection_name = "source_collection" + host = "localhost" + port = 6334 + schema = { + columns = [ + { + name = file_name + type = string + } + { + name = file_size + type = int + } + { + name = my_vector + type = float_vector + } + ] + } + } +} + +sink { + Qdrant { + collection_name = "sink_collection" + host = "localhost" + port = 6334 + } +} +``` + ## Changelog diff --git a/docs/en/connectors/source/Qdrant.md b/docs/en/connectors/source/Qdrant.md index f670efb59557..2e5a864f6c56 100644 --- a/docs/en/connectors/source/Qdrant.md +++ b/docs/en/connectors/source/Qdrant.md @@ -19,7 +19,7 @@ This connector can be used to read data from a Qdrant collection. | host | string | no | localhost | | port | int | no | 6334 | | api_key | string | no | - | -| use_tls | int | no | false | +| use_tls | bool | no | false | | common-options | | no | - | ### collection_name [string] @@ -44,7 +44,7 @@ schema = { Each entry in Qdrant is called a point. -The `float_vector` type columns are read from the vectors of each point, others are read from the JSON payload associated with the point. +Vector columns are read from the vectors of each point. Other columns are read from the JSON payload associated with the point. If a column is marked as primary key, the ID of the Qdrant point is written into it. It can be of type `"string"` or `"int"`. Since Qdrant only [allows](https://qdrant.tech/documentation/concepts/points/#point-ids) positive integers and UUIDs as point IDs. @@ -82,6 +82,51 @@ Whether to use TLS(SSL) connection. Required if using Qdrant cloud(https). Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details. +## Task Example + +The Qdrant collection must already exist before the job starts. Vector field names and dimensions in the collection must match the schema used by SeaTunnel. + +The following example reads payload fields and a named vector from `source_collection`, then writes the rows to `sink_collection`. + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Qdrant { + collection_name = "source_collection" + host = "localhost" + port = 6334 + schema = { + columns = [ + { + name = file_name + type = string + } + { + name = file_size + type = int + } + { + name = my_vector + type = float_vector + } + ] + } + } +} + +sink { + Qdrant { + collection_name = "sink_collection" + host = "localhost" + port = 6334 + } +} +``` + ## Changelog - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/Qdrant.md b/docs/zh/connectors/sink/Qdrant.md index eff0c8e397f4..45c360607018 100644 --- a/docs/zh/connectors/sink/Qdrant.md +++ b/docs/zh/connectors/sink/Qdrant.md @@ -8,6 +8,8 @@ import ChangeLog from '../changelog/connector-qdrant.md'; 该连接器可用于将数据写入 Qdrant 集合。 +目标 collection 必须在作业启动前已经存在。Qdrant 中的向量字段名和维度需要与 SeaTunnel 数据行中的向量列保持一致。 + ## 数据类型映射 | SeaTunnel 数据类型 | Qdrant 数据类型 | @@ -34,7 +36,6 @@ import ChangeLog from '../changelog/connector-qdrant.md'; | 名称 | 类型 | 必填 | 默认值 | |-----------------|--------|----|-----------| | collection_name | string | 是 | - | -| batch_size | int | 否 | 64 | | host | string | 否 | localhost | | port | int | 否 | 6334 | | api_key | string | 否 | - | @@ -43,11 +44,7 @@ import ChangeLog from '../changelog/connector-qdrant.md'; ### collection_name [string] -要从中读取数据的 Qdrant 集合的名称。 - -### batch_size [int] - -每个 upsert 请求到 Qdrant 的批量大小。 +要写入数据的 Qdrant 集合的名称。 ### host [string] @@ -69,6 +66,49 @@ Qdrant 实例的 gRPC 端口。 Sink插件通用参数，请参考[Sink通用选项](../common-options/sink-common-options.md)了解详情。 +## 任务示例 + +下面的示例会把一个 Qdrant source collection 中的记录写入另一个 Qdrant collection。`file_name`、`file_size` 等普通字段会写成 point payload，`my_vector` 会写成命名向量。 + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Qdrant { + collection_name = "source_collection" + host = "localhost" + port = 6334 + schema = { + columns = [ + { + name = file_name + type = string + } + { + name = file_size + type = int + } + { + name = my_vector + type = float_vector + } + ] + } + } +} + +sink { + Qdrant { + collection_name = "sink_collection" + host = "localhost" + port = 6334 + } +} +``` + ## 变更日志 diff --git a/docs/zh/connectors/source/Qdrant.md b/docs/zh/connectors/source/Qdrant.md index 60db36a1cdda..6ee35d524e02 100644 --- a/docs/zh/connectors/source/Qdrant.md +++ b/docs/zh/connectors/source/Qdrant.md @@ -42,7 +42,7 @@ schema = { Qdrant 中的每个条目称为一个点。 -`float_vector` 类型的列从每个点的向量中读取，其他列从与该点关联的 JSON 有效负载中读取。 +向量类型的列会从每个点的向量中读取，其他列会从与该点关联的 JSON payload 中读取。 如果列被标记为主键，Qdrant 点的 ID 将写入其中。它可以是 `"string"` 或 `"int"` 类型。因为 Qdrant 仅[允许](https://qdrant.tech/documentation/concepts/points/#point-ids)使用正整数和 UUID 作为点 ID。 @@ -78,8 +78,53 @@ Qdrant 实例的 gRPC 端口。 ### 通用选项 -源插件的通用参数，请参考[源通用选项](../common-options/source-common-options.md)了解详情。**** +源插件的通用参数，请参考[源通用选项](../common-options/source-common-options.md)了解详情。 + +## 任务示例 + +作业启动前，Qdrant collection 必须已经存在。Qdrant 中的向量字段名和维度需要与 SeaTunnel schema 中的向量列保持一致。 + +下面的示例从 `source_collection` 读取 payload 字段和命名向量，然后写入 `sink_collection`。 + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + Qdrant { + collection_name = "source_collection" + host = "localhost" + port = 6334 + schema = { + columns = [ + { + name = file_name + type = string + } + { + name = file_size + type = int + } + { + name = my_vector + type = float_vector + } + ] + } + } +} + +sink { + Qdrant { + collection_name = "sink_collection" + host = "localhost" + port = 6334 + } +} +``` ## 变更日志 - \ No newline at end of file + From 0011bdf9451ba28f207ce63edfc937d327bf1e31 Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Wed, 1 Jul 2026 19:58:33 +0800 Subject: [PATCH 097/375] [Docs][Connector-V2] Improve Aerospike sink example (#11241) Co-authored-by: DanielCarter-stack <254644355+DanielCarter-stack@users.noreply.github.com> --- docs/en/connectors/sink/Aerospike.md | 64 ++++++++++++++++++---------- docs/zh/connectors/sink/Aerospike.md | 62 +++++++++++++++++---------- 2 files changed, 81 insertions(+), 45 deletions(-) diff --git a/docs/en/connectors/sink/Aerospike.md b/docs/en/connectors/sink/Aerospike.md index db39777597fa..4d8eb6324242 100644 --- a/docs/en/connectors/sink/Aerospike.md +++ b/docs/en/connectors/sink/Aerospike.md @@ -61,35 +61,49 @@ Note: | username | string | No | - | Username for authentication | | password | string | No | - | Password for authentication | | key | string | Yes | - | Field name to use as Aerospike primary key | -| bin_name | string | No | - | Bin name for storing data | +| bin_name | string | No | - | Bin name for storing data. Required when `data_format` is `map` or `string` | | data_format | string | No | string | Data storage format: map/string/kv | | write_timeout | int | No | 200 | Write operation timeout in milliseconds | | schema.field | map | No | {} | Field type mappings (e.g. {"name":"STRING","age":"INTEGER"}) | ### data_format Options -- **map**: Store data as JSON map -- **string**: Store data as JSON string -- **kv**: Store each field as separate bin + +- **map**: Store all non-key fields as a map in `bin_name` +- **string**: Store all non-key fields as a JSON string in `bin_name` +- **kv**: Store each non-key field as a separate bin. `bin_name` is not used ## Task Example -### Simple Example +### Write FakeSource Data To Aerospike ```hocon env { - parallelism = 2 + parallelism = 1 job.mode = "BATCH" } source { FakeSource { - row.num = 10 + row.num = 9 + string.fake.mode = "template" + string.template = ["tyrantlucifer", "hailin", "kris", "fanjia", "zongwen", "gaojun"] + int.fake.mode = "template" + int.template = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + double.fake.mode = "template" + double.template = [44.0, 45.0, 46.0, 47.0] + timestamp.fake.mode = "template" + timestamp.template = [ + "2022-01-01 00:00:00", + "2022-01-01 00:00:01", + "2022-01-01 00:00:02", + "2022-01-01 00:00:03" + ] schema = { fields { - id = "int" - name = "string" - age = "int" - address = "string" + c_id = "int" + c_name = "string" + c_money = "double" + c_birth = "timestamp" } } } @@ -97,22 +111,26 @@ source { sink { Aerospike { - host = "localhost" + host = "aerospike-host" port = 3000 - namespace = "test_namespace" - set = "user_data" - key = "id" - data_format = "map" - write_timeout = 300 - schema.field = { - id = "INTEGER" - name = "STRING" - age = "INTEGER" - address = "STRING" + namespace = "test" + set = "seatunnel" + key = "c_id" + bin_name = "data" + data_format = "string" + username = "" + password = "" + schema { + field { + c_id = "INTEGER" + c_name = "STRING" + c_money = "DOUBLE" + c_birth = "LONG" + } } } } ``` ## Changelog - \ No newline at end of file + diff --git a/docs/zh/connectors/sink/Aerospike.md b/docs/zh/connectors/sink/Aerospike.md index 9515ef6e537d..cf5c54541377 100644 --- a/docs/zh/connectors/sink/Aerospike.md +++ b/docs/zh/connectors/sink/Aerospike.md @@ -61,35 +61,49 @@ import ChangeLog from '../changelog/connector-aerospike.md'; | username | string | 否 | - | 认证用户名 | | password | string | 否 | - | 认证密码 | | key | string | 是 | - | 用作 Aerospike 主键的字段名称 | -| bin_name | string | 否 | - | 数据存储的 bin 名称 | +| bin_name | string | 否 | - | 数据存储的 bin 名称。`data_format` 为 `map` 或 `string` 时需要配置 | | data_format | string | 否 | string | 数据存储格式：map/string/kv | | write_timeout | int | 否 | 200 | 写入操作超时时间（毫秒） | | schema.field | map | 否 | {} | 字段类型映射（示例：{"name":"STRING","age":"INTEGER"}） | ### data_format 选项说明 -- **map**: 以JSON对象格式存储 -- **string**: 以JSON字符串格式存储 -- **kv**: 每个字段存储为独立的bin + +- **map**: 将所有非主键字段作为一个 map 存到 `bin_name` +- **string**: 将所有非主键字段作为 JSON 字符串存到 `bin_name` +- **kv**: 每个非主键字段存储为独立的 bin，此时不使用 `bin_name` ## 任务示例 -### 简单示例 +### 将 FakeSource 数据写入 Aerospike ```hocon env { - parallelism = 2 + parallelism = 1 job.mode = "BATCH" } source { FakeSource { - row.num = 10 + row.num = 9 + string.fake.mode = "template" + string.template = ["tyrantlucifer", "hailin", "kris", "fanjia", "zongwen", "gaojun"] + int.fake.mode = "template" + int.template = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + double.fake.mode = "template" + double.template = [44.0, 45.0, 46.0, 47.0] + timestamp.fake.mode = "template" + timestamp.template = [ + "2022-01-01 00:00:00", + "2022-01-01 00:00:01", + "2022-01-01 00:00:02", + "2022-01-01 00:00:03" + ] schema = { fields { - id = "int" - name = "string" - age = "int" - address = "string" + c_id = "int" + c_name = "string" + c_money = "double" + c_birth = "timestamp" } } } @@ -97,18 +111,22 @@ source { sink { Aerospike { - host = "localhost" + host = "aerospike-host" port = 3000 - namespace = "test_namespace" - set = "user_data" - key = "id" - data_format = "map" - write_timeout = 300 - schema.field = { - id = "INTEGER" - name = "STRING" - age = "INTEGER" - address = "STRING" + namespace = "test" + set = "seatunnel" + key = "c_id" + bin_name = "data" + data_format = "string" + username = "" + password = "" + schema { + field { + c_id = "INTEGER" + c_name = "STRING" + c_money = "DOUBLE" + c_birth = "LONG" + } } } } From 8f29bdaea9d81fa9303258b5348dd0d732db4ba2 Mon Sep 17 00:00:00 2001 From: Jast Date: Wed, 1 Jul 2026 20:34:05 +0800 Subject: [PATCH 098/375] [Fix][Connector-V2] Support PostgreSQL enum in JDBC source (#11232) --- .../dialect/psql/PostgresTypeConverter.java | 7 +++ .../dialect/psql/PostgresTypeMapper.java | 1 + .../psql/PostgresTypeConverterTest.java | 43 +++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeConverter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeConverter.java index 588b2df183b6..465a587e3fb9 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeConverter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeConverter.java @@ -34,6 +34,8 @@ import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; +import java.sql.Types; + // reference http://www.postgres.cn/docs/13/datatype.html @Slf4j @AutoService(TypeConverter.class) @@ -283,6 +285,11 @@ public Column convert(BasicTypeDefine typeDefine) { } break; default: + if (typeDefine.getSqlType() == Types.OTHER) { + builder.dataType(BasicType.STRING_TYPE); + builder.sourceType(typeDefine.getColumnType()); + break; + } throw CommonError.convertToSeaTunnelTypeError( identifier(), typeDefine.getDataType(), typeDefine.getName()); } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeMapper.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeMapper.java index 58bed581edcc..3f5cae4fdb32 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeMapper.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeMapper.java @@ -44,6 +44,7 @@ public Column mappingColumn(ResultSetMetaData metadata, int colIndex) throws SQL .name(columnName) .columnType(nativeType) .dataType(nativeType) + .sqlType(metadata.getColumnType(colIndex)) .nullable(isNullable == ResultSetMetaData.columnNullable) .length((long) precision) .precision((long) precision) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeConverterTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeConverterTest.java index f8058a7c0011..842b2abf8e69 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeConverterTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresTypeConverterTest.java @@ -31,6 +31,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class PostgresTypeConverterTest { @Test public void testConvertUnsupported() { @@ -266,6 +273,42 @@ public void testConvertOtherString() { Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); } + @Test + public void testConvertCustomEnumAsString() { + BasicTypeDefine typeDefine = + BasicTypeDefine.builder() + .name("status") + .columnType("JobStatus") + .dataType("JobStatus") + .sqlType(Types.OTHER) + .nullable(true) + .build(); + + Column column = PostgresTypeConverter.INSTANCE.convert(typeDefine); + + Assertions.assertEquals(typeDefine.getName(), column.getName()); + Assertions.assertEquals(BasicType.STRING_TYPE, column.getDataType()); + Assertions.assertEquals(typeDefine.getColumnType(), column.getSourceType()); + Assertions.assertTrue(column.isNullable()); + } + + @Test + public void testTypeMapperPassesJdbcTypeForCustomEnum() throws SQLException { + ResultSetMetaData metadata = mock(ResultSetMetaData.class); + when(metadata.getColumnLabel(1)).thenReturn("status"); + when(metadata.getColumnTypeName(1)).thenReturn("JobStatus"); + when(metadata.getColumnType(1)).thenReturn(Types.OTHER); + when(metadata.isNullable(1)).thenReturn(ResultSetMetaData.columnNullable); + when(metadata.getPrecision(1)).thenReturn(0); + when(metadata.getScale(1)).thenReturn(0); + + Column column = new PostgresTypeMapper().mappingColumn(metadata, 1); + + Assertions.assertEquals("status", column.getName()); + Assertions.assertEquals(BasicType.STRING_TYPE, column.getDataType()); + Assertions.assertEquals("JobStatus", column.getSourceType()); + } + @Test public void testConvertBinary() { BasicTypeDefine typeDefine = From f47b0bea5c8649448f34edb47e7b3d2bedabb815 Mon Sep 17 00:00:00 2001 From: Jast Date: Wed, 1 Jul 2026 20:44:46 +0800 Subject: [PATCH 099/375] [Fix][API] Support dotted opaque table names (#11220) --- .../api/table/catalog/TablePathTest.java | 31 +++++++++++++++++++ .../kafka/source/KafkaSourceConfig.java | 9 +++++- .../kafka/source/KafkaSourceConfigTest.java | 24 ++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/TablePathTest.java diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/TablePathTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/TablePathTest.java new file mode 100644 index 000000000000..e6244f185c43 --- /dev/null +++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/TablePathTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.api.table.catalog; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TablePathTest { + + @Test + public void testRejectMalformedStructuredTableNameWithTooManyParts() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> TablePath.of("reg.country.user_activity.activity")); + } +} diff --git a/seatunnel-connectors-v2/connector-kafka/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/source/KafkaSourceConfig.java b/seatunnel-connectors-v2/connector-kafka/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/source/KafkaSourceConfig.java index 017325f057d0..249030c736bd 100644 --- a/seatunnel-connectors-v2/connector-kafka/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/source/KafkaSourceConfig.java +++ b/seatunnel-connectors-v2/connector-kafka/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/source/KafkaSourceConfig.java @@ -312,10 +312,17 @@ private TablePath getTablePathFromSchema(ReadonlyConfig readonlyConfig, String t .orElse(ReadonlyConfig.fromMap(Collections.emptyMap())); return schema.getOptional(TableIdentifierOptions.TABLE) - .map(TablePath::of) + .map(KafkaSourceConfig::parseKafkaTablePath) .orElseGet(() -> TablePath.of(null, topicName)); } + private static TablePath parseKafkaTablePath(String tableName) { + if (StringUtils.countMatches(tableName, ".") > 2) { + return TablePath.of(null, tableName); + } + return TablePath.of(tableName); + } + private DeserializationSchema createDeserializationSchema( CatalogTable catalogTable, ReadonlyConfig readonlyConfig) { SeaTunnelRowType seaTunnelRowType = catalogTable.getSeaTunnelRowType(); diff --git a/seatunnel-connectors-v2/connector-kafka/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/source/KafkaSourceConfigTest.java b/seatunnel-connectors-v2/connector-kafka/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/source/KafkaSourceConfigTest.java index e550c9959064..2a3b63a94459 100644 --- a/seatunnel-connectors-v2/connector-kafka/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/source/KafkaSourceConfigTest.java +++ b/seatunnel-connectors-v2/connector-kafka/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/source/KafkaSourceConfigTest.java @@ -107,4 +107,28 @@ void testDeserializationWithSchema() { Assertions.assertNotNull(deserializationSchema); } + + @Test + void testSchemaTableWithMultipleDotsCanBeUsedAsOpaqueTablePath() { + String table = "reg.country.user_activity.activity"; + + Map schemaFields = new HashMap<>(); + schemaFields.put("id", "int"); + schemaFields.put("name", "string"); + + Map schema = new HashMap<>(); + schema.put("fields", schemaFields); + schema.put(TABLE.key(), table); + + Map configMap = new HashMap<>(); + configMap.put("bootstrap.servers", "localhost:9092"); + configMap.put("group.id", "test"); + configMap.put("topic", "test"); + configMap.put("schema", schema); + configMap.put("format", "json"); + + KafkaSourceConfig sourceConfig = new KafkaSourceConfig(ReadonlyConfig.fromMap(configMap)); + + Assertions.assertNotNull(sourceConfig.getMapMetadata().get(TablePath.of(null, table))); + } } From 3b6d43bcc06df6290bd5caacf4233e155aef2ccd Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 1 Jul 2026 20:46:30 +0800 Subject: [PATCH 100/375] [Test][E2E] Wait for failover task graph before cancel (#11236) --- ...tClusterPendingJobLifecycleFailoverIT.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SplitClusterPendingJobLifecycleFailoverIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SplitClusterPendingJobLifecycleFailoverIT.java index 17033d47b469..e44f94dd13d9 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SplitClusterPendingJobLifecycleFailoverIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SplitClusterPendingJobLifecycleFailoverIT.java @@ -28,8 +28,14 @@ import org.apache.seatunnel.engine.common.config.server.ScheduleStrategy; import org.apache.seatunnel.engine.common.exception.SeaTunnelEngineException; import org.apache.seatunnel.engine.common.job.JobStatus; +import org.apache.seatunnel.engine.core.job.PipelineStatus; import org.apache.seatunnel.engine.server.SeaTunnelServer; import org.apache.seatunnel.engine.server.SeaTunnelServerStarter; +import org.apache.seatunnel.engine.server.dag.physical.PhysicalPlan; +import org.apache.seatunnel.engine.server.dag.physical.PhysicalVertex; +import org.apache.seatunnel.engine.server.dag.physical.SubPlan; +import org.apache.seatunnel.engine.server.execution.ExecutionState; +import org.apache.seatunnel.engine.server.master.JobMaster; import org.awaitility.Awaitility; import org.jetbrains.annotations.NotNull; @@ -148,6 +154,7 @@ public void testPendingJobLifecycleInMasterFailover() { 3, standbyMaster.getCluster().getMembers().size())); assertJobStatusWithTimeout(pendingJobAfterFailover, JobStatus.RUNNING, 180); assertPendingQueueNotContainsJob(standbyMaster, pendingJobId); + assertRunningJobGraphWithTimeout(standbyMaster, pendingJobId, 120); pendingJobAfterFailover.cancelJob(); assertJobStatusWithTimeout(pendingJobAfterFailover, JobStatus.CANCELED, 120); @@ -289,6 +296,66 @@ private static void assertJobStatusWithTimeout( expectedStatus, clientJobProxy.getJobStatus())); } + /** + * Waits until failover restore has rebuilt the running graph on the active master. + * + *

The top-level job status can turn RUNNING before every restored pipeline and task vertex + * has reacquired slots and reported RUNNING. Cancelling earlier makes this test depend on a + * restore/cancel race instead of pending-job scheduling. + */ + private static void assertRunningJobGraphWithTimeout( + HazelcastInstanceImpl activeMaster, long jobId, long timeoutSeconds) { + Awaitility.await() + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .untilAsserted( + () -> { + JobMaster jobMaster = getJobMaster(activeMaster, jobId); + Assertions.assertNotNull( + jobMaster, + "Job master should exist before checking restored task states"); + PhysicalPlan physicalPlan = jobMaster.getPhysicalPlan(); + Assertions.assertEquals(JobStatus.RUNNING, physicalPlan.getJobStatus()); + physicalPlan + .getPipelineList() + .forEach( + SplitClusterPendingJobLifecycleFailoverIT + ::assertRunningSubPlan); + }); + } + + /** + * Asserts that one restored pipeline and all of its task vertices are fully running. + * + *

This keeps the following cancel assertion focused on the lifecycle transition instead of + * racing against delayed task deployment after master failover. + */ + private static void assertRunningSubPlan(SubPlan subPlan) { + Assertions.assertEquals(PipelineStatus.RUNNING, subPlan.getPipelineState()); + subPlan.getCoordinatorVertexList() + .forEach(SplitClusterPendingJobLifecycleFailoverIT::assertRunningVertex); + subPlan.getPhysicalVertexList() + .forEach(SplitClusterPendingJobLifecycleFailoverIT::assertRunningVertex); + } + + /** + * Asserts that one task vertex has completed deployment and reported RUNNING. + * + *

A restored vertex can lag behind the top-level job status while resources are assigned, so + * the test must observe the vertex state directly before cancelling. + */ + private static void assertRunningVertex(PhysicalVertex physicalVertex) { + Assertions.assertEquals(ExecutionState.RUNNING, physicalVertex.getExecutionState()); + } + + /** + * Reads the current job master from the active SeaTunnel server embedded in the test cluster. + */ + private static JobMaster getJobMaster(HazelcastInstanceImpl activeMaster, long jobId) { + SeaTunnelServer server = + activeMaster.node.getNodeEngine().getService(SeaTunnelServer.SERVICE_NAME); + return server.getCoordinatorService().getJobMaster(jobId); + } + private static HazelcastInstanceImpl waitAndFindActiveMaster( HazelcastInstanceImpl masterNode1, HazelcastInstanceImpl masterNode2) { final HazelcastInstanceImpl[] activeMasterRef = new HazelcastInstanceImpl[1]; From 04b25acbc027faa43a40cdf9dd6746fae4747c3d Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Wed, 1 Jul 2026 22:26:19 +0800 Subject: [PATCH 101/375] [Docs][Connector-V2] Improve Prometheus connector documentation (#11253) Co-authored-by: DanielCarter-stack --- docs/en/connectors/sink/Prometheus.md | 107 +++++++++----- docs/en/connectors/source/Prometheus.md | 181 ++++++++++++----------- docs/zh/connectors/sink/Prometheus.md | 106 +++++++++----- docs/zh/connectors/source/Prometheus.md | 183 +++++++++++++----------- 4 files changed, 339 insertions(+), 238 deletions(-) diff --git a/docs/en/connectors/sink/Prometheus.md b/docs/en/connectors/sink/Prometheus.md index ecfc0bc3c24d..7a0a0d132de4 100644 --- a/docs/en/connectors/sink/Prometheus.md +++ b/docs/en/connectors/sink/Prometheus.md @@ -15,46 +15,69 @@ import ChangeLog from '../changelog/connector-prometheus.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) - [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) -- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) +- [x] [timer flush](../../introduction/concepts/connector-v2-features.md) ## Description -Used to launch web hooks using data. +Writes metric samples to a Prometheus-compatible remote write endpoint. -> For example, if the data from upstream is [`label: {"__name__": "test1"}, value: 1.2.3,time:2024-08-15T17:00:00`], the body content is the following: `{"label":{"__name__": "test1"}, "value":"1.23","time":"2024-08-15T17:00:00"}` +The sink converts each SeaTunnel row into one Prometheus sample, serializes the data as a remote write request, compresses it with Snappy, and sends it by HTTP POST. Use a remote write endpoint such as `http://prometheus:9090/api/v1/write` or `http://victoria-metrics:8428/api/v1/write`. -**Tips: Prometheus sink only support `post json` webhook and the data from source will be treated as body content in web hook.And does not support passing past data** +Prometheus may reject samples that are too old for the target server's retention and remote write rules. ## Supported DataSource Info -In order to use the Http connector, the following dependencies are required. -They can be downloaded via install-plugin.sh or from the Maven central repository. +In order to use the Prometheus connector, the following dependency is required. +It can be downloaded via `install-plugin.sh` or from the Maven central repository. -| Datasource | Supported Versions | Dependency | -|------------|--------------------|------------------------------------------------------------------------------------------------------------------| -| Http | universal | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-connectors-v2/connector-prometheus) | +| Datasource | Supported Versions | Dependency | +|------------|--------------------|------------| +| Prometheus | universal | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-connectors-v2/connector-prometheus) | ## Sink Options -| Name | Type | Required | Default | Description | -|-----------------------------|--------|----------|---------|-------------------------------------------------------------------------------------------------------------| -| url | String | Yes | - | Http request url | -| headers | Map | No | - | Http headers | -| retry | Int | No | - | The max retry times if request http return to `IOException` | -| retry_backoff_multiplier_ms | Int | No | 100 | The retry-backoff times(millis) multiplier if request http failed | -| retry_backoff_max_ms | Int | No | 10000 | The maximum retry-backoff times(millis) if request http failed | -| connect_timeout_ms | Int | No | 12000 | Connection timeout setting, default 12s. | -| socket_timeout_ms | Int | No | 60000 | Socket timeout setting, default 60s. | -| key_timestamp | Int | NO | - | prometheus timestamp key . | -| key_label | String | yes | - | prometheus label key | -| key_value | Double | yes | - | prometheus value | -| batch_size | Int | false | 1024 | prometheus batch size write | -| flush_interval | Long | false | 300000L | prometheus flush commit interval | -| common-options | | No | - | Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details | +| name | type | required | default value | description | +|-----------------------------|--------|----------|---------------|-------------| +| url | String | Yes | - | Prometheus-compatible remote write URL. | +| headers | Map | No | - | HTTP request headers. | +| retry | Int | No | - | Maximum retry times when the HTTP request fails with `IOException`. | +| retry_backoff_multiplier_ms | Int | No | 100 | Retry backoff multiplier, in milliseconds. | +| retry_backoff_max_ms | Int | No | 10000 | Maximum retry backoff, in milliseconds. | +| key_label | String | Yes | - | Field name whose value is used as Prometheus labels. The field value must be a map. | +| key_value | String | Yes | - | Field name whose value is used as the Prometheus sample value. | +| key_timestamp | String | No | - | Field name whose value is used as the sample timestamp. If omitted, the current system time is used. | +| batch_size | Int | No | 1024 | Maximum number of samples written in one request. Must be greater than 0. | +| flush_interval | Long | No | 300000 | Scheduled flush interval in milliseconds. | +| common-options | config | No | - | Sink common options. | + +### key_label [String] + +The named field must be `map`. It is converted into Prometheus labels. Include `__name__` in the map to set the metric name. + +### key_value [String] + +The named field is converted into the Prometheus sample value. A `double` field is recommended. + +### key_timestamp [String] + +Optional timestamp field. + +Supported field types: + +- `timestamp`: converted to epoch milliseconds with the local time zone +- `bigint`: treated as epoch milliseconds +- `double`: treated as Unix seconds and converted to milliseconds +- `string`: parsed as epoch milliseconds + +When this option is not configured, the sink uses the current system time. + +### common options + +Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. ## Example -simple: +### Write to Prometheus remote write ```hocon env { @@ -73,21 +96,21 @@ source { } plugin_output = "fake" rows = [ - { - kind = INSERT - fields = [{"__name__": "test1"}, 1.23, "2024-08-15T17:00:00"] - }, - { - kind = INSERT - fields = [{"__name__": "test2"}, 1.23, "2024-08-15T17:00:00"] - } + { + kind = INSERT + fields = [{"__name__" : "metric_1"}, 1.23, CURRENT_TIMESTAMP] + }, + { + kind = INSERT + fields = [{"__name__" : "metric_2"}, 1.23, CURRENT_TIMESTAMP] + } ] } } - sink { Prometheus { + plugin_input = "fake" url = "http://prometheus:9090/api/v1/write" key_label = "c_map" key_value = "c_double" @@ -95,9 +118,23 @@ sink { batch_size = 1 } } +``` + +### Write to a Prometheus-compatible remote write API +```hocon +sink { + Prometheus { + plugin_input = "fake" + url = "http://victoria-metrics:8428/api/v1/write" + key_label = "c_map" + key_value = "c_double" + key_timestamp = "c_timestamp" + batch_size = 5 + } +} ``` ## Changelog - \ No newline at end of file + diff --git a/docs/en/connectors/source/Prometheus.md b/docs/en/connectors/source/Prometheus.md index 7b6afaea7622..e7c59f55d405 100644 --- a/docs/en/connectors/source/Prometheus.md +++ b/docs/en/connectors/source/Prometheus.md @@ -6,7 +6,9 @@ import ChangeLog from '../changelog/connector-prometheus.md'; ## Description -Used to read data from Prometheus. +Reads metric samples from Prometheus-compatible HTTP APIs. + +The connector uses the Prometheus query API. Configure `url` as the base address, such as `http://prometheus:9090` or `http://victoria-metrics:8428`. SeaTunnel appends `/api/v1/query` for `Instant` queries and `/api/v1/query_range` for `Range` queries. ## Key features @@ -16,133 +18,144 @@ Used to read data from Prometheus. ## Options -| name | type | required | default value | -|-----------------------------|---------|----------|-----------------| -| url | String | Yes | - | -| query | String | Yes | - | -| query_type | String | Yes | Instant | -| content_field | String | Yes | $.data.result.* | -| schema.fields | Config | Yes | - | -| format | String | No | json | -| params | Map | Yes | - | -| poll_interval_millis | int | No | - | -| retry | int | No | - | -| retry_backoff_multiplier_ms | int | No | 100 | -| retry_backoff_max_ms | int | No | 10000 | -| enable_multi_lines | boolean | No | false | -| common-options | config | No | - | - -### url [String] - -http request url - -### query [String] - -Prometheus expression query string +| name | type | required | default value | description | +|-----------------------------|---------|----------|---------------|-------------| +| url | String | Yes | - | Prometheus-compatible server base URL. | +| query | String | Yes | - | PromQL expression. | +| query_type | String | No | Instant | Query type. Valid values are `Instant` and `Range`. | +| start | String | Required when `query_type = Range` | - | Range query start time. | +| end | String | Required when `query_type = Range` | - | Range query end time. | +| step | String | Required when `query_type = Range` | - | Range query resolution step, for example `15s`. | +| time | Long | No | - | Instant query evaluation time, as a Unix timestamp. | +| timeout | Long | No | - | Query timeout passed to Prometheus. | +| headers | Map | No | - | HTTP request headers. | +| params | Map | No | - | Extra HTTP request parameters. SeaTunnel adds `query` and query time parameters automatically. | +| content_field | String | No | - | JSONPath used to extract the sample list. For Prometheus responses, use `$.data.result.*`. | +| schema.fields | Config | Required when `format = json` | - | Output schema. | +| format | String | No | text | Response format. Use `json` for Prometheus metric samples. | +| poll_interval_millis | int | No | - | Request interval in stream mode, in milliseconds. | +| retry | int | No | - | Maximum retry times when the HTTP request fails with `IOException`. | +| retry_backoff_multiplier_ms | int | No | 100 | Retry backoff multiplier, in milliseconds. | +| retry_backoff_max_ms | int | No | 10000 | Maximum retry backoff, in milliseconds. | +| common-options | config | No | - | Source common options. | ### query_type [String] -Instant/Range - -1. Instant : The following endpoint evaluates an instant query at a single point in time -2. Range : The following endpoint evaluates an expression query over a range of time - -https://prometheus.io/docs/prometheus/latest/querying/api/ - -### params [Map] - -http request params - -### poll_interval_millis [int] +`Instant` evaluates the query at a single time. `Range` evaluates the query over a time range. -request http api interval(millis) in stream mode +### start / end [String] -### retry [int] +Used only when `query_type = Range`. -The max retry times if request http return to `IOException` +Supported values: -### retry_backoff_multiplier_ms [int] +- `CURRENT_TIMESTAMP` +- ISO-8601 timestamp, for example `2025-05-13T02:25:23Z` +- Unix timestamp in seconds, for example `1747103123.083` -The retry-backoff times(millis) multiplier if request http failed +### step [String] -### retry_backoff_max_ms [int] - -The maximum retry-backoff times(millis) if request http failed - -### format [String] - -the format of upstream data, default `json`. +Used only when `query_type = Range`. It is the query resolution step accepted by Prometheus, such as `15s`, `1m`, or a number of seconds. ### schema [Config] -Fill in a fixed value +Prometheus source returns three fields in this order: ```hocon - schema = { - fields { - metric = "map" - value = double - time = long - } - } - +schema = { + fields { + metric = "map" + value = double + time = long + } +} ``` -#### fields [Config] - -the schema fields of upstream data +The `metric` field contains labels, including `__name__`. The `value` field is the metric value. The `time` field is the sample timestamp in milliseconds. ### common options -Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details +Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details. ## Example -### Instant +### Instant query ```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + source { Prometheus { - plugin_output = "http" - url = "http://mockserver:1080" - query = "up" + plugin_output = "prometheus_metrics" + url = "http://prometheus:9090" + query = "metric_1" query_type = "Instant" content_field = "$.data.result.*" format = "json" schema = { - fields { - metric = "map" - value = double - time = long - } - } + fields { + metric = "map" + value = double + time = long + } } + } } ``` -### Range +### Range query ```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + source { Prometheus { - plugin_output = "http" - url = "http://mockserver:1080" - query = "up" + plugin_output = "prometheus_metrics" + url = "http://prometheus:9090" + query = "metric_1" query_type = "Range" + start = "CURRENT_TIMESTAMP" + end = "CURRENT_TIMESTAMP" + step = "15s" content_field = "$.data.result.*" format = "json" - start = "2024-07-22T20:10:30.781Z" - end = "2024-07-22T20:11:00.781Z" - step = "15s" schema = { - fields { - metric = "map" - value = double - time = long - } - } + fields { + metric = "map" + value = double + time = long + } } } +} +``` + +### Read from a Prometheus-compatible API + +```hocon +source { + Prometheus { + plugin_output = "metrics" + url = "http://victoria-metrics:8428" + query = "metric_1" + query_type = "Instant" + content_field = "$.data.result.*" + format = "json" + schema = { + fields { + metric = "map" + value = double + time = long + } + } + } +} ``` ## Changelog diff --git a/docs/zh/connectors/sink/Prometheus.md b/docs/zh/connectors/sink/Prometheus.md index 0f90018970e7..fcecca5be1ef 100644 --- a/docs/zh/connectors/sink/Prometheus.md +++ b/docs/zh/connectors/sink/Prometheus.md @@ -14,46 +14,69 @@ import ChangeLog from '../changelog/connector-prometheus.md'; - [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md) - [ ] [cdc](../../introduction/concepts/connector-v2-features.md) -- [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md) -- [ ] [timer flush](../../introduction/concepts/connector-v2-features.md) +- [x] [支持多表写入](../../introduction/concepts/connector-v2-features.md) +- [x] [定时刷新](../../introduction/concepts/connector-v2-features.md) ## 描述 -接收Source端传入的数据，利用数据触发 web hooks。 +向 Prometheus 兼容的 remote write 接口写入指标样本。 -> 例如，来自上游的数据为 [`label: {"__name__": "test1"}, value: 1.2.3,time:2024-08-15T17:00:00`], 则body内容如下: `{"label":{"__name__": "test1"}, "value":"1.23","time":"2024-08-15T17:00:00"}` +这个 sink 会把每一行 SeaTunnel 数据转换成一个 Prometheus 样本，再按 remote write 协议序列化，用 Snappy 压缩后通过 HTTP POST 发出。常见地址类似 `http://prometheus:9090/api/v1/write` 或 `http://victoria-metrics:8428/api/v1/write`。 -**Tips: Prometheus 数据接收器 仅支持 `post json` 类型的 web hook，source 数据将被视为 webhook 中的 body 内容。并且不支持传递过去太久的数据** +如果样本时间太早，目标服务可能会按自己的保留策略或 remote write 规则拒绝写入。 ## 支持的数据源信息 -想使用 Prometheus 连接器，需要安装以下必要的依赖。可以通过运行 install-plugin.sh 脚本或者从 Maven 中央仓库下载这些依赖 +想使用 Prometheus 连接器，需要安装以下依赖。可以通过 `install-plugin.sh` 脚本安装，也可以从 Maven 中央仓库下载。 -| 数据源 | 支持版本 | 依赖 | -|------|-----------|------------------------------------------------------------------------------------------------------------------| -| Http | universal | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-connectors-v2/connector-prometheus) | +| 数据源 | 支持版本 | 依赖 | +|--------|----------|------| +| Prometheus | universal | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-connectors-v2/connector-prometheus) | ## 接收器选项 -| Name | Type | Required | Default | Description | -|-----------------------------|--------|----------|---------|-------------------------------------------------------------------| -| url | String | Yes | - | Http 请求链接 | -| headers | Map | No | - | Http 标头 | -| retry | Int | No | - | 如果请求http返回`IOException`的最大重试次数 | -| retry_backoff_multiplier_ms | Int | No | 100 | http请求失败，重试回退次数（毫秒）乘数 | -| retry_backoff_max_ms | Int | No | 10000 | http请求失败，最大重试回退时间(毫秒) | -| connect_timeout_ms | Int | No | 12000 | 连接超时设置，默认12s | -| socket_timeout_ms | Int | No | 60000 | 套接字超时设置，默认为60s | -| key_timestamp | Int | NO | - | prometheus时间戳的key. | -| key_label | String | yes | - | prometheus标签的key | -| key_value | Double | yes | - | prometheus值的key | -| batch_size | Int | false | 1024 | prometheus批量写入大小 | -| flush_interval | Long | false | 300000L | prometheus定时写入 | -| common-options | | No | - | Sink插件常用参数，请参考 [Sink常用选项 ](../common-options/sink-common-options.md) 了解详情 | +| 名称 | 类型 | 是否必填 | 默认值 | 描述 | +|-----------------------------|--------|----------|--------|------| +| url | String | 是 | - | Prometheus 兼容 remote write 地址。 | +| headers | Map | 否 | - | HTTP 请求头。 | +| retry | Int | 否 | - | HTTP 请求出现 `IOException` 时的最大重试次数。 | +| retry_backoff_multiplier_ms | Int | 否 | 100 | 重试退避时间乘数，单位毫秒。 | +| retry_backoff_max_ms | Int | 否 | 10000 | 最大重试退避时间，单位毫秒。 | +| key_label | String | 是 | - | 作为 Prometheus 标签的字段名。字段值必须是 map。 | +| key_value | String | 是 | - | 作为 Prometheus 样本值的字段名。 | +| key_timestamp | String | 否 | - | 作为样本时间戳的字段名。不配置时使用当前系统时间。 | +| batch_size | Int | 否 | 1024 | 单次请求最多写入的样本数，必须大于 0。 | +| flush_interval | Long | 否 | 300000 | 定时刷新间隔，单位毫秒。 | +| common-options | config | 否 | - | Sink 通用选项。 | + +### key_label [String] + +对应字段必须是 `map`，会被转换为 Prometheus 标签。建议在 map 中包含 `__name__`，用来表示指标名。 + +### key_value [String] + +对应字段会被转换为 Prometheus 样本值。推荐使用 `double` 类型字段。 + +### key_timestamp [String] + +可选的时间戳字段。 + +支持以下字段类型： + +- `timestamp`：按本地时区转换为毫秒级时间戳 +- `bigint`：按毫秒级时间戳处理 +- `double`：按 Unix 秒级时间戳处理，并转换为毫秒 +- `string`：按毫秒级时间戳解析 + +不配置这个选项时，sink 会使用当前系统时间。 + +### common options + +Sink 插件通用参数，请参考 [Sink Common Options](../common-options/sink-common-options.md)。 ## 示例 -简单示例: +### 写入 Prometheus remote write ```hocon env { @@ -72,21 +95,21 @@ source { } plugin_output = "fake" rows = [ - { - kind = INSERT - fields = [{"__name__": "test1"}, 1.23, "2024-08-15T17:00:00"] - }, - { - kind = INSERT - fields = [{"__name__": "test2"}, 1.23, "2024-08-15T17:00:00"] - } + { + kind = INSERT + fields = [{"__name__" : "metric_1"}, 1.23, CURRENT_TIMESTAMP] + }, + { + kind = INSERT + fields = [{"__name__" : "metric_2"}, 1.23, CURRENT_TIMESTAMP] + } ] } } - sink { Prometheus { + plugin_input = "fake" url = "http://prometheus:9090/api/v1/write" key_label = "c_map" key_value = "c_double" @@ -96,6 +119,21 @@ sink { } ``` +### 写入 Prometheus 兼容 remote write 接口 + +```hocon +sink { + Prometheus { + plugin_input = "fake" + url = "http://victoria-metrics:8428/api/v1/write" + key_label = "c_map" + key_value = "c_double" + key_timestamp = "c_timestamp" + batch_size = 5 + } +} +``` + ## 变更日志 diff --git a/docs/zh/connectors/source/Prometheus.md b/docs/zh/connectors/source/Prometheus.md index 98ed165e0fbe..14c11552cec1 100644 --- a/docs/zh/connectors/source/Prometheus.md +++ b/docs/zh/connectors/source/Prometheus.md @@ -6,7 +6,9 @@ import ChangeLog from '../changelog/connector-prometheus.md'; ## 描述 -用于读取prometheus数据。 +从 Prometheus 兼容的 HTTP API 读取指标样本。 + +连接器使用 Prometheus 查询 API。`url` 只需要填写基础地址，例如 `http://prometheus:9090` 或 `http://victoria-metrics:8428`。SeaTunnel 会按查询类型自动追加 `/api/v1/query` 或 `/api/v1/query_range`。 ## 主要特性 @@ -16,135 +18,146 @@ import ChangeLog from '../changelog/connector-prometheus.md'; ## 源选项 -| 名称 | 类型 | 是否必填 | 默认值 | -|-----------------------------|---------|------|-----------------| -| url | String | Yes | - | -| query | String | Yes | - | -| query_type | String | Yes | Instant | -| content_field | String | Yes | $.data.result.* | -| schema.fields | Config | Yes | - | -| format | String | No | json | -| params | Map | Yes | - | -| poll_interval_millis | int | No | - | -| retry | int | No | - | -| retry_backoff_multiplier_ms | int | No | 100 | -| retry_backoff_max_ms | int | No | 10000 | -| enable_multi_lines | boolean | No | false | -| common-options | config | No | | - -### url [String] - -http 请求路径。 - -### query [String] - -Prometheus 表达式查询字符串 +| 名称 | 类型 | 是否必填 | 默认值 | 描述 | +|-----------------------------|---------|----------|--------|------| +| url | String | 是 | - | Prometheus 兼容服务的基础地址。 | +| query | String | 是 | - | PromQL 查询表达式。 | +| query_type | String | 否 | Instant | 查询类型，可选值为 `Instant` 和 `Range`。 | +| start | String | `query_type = Range` 时必填 | - | 范围查询开始时间。 | +| end | String | `query_type = Range` 时必填 | - | 范围查询结束时间。 | +| step | String | `query_type = Range` 时必填 | - | 范围查询步长，例如 `15s`。 | +| time | Long | 否 | - | 即时查询的评估时间，使用 Unix 时间戳。 | +| timeout | Long | 否 | - | 传给 Prometheus 的查询超时时间。 | +| headers | Map | 否 | - | HTTP 请求头。 | +| params | Map | 否 | - | 额外的 HTTP 请求参数。SeaTunnel 会自动加入 `query` 和查询时间参数。 | +| content_field | String | 否 | - | 用来提取样本列表的 JSONPath。Prometheus 响应通常填写 `$.data.result.*`。 | +| schema.fields | Config | `format = json` 时必填 | - | 输出字段结构。 | +| format | String | 否 | text | 响应格式。读取 Prometheus 指标样本时请设置为 `json`。 | +| poll_interval_millis | int | 否 | - | 流模式下请求间隔，单位毫秒。 | +| retry | int | 否 | - | HTTP 请求出现 `IOException` 时的最大重试次数。 | +| retry_backoff_multiplier_ms | int | 否 | 100 | 重试退避时间乘数，单位毫秒。 | +| retry_backoff_max_ms | int | 否 | 10000 | 最大重试退避时间，单位毫秒。 | +| common-options | config | 否 | - | Source 通用选项。 | ### query_type [String] -Instant/Range - -1. Instant : 简单指标的即时查询。 -2. Range : 一段时间内指标数据。 - -https://prometheus.io/docs/prometheus/latest/querying/api/ - -### params [Map] - -http 请求参数 - -### poll_interval_millis [int] +`Instant` 表示查询某一个时间点的指标值。`Range` 表示查询一段时间范围内的指标值。 -流模式下请求HTTP API间隔(毫秒) +### start / end [String] -### retry [int] +仅在 `query_type = Range` 时使用。 -The max retry times if request http return to `IOException` +支持以下写法： -### retry_backoff_multiplier_ms [int] +- `CURRENT_TIMESTAMP` +- ISO-8601 时间，例如 `2025-05-13T02:25:23Z` +- Unix 秒级时间戳，例如 `1747103123.083` -请求http返回到' IOException '的最大重试次数 +### step [String] -### retry_backoff_max_ms [int] - -http请求失败，最大重试回退时间(毫秒) - -### format [String] - -上游数据的格式，默认为json。 +仅在 `query_type = Range` 时使用。它表示 Prometheus 接受的查询步长，例如 `15s`、`1m`，也可以是秒数。 ### schema [Config] -按照如下填写一个固定值 +Prometheus source 固定输出下面三个字段，顺序也固定： ```hocon - schema = { - fields { - metric = "map" - value = double - time = long - } - } - +schema = { + fields { + metric = "map" + value = double + time = long + } +} ``` -#### fields [Config] - -上游数据的模式字段 +`metric` 字段保存指标标签，包括 `__name__`。`value` 字段是指标值。`time` 字段是毫秒级样本时间戳。 ### common options -源插件常用参数，请参考[Source Common Options](../common-options/source-common-options.md) 了解详细信息 +源插件通用参数，请参考 [Source Common Options](../common-options/source-common-options.md)。 ## 示例 -### Instant +### 即时查询 ```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + source { Prometheus { - plugin_output = "http" - url = "http://mockserver:1080" - query = "up" + plugin_output = "prometheus_metrics" + url = "http://prometheus:9090" + query = "metric_1" query_type = "Instant" content_field = "$.data.result.*" format = "json" schema = { - fields { - metric = "map" - value = double - time = long - } - } + fields { + metric = "map" + value = double + time = long + } } + } } ``` -### Range +### 范围查询 ```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + source { Prometheus { - plugin_output = "http" - url = "http://mockserver:1080" - query = "up" + plugin_output = "prometheus_metrics" + url = "http://prometheus:9090" + query = "metric_1" query_type = "Range" + start = "CURRENT_TIMESTAMP" + end = "CURRENT_TIMESTAMP" + step = "15s" content_field = "$.data.result.*" format = "json" - start = "2024-07-22T20:10:30.781Z" - end = "2024-07-22T20:11:00.781Z" - step = "15s" schema = { - fields { - metric = "map" - value = double - time = long - } - } + fields { + metric = "map" + value = double + time = long + } } } +} +``` + +### 读取 Prometheus 兼容接口 + +```hocon +source { + Prometheus { + plugin_output = "metrics" + url = "http://victoria-metrics:8428" + query = "metric_1" + query_type = "Instant" + content_field = "$.data.result.*" + format = "json" + schema = { + fields { + metric = "map" + value = double + time = long + } + } + } +} ``` ## 变更日志 - \ No newline at end of file + From c62a5e6cdef6abf8145bff6de5fdea91b3ed59bb Mon Sep 17 00:00:00 2001 From: Daniel Carter <806666800@qq.com> Date: Wed, 1 Jul 2026 22:30:11 +0800 Subject: [PATCH 102/375] [Docs][Connector-V2] Improve Druid connector docs (#11255) Co-authored-by: DanielCarter-stack --- docs/en/connectors/sink/Druid.md | 133 +++++++++++++++++++++++++--- docs/zh/connectors/sink/Druid.md | 144 +++++++++++++++++++++++++++---- 2 files changed, 246 insertions(+), 31 deletions(-) diff --git a/docs/en/connectors/sink/Druid.md b/docs/en/connectors/sink/Druid.md index 5385712c4242..ce85fd430937 100644 --- a/docs/en/connectors/sink/Druid.md +++ b/docs/en/connectors/sink/Druid.md @@ -6,7 +6,7 @@ import ChangeLog from '../changelog/connector-druid.md'; ## Description -Write data to Druid +Write data to Apache Druid through the Druid indexing task API. ## Key features @@ -30,7 +30,7 @@ Write data to Druid ## Options -| name | type | required | default value | +| name | type | required | default value | |----------------|--------|----------|---------------| | coordinatorUrl | string | yes | - | | datasource | string | yes | - | @@ -39,40 +39,148 @@ Write data to Druid ### coordinatorUrl [string] -The coordinatorUrl host and port of Druid, example: "myHost:8888" +The Druid coordinator or router host and port, for example `router:8888`. + +SeaTunnel sends indexing tasks to `http://{coordinatorUrl}/druid/indexer/v1/task`, so configure only the host and port. Do not include the protocol or API path. ### datasource [string] -The datasource name you want to write, example: "seatunnel" +The Druid datasource name to write. + +When the upstream source has multiple tables, you can use placeholders such as `${table_name}` to route each upstream table to a different Druid datasource. ### batchSize [int] -The number of rows flushed to Druid per batch. Default value is `1024`. +The number of rows buffered before SeaTunnel sends one indexing task to Druid. The default value is `10000`. + +SeaTunnel also flushes the remaining buffered rows when the writer closes. ### common options -Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details +Sink plugin common parameters, please refer to [Sink Common Options](../common-options/sink-common-options.md) for details. + +For multi-table writes, `multi_table_sink_replica` can be used with the common sink options. + +## Write Behavior + +The connector converts each SeaTunnel row into inline CSV data and submits it to Druid as a native batch indexing task. + +The connector adds a processing-time column named `timestamp` to satisfy Druid's primary timestamp requirement. This generated timestamp is used by Druid ingestion; source `TIMESTAMP` fields are written as string dimensions according to the mapping above. ## Example -Simple example: +### Write One Table ```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + schema = { + fields { + c_boolean = boolean + c_timestamp = timestamp + c_string = string + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(16, 1)" + } + } + rows = [ + { + kind = INSERT + fields = [true, "2020-02-02T02:02:02", "NEW", 1, 2, 3, 4, 4.3, 5.3, 6.3] + }, + { + kind = INSERT + fields = [false, "2012-12-21T12:34:56", "AAA", 1, 1, 333, 323232, 3.1, 9.33333, 99999.99999999] + } + ] + } +} + sink { Druid { - coordinatorUrl = "testHost:8888" - datasource = "seatunnel" + coordinatorUrl = "router:8888" + datasource = "testDataSource" } } ``` -Use placeholders get upstream table metadata example: +### Write Multiple Tables + +Use `${table_name}` to write each upstream table to a Druid datasource with the same table name. ```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + tables_configs = [ + { + schema = { + table = "druid_sink_1" + fields { + id = int + val_bool = boolean + val_tinyint = tinyint + val_smallint = smallint + val_int = int + val_bigint = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3, 5.3, 6.3, "NEW"] + } + ] + }, + { + schema = { + table = "druid_sink_2" + fields { + id = int + val_bool = boolean + val_tinyint = tinyint + val_smallint = smallint + val_int = int + val_bigint = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3, 5.3, 6.3] + } + ] + } + ] + } +} + sink { Druid { - coordinatorUrl = "testHost:8888" - datasource = "${table_name}_test" + coordinatorUrl = "router:8888" + datasource = "${table_name}" } } ``` @@ -80,4 +188,3 @@ sink { ## Changelog - diff --git a/docs/zh/connectors/sink/Druid.md b/docs/zh/connectors/sink/Druid.md index 4367ff1b6cc0..baa7353d0c7e 100644 --- a/docs/zh/connectors/sink/Druid.md +++ b/docs/zh/connectors/sink/Druid.md @@ -6,9 +6,9 @@ import ChangeLog from '../changelog/connector-druid.md'; ## 描述 -一个使用向 Druid 发送消息的接收器插件 +通过 Druid 索引任务 API 将数据写入 Apache Druid。 -## 关键特性 +## 主要特性 - [ ] [精确一次](../../introduction/concepts/connector-v2-features.md) - [x] [支持多表写入](../../introduction/concepts/connector-v2-features.md) @@ -30,49 +30,157 @@ import ChangeLog from '../changelog/connector-druid.md'; ## 选项 -| 名称 | 类型 | 必需 | 默认值 | -|----------------|--------|----|---------------| -| coordinatorUrl | string | 是 | - | -| datasource | string | 是 | - | -| batchSize | int | 否 | 10000 | -| common-options | | 否 | - | +| 名称 | 类型 | 必需 | 默认值 | +|----------------|--------|------|--------| +| coordinatorUrl | string | 是 | - | +| datasource | string | 是 | - | +| batchSize | int | 否 | 10000 | +| common-options | | 否 | - | ### coordinatorUrl [string] -Druid的协调器URL主机和端口，示例: "myHost:8888" +Druid 协调器或路由节点的主机和端口，例如 `router:8888`。 + +SeaTunnel 会向 `http://{coordinatorUrl}/druid/indexer/v1/task` 提交索引任务，所以这里只需要填写主机和端口，不要带协议和 API 路径。 ### datasource [string] -要写入的数据源名称，示例: "seatunnel" +要写入的 Druid datasource 名称。 + +当上游有多张表时，可以使用 `${table_name}` 这类占位符，把每张上游表写入不同的 Druid datasource。 ### batchSize [int] -每批刷新为Druid的行数。默认值为 `1024`. +SeaTunnel 缓存多少行之后向 Druid 提交一次索引任务。默认值为 `10000`。 + +写入器关闭时，SeaTunnel 也会把剩余缓存数据提交到 Druid。 ### common options -Sink插件常用参数，详见 [Sink Common Options](../common-options/sink-common-options.md) for details +Sink 插件通用参数，详见 [Sink Common Options](../common-options/sink-common-options.md)。 + +多表写入时，可以配合通用参数里的 `multi_table_sink_replica` 使用。 + +## 写入行为 + +连接器会把 SeaTunnel 每一行数据转换成内联 CSV 数据，然后作为 Druid 原生批量索引任务提交。 + +Druid 写入需要主时间列。连接器会自动追加一个名为 `timestamp` 的处理时间列给 Druid 使用；上游的 `TIMESTAMP` 字段会按上面的类型映射写成字符串维度。 ## 示例 -简单的例子: +### 写入单表 ```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + plugin_output = "fake" + schema = { + fields { + c_boolean = boolean + c_timestamp = timestamp + c_string = string + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(16, 1)" + } + } + rows = [ + { + kind = INSERT + fields = [true, "2020-02-02T02:02:02", "NEW", 1, 2, 3, 4, 4.3, 5.3, 6.3] + }, + { + kind = INSERT + fields = [false, "2012-12-21T12:34:56", "AAA", 1, 1, 333, 323232, 3.1, 9.33333, 99999.99999999] + } + ] + } +} + sink { Druid { - coordinatorUrl = "testHost:8888" - datasource = "seatunnel" + coordinatorUrl = "router:8888" + datasource = "testDataSource" } } ``` -使用占位符获取上游表元数据示例: +### 写入多表 + +使用 `${table_name}` 可以把每张上游表写入同名的 Druid datasource。 ```hocon +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + FakeSource { + tables_configs = [ + { + schema = { + table = "druid_sink_1" + fields { + id = int + val_bool = boolean + val_tinyint = tinyint + val_smallint = smallint + val_int = int + val_bigint = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3, 5.3, 6.3, "NEW"] + } + ] + }, + { + schema = { + table = "druid_sink_2" + fields { + id = int + val_bool = boolean + val_tinyint = tinyint + val_smallint = smallint + val_int = int + val_bigint = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3, 5.3, 6.3] + } + ] + } + ] + } +} + sink { Druid { - coordinatorUrl = "testHost:8888" - datasource = "${table_name}_test" + coordinatorUrl = "router:8888" + datasource = "${table_name}" } } ``` From beae69d9d0588154895a5d453c50d08791bd2879 Mon Sep 17 00:00:00 2001 From: "zhiwei.niu" Date: Thu, 2 Jul 2026 09:16:50 +0800 Subject: [PATCH 103/375] [Feature][Doc] Develop skills and coding guidelines for the Test suite (#11212) Co-authored-by: xuepeng <58297137+chl-wxp@users.noreply.github.com> --- .skills/seatunnel-test-suite/SKILL.md | 240 ++++++++ .../seatunnel-test-suite/agents/openai.yaml | 19 + docs/en/developer/test-coding-guide.md | 539 ++++++++++++++++++ docs/sidebars.js | 1 + docs/zh/developer/test-coding-guide.md | 518 +++++++++++++++++ 5 files changed, 1317 insertions(+) create mode 100644 .skills/seatunnel-test-suite/SKILL.md create mode 100644 .skills/seatunnel-test-suite/agents/openai.yaml create mode 100644 docs/en/developer/test-coding-guide.md create mode 100644 docs/zh/developer/test-coding-guide.md diff --git a/.skills/seatunnel-test-suite/SKILL.md b/.skills/seatunnel-test-suite/SKILL.md new file mode 100644 index 000000000000..fc4c91bbd208 --- /dev/null +++ b/.skills/seatunnel-test-suite/SKILL.md @@ -0,0 +1,240 @@ +--- +name: seatunnel-test-suite +description: Write or review Apache SeaTunnel E2E (Testcontainers) and unit tests so they are stable, leak-free, and deterministic. Use when creating a new *IT/*Test class, reviewing a test diff, or refactoring a flaky or leaking test. +--- + +# SeaTunnel Test Suite + +Self-contained operating manual for writing and reviewing SeaTunnel tests: the rule tables, citable IDs, and +the patterns needed to act. Everything required to apply or cite a rule is in this file. + +## When to use + +- Writing a new connector E2E test (`*IT.java`) or unit test (`*Test.java`). +- Reviewing a test diff for flakiness, leaks, or layout violations before a PR. +- Stabilizing a flaky or leaking test, or bringing an older test up to these conventions. + +## Framework contract (E2E) + +Every E2E test **must** follow this structural contract — violating it means the test silently does nothing. + +| Requirement | Detail | +|-------------|--------| +| Extend `TestSuiteBase` | Provides the engine container lifecycle, `NETWORK`, and test dispatch | +| `@TestInstance(Lifecycle.PER_CLASS)` | Inherited from `TestSuiteBase` — one instance shared across all methods; fields need not be `static` | +| `@TestTemplate` on each test method | NOT `@Test`. The `TestCaseInvocationContextProvider` only dispatches `@TestTemplate` methods | +| `TestContainer` parameter | Each `@TestTemplate` method must accept a single `TestContainer container` parameter | +| `@DisabledOnContainer` (optional) | Add **only** when the scenario genuinely can't run on an engine — see below. Default to no annotation so the test runs on all engines. | + +Minimal skeleton — runs on every engine (Zeta, Flink, Spark), which is the default and correct case for +most batch source/sink tests: + +```java +public class MyConnectorIT extends TestSuiteBase { + + @TestTemplate + public void testSourceToSink(TestContainer container) throws Exception { + Container.ExecResult result = container.executeJob("/my_connector_to_assert.conf"); + Assertions.assertEquals(0, result.getExitCode()); + } +} +``` + +### When to add `@DisabledOnContainer` + +Do not exclude an engine by default. Add the annotation only when a scenario provably cannot run on that +engine, and word `disabledReason` for the *specific* scenario — not a blanket "engine X is unsupported". +The exclusion can go on the class (whole suite) or a single `@TestTemplate` method (just that case). Real +reasons from the codebase: + +| Excluded engines | Why (the real constraint) | +|------------------|---------------------------| +| Spark | CDC / streaming jobs — Spark doesn't support the continuous/changelog mode | +| Spark + Flink | checkpoint-restore tests; scenarios needing Zeta-only features | +| Spark + Flink (all but Zeta) | continuous-discovery long-running jobs; SeaTunnel-only behavior | +| Spark | drops the RowKind of a record, so changelog assertions fail | + +```java +// Only this CDC case can't run on Spark — exclude at the method, not the class: +@TestTemplate +@DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = "Spark does not support the CDC streaming job") +public void testCdcStreaming(TestContainer container) throws Exception { + // ... +} +``` + +## The rules + +Two namespaces: **E1–E7 govern E2E (`*IT`) tests** (cite as `[E3]`); **U1–U6 govern unit (`*Test`) +tests** (cite as `[U5]`). E6 is Zeta-engine-only. Pick the namespace that matches the file under review — +never apply an E-rule to a unit test or vice versa. + +| # | Rule | Check (the smell) | Fix | +|---|------|-------------------|-----| +| E1 | Dynamic ports | literal `127.0.0.1:` / `localhost:` in Java code | `container.getHost()` + `container.getMappedPort(p)`; `.conf` files reference the network alias + internal port (see E7) | +| E2 | Condition-based waiting | any `Thread.sleep(...)` | `Awaitility.await().atMost(...).pollInterval(...).untilAsserted(...)`; timeout from the scenario table below | +| E3 | Release resources | an opened client/connection/container with no close | implement `TestResource`, close in `tearDown()` reverse-order + null-safe; try-with-resources for method-scoped | +| E4 | Async job submission | inline `executeJob` for a streaming/CDC job (never returns) | `CompletableFuture.supplyAsync(...)`, gate on `RUNNING`, act, verify, `cancel(true)` | +| E5 | Share one container | a fresh container started per test method | share one container across all methods in a class (`@TestInstance(PER_CLASS)`) | +| E5b | Pin image versions | `:latest` image tag | pin a specific version, e.g. `mysql:8.0.32`, never `mysql:latest` (see subsection) | +| E5c | Cover all data types | only `String`/`int` exercised | cover the connector's full data-type set, not just the easy ones | +| E6 | Thread whitelist (Zeta) | `There are still threads running in the container` | close the client first (E3); if the lib thread is unrecyclable, whitelist its prefix in `isIssueWeAlreadyKnow(...)` | +| E7 | Docker network | container not reachable from engine container | `.withNetwork(NETWORK).withNetworkAliases("my-alias")`; start with `Startables.deepStart(...)`; add `Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))` for debug logs (full setup below) | + +### E7 — network setup + +The SeaTunnel engine runs **inside** its own Docker container. External service containers must share the +same Docker network so the engine can reach them (this is also why E1's `.conf` files use the alias, not the +mapped host port). Setup pattern: + +```java +// In your IT class @BeforeAll (NETWORK is inherited from TestSuiteBase): +container = new GenericContainer<>(DockerImageName.parse("my-service:1.2.3")) + .withNetwork(NETWORK) // inherited field from TestSuiteBase + .withNetworkAliases("my-service-host") // reachable by this name from engine + .withExposedPorts(3306) + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger("my-service:1.2.3"))); +Startables.deepStart(Stream.of(container)).join(); // parallel startup +``` + +In your `.conf` file (runs inside the engine container): +```hocon +host = "my-service-host" +port = 3306 +``` + +In your Java test setup (runs on the host): +```java +String jdbcUrl = String.format("jdbc:mysql://%s:%d/test", + container.getHost(), container.getMappedPort(3306)); +``` + +### E2 — timeout reference + +| Scenario | atMost | pollInterval | +|----------|--------|--------------| +| Container / client readiness | 2 min | 1 s | +| Job reaches `RUNNING` | 1 min | 2 s | +| Batch job result verified | 60 s | 2 s | +| MQ / Kafka consumption | 30–60 s | 1 s | +| CDC / schema-change propagation | 60–120 s | 2–5 s | + +Chain `.ignoreExceptions()` onto the `Awaitility.await()` (the E2 fix) whenever the client throws until the +service is up — otherwise the first poll's exception fails the wait instead of retrying. Never pick a bare +round number without a scenario to justify it. + +### E5b — image version pinning + +Never use `:latest` — upstream images can change behavior without warning, breaking CI with no code diff to +bisect. Pin to a specific minor or patch version: + +```java +// Bad — breaks unpredictably +private static final String IMAGE = "postgres:latest"; + +// Good — reproducible +private static final String IMAGE = "postgres:14.5"; +``` + +### E6 — thread whitelist, in one paragraph + +After the last job finishes, `SeaTunnelContainer` snapshots the server JVM and fails if a non-system thread +survives 120s. Daemon threads from third-party clients (JDBC drivers, HTTP pools) are sometimes unrecyclable. +For those only, add a **specific name prefix** to `isIssueWeAlreadyKnow(String)` in `SeaTunnelContainer`, with +a comment naming the library. A thread you *can* close belongs in `tearDown()`, not the whitelist. System +threads (`hz.main`, `pool-N-thread-N`, …) are already handled by `isSystemThread(...)` — don't duplicate them. + +### UT rules + +Use this table for `*Test` classes (Surefire, no engine container): + +| # | Rule | Smell | Fix | +|---|------|-------|-----| +| U1 | Test behavior, not internals | asserts on private fields or internal call order | assert observable output/state; verify `void` calls only when side-effect IS the contract | +| U2 | No timing or IO dependence | `Thread.sleep`, `System.currentTimeMillis()`, filesystem, network | remove timing; mock/fake IO boundaries; inject clocks if needed | +| U3 | Mock at boundaries only | mocking domain objects or `new`-able value types | only mock external interfaces/clients/maps/services the class depends on via constructor or setter | +| U4 | One behavior per method | multi-branch `if`/`for` inside a single test | split into one method per branch; name each method as the behavior it proves | +| U5 | Negative paths assert message | `assertThrows(FooException.class, ...)` with no message check | also call `assertThat(ex.getMessage()).contains("expected fragment")` | +| U6 | Naming | `testFoo`, `testBar`, `test123` | class `*Test`; methods like `shouldReturn404WhenUserMissing`, `throwsOnNullHost` | + +**Mock setup shape** — each test method is Arrange-Act-Assert: +1. *Arrange* — in `@BeforeEach`, `mock(...)` every external boundary, wire the chain with `when(...).thenReturn(...)`, then construct the SUT. Per test, stub only the values that select the branch under test. +2. *Act* — call the one method under test. +3. *Assert* — check observable output (U1), not call chains. + +Pitfalls that make a mocked test pass for the wrong reason: +- When the SUT reaches a collaborator through a chain (`a.getB().getC(key)`), stub the whole chain, and stub each lookup with the *exact* key constant the production code uses — a mismatched key returns `null` and silently changes the branch taken. +- To exercise a downstream branch, stub every upstream guard non-null first; a null upstream value short-circuits early, so stubbing only the downstream value tests nothing. +- For negative paths, assert the exception message fragment, not just the type (U5) — a wrong-cause throw of the same type would otherwise pass. + +**When NOT to mock:** +- Simple value objects, POJOs, DTOs — use real instances. +- The class under test itself — never mock the SUT. +- When an in-memory fake (e.g. `HashMap` instead of an interface) is simpler and equally isolated. + +## Layout (where things go) + +One Maven module per connector under `seatunnel-e2e/seatunnel-connector-v2-e2e/connector--e2e/`. Files +load from the test **classpath root** (`src/test/resources/`): + +- Test class → `src/test/java/org/apache/seatunnel/e2e/connector//IT.java` + (CDC connectors use the `...connectors.seatunnel.cdc.` package). `*IT` = integration (Failsafe), + `*Test` = unit (Surefire). +- Job configs → `src/test/resources/_source_to_sink.conf`, referenced with a **leading slash** + (`executeJob("/_source_to_sink.conf")`). License header required (`#` comments). +- DDL → `src/test/resources/ddl/

.sql` (CDC `UniqueDatabase` resolves `ddl/