From cbe39e58680c206584fadfe8a1f5542b159101ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= <159546+serprex@users.noreply.github.com> Date: Wed, 13 May 2026 06:34:16 +0000 Subject: [PATCH] Replace clickhouse-cpp with clickhouse-c The clickhouse-cpp library's use of [RAII] and C++ exceptions conflicted with PostgreSQL's memory management and `PG_TRY`/`setjmp`/`longjmp` patterns. Analysis of these conflicts lead to the conclusion that continuing to use the C++ library was unsafe. Instead, replace the vendored clickhouse-cpp library with the new headers-only [clickhouse-c] client, also vendored. This package provides an interface for memory management functions, the `chc_alloc` struct, mapped here to the Postgres `palloc` family of functions, to ensure consistent memory handling. It also supports fetching results by block, rather than all at once, for reduced memory consumption. This change results in greatly reduced build time without the need to compile the vendored C++ library, and reduces the size of the resulting shared library ~75%. The new library supports all the same features as clickhouse-cpp, but at a lower level, such that we must handle the conversion of values from the ClickHouse binary wire format. Most of the conversions are fairly straightforward, with a few exceptions: * Encoding and decoding Decimal values * Encoding parameters requires special quoting and escaping * Connecting via TCP or TLS requires lower-level socket configuration However, it also means we no longer rely on clickhouse-cpp's vendored dependencies (absl, cityhash, lz4, and zstd). Instead, we simply import the appropriate headers to support compression and encryption and require the necessary libraries (`liblz4` and `libzstd`). In other words, no more vendoring aside from the headers. Take advantage of the consistent use of PostgreSQL's memory management by creating memory contexts specific to each operation for easy cleanup. Replace the `src/binary.cpp` and `src/convert.c` files with the `src/binary` directory with various responsibilities split by `.c` file for clearer organization: * `binary.c` - core glue for the driver * `binary_internal.h` - Private state for the driver * `connection.c` - TCP with and without TLS connections * `convert.c` - Binary value conversion, mostly unchanged from `src/convert.c` * `decode.c` - Convert ClickHouse wire column values to Postgres Datums; used for `SELECT` * `encode.c` - Convert Postgres Datums to ClickHouse binary values and append to columns; used for `INSERT` * `insert.c` - Handles `INSERT` process: prepare, insert (flush), and finalization, along with appending values to ClickHouse columns * `select.c` - Handles `SELECT` process: simple query, fetching rows (pump) and cleanup; returns results by block, rather than buffering all the results at once Preserve all previous behaviors with three exceptions: * UInt16 values were previously incorrectly cast to `int16` instead of `int32`. Fixed here, along with the tests, bringing the behavior in line with the http driver. * Bool values cast to `bool`. * Improved some messages and added additional query context to error messages. Update the `Dockerfile` to require `liblz4` and `libzstd`. Vastly simplify `Makefile`, now that we no longer need to manage a vendored C++ library that uses `cmake`. Instead we simply add all the C files to `OBJS` and pull the vendored submodule if it's not already present (it ships with release packages). We also add the no-longer vendored `lz4` and `zstd` dependencies to `PG_LDFLAGS`. We also do away with `clang-format`, which we'd used only for the C++ files, and teach `pg_bsd_indent` and `clang-tidy` to find the new `*.c` files in `src/binary`. Update the GitHub workflow to eliminate caching, since we no longer build `clickhouse-cpp`, and to install the `lz4` and `zstd` dependencies. We also remove the separate static and dynamic tests, since there is just the static build now. [RAII]: https://en.wikipedia.org/wiki/Resource_acquisition_is_initialization [clickhouse-c]: https://github.com/ClickHouse/clickhouse-c Co-authored-by: David E. Wheeler --- .clang-format | 295 ------ .github/workflows/bake.yml | 5 - .github/workflows/clickhouse.yml | 18 +- .github/workflows/lint.yml | 4 +- .github/workflows/postgres.yml | 18 +- .github/workflows/release.yml | 2 +- .gitmodules | 6 +- .pre-commit-config.yaml | 4 +- CHANGELOG.md | 17 + Dockerfile | 4 +- Makefile | 112 +-- README.md | 27 +- dev/README.md | 1 - dev/indent.sh | 4 +- src/binary.cpp | 1353 ------------------------- src/binary/binary.c | 94 ++ src/binary/binary_internal.h | 210 ++++ src/binary/connection.c | 261 +++++ src/{ => binary}/convert.c | 42 +- src/binary/decode.c | 888 +++++++++++++++++ src/binary/encode.c | 428 ++++++++ src/binary/insert.c | 1475 ++++++++++++++++++++++++++++ src/binary/select.c | 462 +++++++++ src/connection.c | 16 +- src/fdw.c | 17 +- src/include/binary.h | 116 +++ src/include/binary.hh | 110 --- src/include/fdw.h | 5 + src/option.c | 9 +- src/pglink.c | 152 ++- test/expected/binary.out | 3 +- test/expected/binary_inserts.out | 6 +- test/expected/binary_inserts_1.out | 6 +- test/expected/binary_queries_5.out | 2 +- test/expected/binary_queries_6.out | 747 -------------- test/expected/binary_queries_7.out | 745 -------------- test/expected/import_schema.out | 15 +- test/expected/import_schema_1.out | 15 +- test/expected/import_schema_2.out | 774 --------------- test/expected/json_3.out | 58 +- test/expected/json_4.out | 42 +- test/expected/json_5.out | 42 +- test/expected/json_6.out | 42 +- test/expected/result_map.txt | 17 +- test/sql/import_schema.sql | 1 + vendor/clickhouse-c | 1 + vendor/clickhouse-cpp | 1 - 47 files changed, 4298 insertions(+), 4374 deletions(-) delete mode 100644 .clang-format delete mode 100644 src/binary.cpp create mode 100644 src/binary/binary.c create mode 100644 src/binary/binary_internal.h create mode 100644 src/binary/connection.c rename src/{ => binary}/convert.c (94%) create mode 100644 src/binary/decode.c create mode 100644 src/binary/encode.c create mode 100644 src/binary/insert.c create mode 100644 src/binary/select.c create mode 100644 src/include/binary.h delete mode 100644 src/include/binary.hh delete mode 100644 test/expected/binary_queries_6.out delete mode 100644 test/expected/binary_queries_7.out delete mode 100644 test/expected/import_schema_2.out create mode 160000 vendor/clickhouse-c delete mode 160000 vendor/clickhouse-cpp diff --git a/.clang-format b/.clang-format deleted file mode 100644 index a2832f58..00000000 --- a/.clang-format +++ /dev/null @@ -1,295 +0,0 @@ ---- -# Language: Cpp -AccessModifierOffset: -2 -AlignAfterOpenBracket: Align -AlignArrayOfStructures: None -AlignConsecutiveAssignments: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: true -AlignConsecutiveBitFields: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignConsecutiveDeclarations: - Enabled: true - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: true - AlignFunctionPointers: true - PadOperators: false -AlignConsecutiveMacros: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignConsecutiveShortCaseStatements: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCaseArrows: false - AlignCaseColons: false -AlignConsecutiveTableGenBreakingDAGArgColons: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignConsecutiveTableGenCondOperatorColons: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignConsecutiveTableGenDefinitionColons: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignEscapedNewlines: Right -AlignOperands: Align -AlignTrailingComments: - Kind: Always - OverEmptyLines: 0 -AllowAllArgumentsOnNextLine: true -AllowAllParametersOfDeclarationOnNextLine: true -AllowBreakBeforeNoexceptSpecifier: Never -AllowShortBlocksOnASingleLine: Never -AllowShortCaseExpressionOnASingleLine: true -AllowShortCaseLabelsOnASingleLine: false -AllowShortCompoundRequirementOnASingleLine: true -AllowShortEnumsOnASingleLine: true -AllowShortFunctionsOnASingleLine: All -AllowShortIfStatementsOnASingleLine: Never -AllowShortLambdasOnASingleLine: All -AllowShortLoopsOnASingleLine: false -AllowShortNamespacesOnASingleLine: false -AlwaysBreakAfterDefinitionReturnType: All -AlwaysBreakBeforeMultilineStrings: false -AttributeMacros: - - __capability -BinPackArguments: true -BinPackLongBracedList: true -BinPackParameters: BinPack -BitFieldColonSpacing: Both -BracedInitializerIndentWidth: -1 -BraceWrapping: - AfterCaseLabel: true - AfterClass: true - AfterControlStatement: Always - AfterEnum: true - AfterExternBlock: true - AfterFunction: true - AfterNamespace: true - AfterObjCDeclaration: true - AfterStruct: true - AfterUnion: true - BeforeCatch: true - BeforeElse: true - BeforeLambdaBody: true - BeforeWhile: true - IndentBraces: false - SplitEmptyFunction: true - SplitEmptyRecord: true - SplitEmptyNamespace: true -BreakAdjacentStringLiterals: true -BreakAfterAttributes: Leave -BreakAfterJavaFieldAnnotations: false -BreakAfterReturnType: AllDefinitions -BreakArrays: true -BreakBeforeBinaryOperators: All -BreakBeforeConceptDeclarations: Always -BreakBeforeBraces: Custom -BreakBeforeInlineASMColon: OnlyMultiline -BreakBeforeTemplateCloser: false -BreakBeforeTernaryOperators: true -BreakBinaryOperations: Never -BreakConstructorInitializers: BeforeColon -BreakFunctionDefinitionParameters: false -BreakInheritanceList: BeforeColon -BreakStringLiterals: true -BreakTemplateDeclarations: MultiLine -ColumnLimit: 120 -CommentPragmas: "^ IWYU pragma:" -CompactNamespaces: false -ConstructorInitializerIndentWidth: 4 -ContinuationIndentWidth: 0 -Cpp11BracedListStyle: false -DerivePointerAlignment: false -DisableFormat: false -EmptyLineAfterAccessModifier: Never -EmptyLineBeforeAccessModifier: LogicalBlock -EnumTrailingComma: Leave -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: false -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH -IfMacros: - - KJ_IF_MAYBE -IncludeBlocks: Preserve -IncludeCategories: - - Regex: '^"(llvm|llvm-c|clang|clang-c)/' - Priority: 2 - SortPriority: 0 - CaseSensitive: false - - Regex: '^(<|"(gtest|gmock|isl|json)/)' - Priority: 3 - SortPriority: 0 - CaseSensitive: false - - Regex: ".*" - Priority: 1 - SortPriority: 0 - CaseSensitive: false -IncludeIsMainRegex: "(Test)?$" -IncludeIsMainSourceRegex: "" -IndentAccessModifiers: false -IndentCaseBlocks: false -IndentCaseLabels: true -IndentExportBlock: true -IndentExternBlock: AfterExternBlock -IndentGotoLabels: true -IndentPPDirectives: None -IndentRequiresClause: true -IndentWidth: 4 -IndentWrappedFunctionNames: false -InsertBraces: false -InsertNewlineAtEOF: false -InsertTrailingCommas: None -IntegerLiteralSeparator: - Binary: 0 - BinaryMinDigits: 0 - Decimal: 0 - DecimalMinDigits: 0 - Hex: 0 - HexMinDigits: 0 -JavaScriptQuotes: Leave -JavaScriptWrapImports: true -KeepEmptyLines: - AtEndOfFile: false - AtStartOfBlock: true - AtStartOfFile: true -KeepFormFeed: true -LambdaBodyIndentation: Signature -LineEnding: DeriveLF -MacroBlockBegin: "" -MacroBlockEnd: "" -MainIncludeChar: Quote -MaxEmptyLinesToKeep: 1 -NamespaceIndentation: None -ObjCBinPackProtocolList: Auto -ObjCBlockIndentWidth: 2 -ObjCBreakBeforeNestedBlockParam: true -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: true -OneLineFormatOffRegex: "" -PackConstructorInitializers: BinPack -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 19 -PenaltyBreakBeforeMemberAccess: 150 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakOpenParenthesis: 0 -PenaltyBreakScopeResolution: 500 -PenaltyBreakString: 1000 -PenaltyBreakTemplateDeclaration: 10 -PenaltyExcessCharacter: 1000000 -PenaltyIndentedWhitespace: 0 -PenaltyReturnTypeOnItsOwnLine: 60 -PointerAlignment: Right -PPIndentWidth: -1 -QualifierAlignment: Leave -ReferenceAlignment: Right -ReflowComments: Always -RemoveBracesLLVM: false -RemoveEmptyLinesInUnwrappedLines: false -RemoveParentheses: Leave -RemoveSemicolon: false -RequiresClausePosition: OwnLine -RequiresExpressionIndentation: OuterScope -SeparateDefinitionBlocks: Leave -ShortNamespaceLines: 1 -SkipMacroDefinitionBody: false -SortIncludes: - Enabled: true - IgnoreCase: false -SortJavaStaticImport: Before -SortUsingDeclarations: LexicographicNumeric -SpaceAfterCStyleCast: false -SpaceAfterLogicalNot: false -SpaceAfterOperatorKeyword: false -SpaceAfterTemplateKeyword: true -SpaceAroundPointerQualifiers: Default -SpaceBeforeAssignmentOperators: true -SpaceBeforeCaseColon: false -SpaceBeforeCpp11BracedList: false -SpaceBeforeCtorInitializerColon: true -SpaceBeforeInheritanceColon: true -SpaceBeforeJsonColon: false -SpaceBeforeParens: ControlStatements -SpaceBeforeParensOptions: - AfterControlStatements: false - AfterForeachMacros: false - AfterFunctionDefinitionName: false - AfterFunctionDeclarationName: false - AfterIfMacros: false - AfterNot: false - AfterOverloadedOperator: false - AfterPlacementOperator: true - AfterRequiresInClause: false - AfterRequiresInExpression: false - BeforeNonEmptyParentheses: false -SpaceBeforeRangeBasedForLoopColon: true -SpaceBeforeSquareBrackets: false -SpaceInEmptyBlock: false -SpacesBeforeTrailingComments: 1 -SpacesInAngles: Never -SpacesInContainerLiterals: true -SpacesInLineCommentPrefix: - Minimum: 1 - Maximum: -1 -SpacesInParens: Never -SpacesInParensOptions: - ExceptDoubleParentheses: false - InCStyleCasts: false - InConditionalStatements: false - InEmptyParentheses: false - Other: false -SpacesInSquareBrackets: false -Standard: Latest -StatementAttributeLikeMacros: - - Q_EMIT -StatementMacros: - - Q_UNUSED - - QT_REQUIRE_VERSION -TableGenBreakInsideDAGArg: DontBreak -TabWidth: 4 -UseTab: Always -VerilogBreakBetweenInstancePorts: true -WhitespaceSensitiveMacros: - - BOOST_PP_STRINGIZE - - CF_SWIFT_NAME - - NS_SWIFT_NAME - - PP_STRINGIZE - - STRINGIZE -WrapNamespaceBodyWithEmptyLines: Leave diff --git a/.github/workflows/bake.yml b/.github/workflows/bake.yml index 1e855771..8d216a24 100644 --- a/.github/workflows/bake.yml +++ b/.github/workflows/bake.yml @@ -29,11 +29,6 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Vendor Build - uses: actions/cache@v4 - with: - path: vendor/_build - key: ${{ runner.os }}-vendor-build - name: Set Bake Variables run: make bake-vars REGISTRY=ghcr.io/clickhouse PG_VERSIONS=${{ matrix.pgv }} >> $GITHUB_ENV - name: Build${{ startsWith(github.ref, 'refs/tags') && ' and Push' || '' }} diff --git a/.github/workflows/clickhouse.yml b/.github/workflows/clickhouse.yml index 441ab1ce..b7420157 100644 --- a/.github/workflows/clickhouse.yml +++ b/.github/workflows/clickhouse.yml @@ -27,7 +27,7 @@ jobs: container: pgxn/pgxn-tools steps: - name: Start Postgres - run: pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev libre2-dev + run: pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev libre2-dev - name: Install Extensions run: pgxn install re2 - name: Checkout the Repository @@ -36,19 +36,5 @@ jobs: - name: Start ClickHouse env: { CH_RELEASE: "${{ matrix.ch }}" } run: .github/ubuntu/clickhouse.sh - - name: Cache Dependencies - uses: actions/cache@v5 - with: - path: vendor/_build/* - key: vendor-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.git/modules/clickhouse-cpp/refs/heads/master') }} - # Required to prevent clickhouse-cpp from rebuilding because it depends - # on this file in the Makefile. https://github.com/actions/checkout/issues/968 - - name: Reset Vendor Timestamp - run: cd vendor/clickhouse-cpp && touch -d $(git log -1 --format="@%ct" CMakeLists.txt) CMakeLists.txt - - name: Test DSO + - name: Test run: pg-build-test - - name: Clean - run: make clean NO_VENDOR_CLEAN=1 - - name: Test Static - run: pg-build-test - env: { CH_BUILD: static } diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 76bc162d..43273af8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,9 +14,7 @@ jobs: env: { pg: 18 } steps: - name: Install Postgres - run: NO_CLUSTER=1 pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev clang-tidy - - name: Install clang-format # version in Ubuntu is too old - run: curl -sLo /usr/local/bin/clang-format https://github.com/cpp-linter/clang-tools-static-binaries/releases/latest/download/clang-format-22_linux-amd64 && chmod +x /usr/local/bin/clang-format + run: NO_CLUSTER=1 pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev clang-tidy - name: Checkout the Repository uses: actions/checkout@v6 - name: Install pre-commit diff --git a/.github/workflows/postgres.yml b/.github/workflows/postgres.yml index 6f44f561..f2552aaf 100644 --- a/.github/workflows/postgres.yml +++ b/.github/workflows/postgres.yml @@ -19,7 +19,7 @@ jobs: container: pgxn/pgxn-tools steps: - name: Start Postgres ${{ matrix.pg }} - run: pg-start ${{ matrix.pg }} libcurl4-openssl-dev uuid-dev libre2-dev + run: pg-start ${{ matrix.pg }} libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev libre2-dev - name: Install Extensions run: pgxn install re2 - name: Checkout the Repository @@ -27,19 +27,5 @@ jobs: with: { submodules: true } - name: Start ClickHouse run: .github/ubuntu/clickhouse.sh - - name: Cache Dependencies - uses: actions/cache@v5 - with: - path: vendor/_build/* - key: vendor-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.git/modules/clickhouse-cpp/refs/heads/master') }} - # Required to prevent clickhouse-cpp from rebuilding because it depends - # on this file in the Makefile. https://github.com/actions/checkout/issues/968 - - name: Reset Vendor Timestamp - run: cd vendor/clickhouse-cpp && touch -d $(git log -1 --format="@%ct" CMakeLists.txt) CMakeLists.txt - - name: Test DSO + - name: Test run: pg-build-test - - name: Clean - run: make clean NO_VENDOR_CLEAN=1 - - name: Test Static - run: pg-build-test - env: { CH_BUILD: static } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d3ebe1c5..24014b01 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v6 - name: Start Postgres - run: pg-start 18 libcurl4-openssl-dev uuid-dev + run: pg-start 18 libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev - name: Start ClickHouse env: { CH_RELEASE: "${{ matrix.ch }}" } run: .github/ubuntu/clickhouse.sh diff --git a/.gitmodules b/.gitmodules index 55b8de33..bd173a51 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "clickhouse-cpp"] - path = vendor/clickhouse-cpp - url = https://github.com/ClickHouse/clickhouse-cpp.git +[submodule "vendor/clickhouse-c"] + path = vendor/clickhouse-c + url = https://github.com/ClickHouse/clickhouse-c diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88ed6f01..514c14ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,8 +36,8 @@ repos: - id: clang-tidy name: clang-tidy static analysis language: system - entry: sh -c 'test -f compile_commands.json && clang-tidy -p . "$@" || true' -- - files: '\.(c|cpp)$' + entry: sh -c 'test -f compile_commands.json && clang-tidy -p . "$@"' -- + files: '\.c$' exclude: "^vendor/" - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.1.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7add1799..47f44542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ All notable changes to this project will be documented in this file. It uses the ### ⚡ Improvements +* Replaced the `clickhouse-cpp` binary client with [ClickHouse/clickhouse-c], + pulled in as a git submodule and included in the release package under + `vendor/clickhouse-c`. This change eliminates conflicts between the C++ + and Postgres memory & exception handling and streams query results by the + ClickHouse block for reduced memory consumption. It also greatly reduces + build time and the size of the library by over 75%. Thanks to Philip Dubé + for the the new library and the PR ([#254]). * Added multidimensional array support across SELECT and INSERT to both the binary and http drivers. Rectangular ClickHouse `Array(Array(...))` values now map to PostgreSQL multidimensional arrays, jagged arrays not supported, @@ -22,11 +29,21 @@ All notable changes to this project will be documented in this file. It uses the `re2regexpquotemeta`, and `re2splitbyregexp`. Thanks to Philip Dubé for the PR ([#232]). +### 🐞 Bug Fixes + +* Fixed incorrect casting of ClickHouse `UInt16` values to `int16` in the + Binary driver. They now correctly convert to `int32` (Postgres `INT4`). + Part of the omnibus binary c driver conversion contributed by Philip Dubé + ([#233]). + [v0.3.1]: https://github.com/ClickHouse/pg_clickhouse/compare/v0.3.0...v0.3.1 [#232]: https://github.com/ClickHouse/pg_clickhouse/pull/232 "ClickHouse/pg_clickhouse#232 pushdown for new functions in pg_re2 0.3" [#233]: https://github.com/ClickHouse/pg_clickhouse/pull/233 "ClickHouse/pg_clickhouse#233 Support multidimensional arrays" + [ClickHouse/clickhouse-c]: https://github.com/ClickHouse/clickhouse-c + [#254]: https://github.com/ClickHouse/pg_clickhouse/pull/254 + "ClickHouse/pg_clickhouse#254 Replace clickhouse-cpp with clickhouse-c" ## [v0.3.0] — 2026-05-11 diff --git a/Dockerfile b/Dockerfile index 4b8b4ee1..185ecb32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ make \ cmake \ libssl-dev \ + liblz4-dev \ + libzstd-dev \ libre2-dev \ pgxnclient \ unzip \ @@ -24,7 +26,7 @@ RUN cd /tmp && pgxn download re2 && unzip re2-*.zip && rm re2-*.zip && cd re2-* FROM postgres:$PG_MAJOR-trixie # Install dependencies -RUN apt-get update && apt-get install -y --no-install-recommends libcurl4t64 uuid libre2-11 ca-certificates \ +RUN apt-get update && apt-get install -y --no-install-recommends libcurl4t64 uuid libre2-11 liblz4-1 libzstd1 ca-certificates \ && apt-get clean \ && rm -rf /var/cache/apt/* /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index 2d348c8d..2ef4e07b 100644 --- a/Makefile +++ b/Makefile @@ -16,97 +16,42 @@ CURL_CONFIG ?= curl-config OS ?= $(shell uname -s | tr A-Z a-z) ARCH = $(shell uname -m) -# Collect all the C++ and C files to compile into MODULE_big. -OBJS = $(subst .cpp,.o, $(wildcard src/*.cpp src/*/*.cpp)) \ - $(subst .c,.o, $(wildcard src/*.c src/*/*.c)) - -# Build static on Darwin by default. -ifndef CH_BUILD -# ifeq ($(OS),darwin) - CH_BUILD = static -# else -# CH_BUILD = dynamic -# endif -endif +# Collect all the C files to compile into MODULE_big. +OBJS = $(subst .c,.o, $(wildcard src/*.c src/*/*.c)) -# clickhouse-cpp source and build directories. -CH_CPP_DIR = vendor/clickhouse-cpp -CH_CPP_BUILD_DIR = vendor/_build/$(OS)-$(ARCH)-$(CH_BUILD)-$(shell git submodule status $(CH_CPP_DIR) | awk '{print substr($$1, 0, 7)}') - -# List the clickhouse-cpp libraries we require. -CH_CPP_LIB = $(CH_CPP_BUILD_DIR)/clickhouse/libclickhouse-cpp-lib$(DLSUFFIX) -CH_CPP_FLAGS = -D CMAKE_BUILD_TYPE=Release -D WITH_OPENSSL=ON - -# Are we statically compiling clickhouse-cpp into the extension or no? -ifeq ($(CH_BUILD), static) -# We'll need all the clickhouse-cpp static libraries. - CH_CPP_LIB = $(CH_CPP_BUILD_DIR)/clickhouse/libclickhouse-cpp-lib.a - SHLIB_LINK = $(CH_CPP_LIB) \ - $(CH_CPP_BUILD_DIR)/contrib/cityhash/cityhash/libcityhash.a \ - $(CH_CPP_BUILD_DIR)/contrib/absl/absl/libabsl_int128.a \ - $(CH_CPP_BUILD_DIR)/contrib/lz4/lz4/liblz4.a \ - $(CH_CPP_BUILD_DIR)/contrib/zstd/zstd/libzstdstatic.a -else -# Build and install the shared library. - SHLIB_LINK = -L$(CH_CPP_BUILD_DIR)/clickhouse -lclickhouse-cpp-lib - CH_CPP_FLAGS += -D BUILD_SHARED_LIBS=ON -endif +# clickhouse-c is a header-only single-header library. Override +# CH_C_DIR to point elsewhere when developing against a local checkout. +CH_C_DIR ?= vendor/clickhouse-c # Add include directories. -PG_CPPFLAGS = -I./src/include -I$(CH_CPP_DIR) -I$(CH_CPP_DIR)/contrib/absl - -# Include other libraries compiled into clickhouse-cpp. -PG_LDFLAGS = -lstdc++ -lssl -lcrypto $(shell $(CURL_CONFIG) --libs) +PG_CPPFLAGS = -I./src/include -I$(CH_C_DIR) -# clickhouse-cpp requires C++ v17. -PG_CXXFLAGS = -std=c++17 +# Link OpenSSL (for TLS in the binary driver), curl (for the HTTP driver), +# libuuid (for http_streaming.c's query-id generator), and lz4 / zstd +# (for the binary driver's compressed-frame codecs). +PG_LDFLAGS = -lssl -lcrypto -llz4 -lzstd $(shell $(CURL_CONFIG) --libs) -# Suppress annoying pre-c99 warning and include curl flags. -PG_CFLAGS = -Wno-declaration-after-statement -Werror=type-limits $(shell $(CURL_CONFIG) --cflags) - -# We'll need libuuid except on darwin, where it's included in the OS. +# libuuid is provided by the OS on darwin; explicit link elsewhere. ifneq ($(OS),darwin) PG_LDFLAGS += -luuid endif -# Clean up the clickhouse-cpp build directory and generated files. +# Suppress annoying pre-c99 warning and include curl flags. +PG_CFLAGS = -Wno-declaration-after-statement -Werror=type-limits $(shell $(CURL_CONFIG) --cflags) + +# Clean up generated files. EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql src/include/version.h compile_commands.json test/schedule $(EXTENSION)-$(DISTVERSION).zip -ifndef NO_VENDOR_CLEAN - EXTRA_CLEAN += $(CH_CPP_BUILD_DIR) -endif # Import PGXS. PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) -# We'll need the clickhouse-cpp library and rpath so it can be found. -SHLIB_LINK += -Wl,-rpath,$(pkglibdir)/ - -# PostgreSQL 15 and earlier violate a C++ v17 storage specifier error. -ifeq ($(shell test $(MAJORVERSION) -lt 16; echo $$?),0) - PG_CXXFLAGS += -Wno-register -endif - -# Add the flags to the bitcode compiler variables. -COMPILE.cc.bc += $(PG_CPPFLAGS) -COMPILE.cxx.bc += $(PG_CXXFLAGS) - -# shlib is the final output product: clickhouse-cpp and all .o dependencies. -$(shlib): $(CH_CPP_LIB) $(OBJS) - -# Clone clickhouse-cpp submodule. -$(CH_CPP_DIR)/CMakeLists.txt: +# Clone clickhouse-c submodule. +$(CH_C_DIR)/clickhouse.h: .gitmodules git submodule update --init -# Require the vendored clickhouse-cpp and the version header. -$(OBJS): $(CH_CPP_LIB) src/include/version.h - -# Build clickhouse-cpp. -$(CH_CPP_LIB): export CXXFLAGS=-fPIC -$(CH_CPP_LIB): export CFLAGS=-fPIC -$(CH_CPP_LIB): $(CH_CPP_DIR)/CMakeLists.txt # Sync with "Reset Vendor Timestamp" steps in workflows. - cmake -B $(CH_CPP_BUILD_DIR) -S $(CH_CPP_DIR) $(CH_CPP_FLAGS) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - cmake --build $(CH_CPP_BUILD_DIR) --parallel $$(nproc) --target all +# Require clickhouse-c and the version header. +$(OBJS): $(CH_C_DIR)/clickhouse.h src/include/version.h # Require the versioned C source and SQL script. all: sql/$(EXTENSION)--$(EXTVERSION).sql @@ -119,17 +64,6 @@ sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql src/include/version.h: src/include/version.h.in sed -e 's,__VERSION__,$(DISTVERSION),g' $< > $@ -# Configure install/uninstall of the clickhouse-cpp library. -ifneq ($(CH_BUILD), static) -# Copy all dynamic files; use -a to preserve symlinks. -install-ch-cpp: $(CH_CPP_LIB) $(shlib) - cp -a $(CH_CPP_BUILD_DIR)/clickhouse/libclickhouse-cpp-lib*$(DLSUFFIX)* $(DESTDIR)$(pkglibdir)/ -uninstall-ch-cpp: - rm -f $(DESTDIR)$(pkglibdir)/libclickhouse-cpp-lib*$(DLSUFFIX)* -install: install-ch-cpp -uninstall: uninstall-ch-cpp -endif - # Build a PGXN distribution bundle. dist: $(EXTENSION)-$(DISTVERSION).zip @@ -187,7 +121,7 @@ bake-vars: @echo "revision=$(REVISION)" @echo "pg_versions=$(PG_VERSIONS)" -# Format the .c, .h, and .hh files according to the PostgreSQL indentation +# Format the .c and .h files according to the PostgreSQL indentation # standard. Requires `pg_bsd_indent` to be in the path. indent: dev/indent.sh @$< @@ -199,7 +133,7 @@ lint: .pre-commit-config.yaml .PHONY: clang-tidy # Run clang-tidy static analysis (requires compile_commands.json) clang-tidy: compile_commands.json - clang-tidy -p . $(wildcard src/*.c src/*.cpp) + clang-tidy -p . $(wildcard src/*.c src/*/*.c) ## .git/hooks/pre-commit: Install the pre-commit hook .git/hooks/pre-commit: @@ -216,8 +150,8 @@ lsp: compile_commands.json # Requires https://github.com/rizsotto/Bear. compile_commands.json: - $(MAKE) clean -j $$(nproc) NO_VENDOR_CLEAN=$(NO_VENDOR_CLEAN) - bear -- $(MAKE) all -j $$(nproc) NO_VENDOR_CLEAN=$(NO_VENDOR_CLEAN) + $(MAKE) clean -j $$(nproc) + bear -- $(MAKE) all -j $$(nproc) # ClickHouse Docker Containers start-containers: dev/Makefile dev/docker-compose.yml diff --git a/README.md b/README.md index 58ec4b93..31beb463 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,11 @@ sudo apt install \ postgresql-server-18 \ libcurl4-openssl-dev \ uuid-dev \ + liblz4-dev \ + libzstd-dev \ libssl-dev \ make \ - cmake \ - g++ + gcc ``` #### RedHat / CentOS / Yum @@ -93,9 +94,10 @@ sudo yum install \ postgresql-server \ libcurl-devel \ libuuid-devel \ + lz4-devel \ + libzstd-devel \ openssl-libs \ automake \ - cmake \ gcc ``` @@ -121,18 +123,6 @@ make sudo make install ``` - - If your host has several PostgreSQL installations, you might need to specify the appropriate version of `pg_config`: @@ -239,8 +229,8 @@ CREATE EXTENSION pg_clickhouse SCHEMA env; ## Dependencies The `pg_clickhouse` extension requires [PostgreSQL] 13 or higher, [libcurl], -[libuuid]. Building the extension requires a C and C++ compiler, [libSSL], [GNU -make], and [CMake]. +[libuuid], [liblz4], and [libzstd]. Building the extension requires a C +compiler, [libSSL], and [GNU make]. ## Road Map @@ -294,8 +284,9 @@ adding DML features. Our road map: [PostgreSQL]: https://www.postgresql.org "PostgreSQL: The World's Most Advanced Open Source Relational Database" [libcurl]: https://curl.se/libcurl/ "libcurl — your network transfer library" [libuuid]: https://linux.die.net/man/3/libuuid "libuuid - DCE compatible Universally Unique Identifier library" + [liblz4]: https://lz4.org "LZ4 - Extremely fast compression" + [libzstd]: https://facebook.github.io/zstd/ "Zstandard - Fast real-time compression algorithm" [GNU make]: https://www.gnu.org/software/make "GNU Make" - [CMake]: https://cmake.org/ "CMake: A Powerful Software Build System" [LibSSL]: https://openssl-library.org "OpenSSL Library" [TPC-H]: https://www.tpc.org/tpch/ [re2]: https://github.com/ClickHouse/pg_re2 "pg_re2: ClickHouse-compatible regex functions using RE2" diff --git a/dev/README.md b/dev/README.md index c855e81a..dc7d729b 100644 --- a/dev/README.md +++ b/dev/README.md @@ -14,7 +14,6 @@ make start docker logs -f ch_25_11 # Wait a few minutes for the services to start (log output will stop) docker exec -it ch_25_11 bash -make clean NO_VENDOR_CLEAN=1 # Option to prevent clickhouse-cpp rebuild pg-build-test # or make && make install && make installcheck ``` diff --git a/dev/indent.sh b/dev/indent.sh index 942e6fb7..a407b22c 100755 --- a/dev/indent.sh +++ b/dev/indent.sh @@ -2,10 +2,8 @@ set -e -for fn in src/**/*.h* src/**/*.h.in src/*.c; do +for fn in src/**/*.h* src/**/*.h.in src/*.c src/**/*.c; do printf "%s\n" "$fn" pg_bsd_indent -bad -bap -bbb -bc -bl -cli1 -cp33 -cdb -nce -d0 -di12 -nfc1 -i4 -l79 -lp -lpl -nip -npro -sac -tpg -ts4 "$fn" done rm -f ./*.BAK - -clang-format --style=file:.clang-format -i src/binary.cpp diff --git a/src/binary.cpp b/src/binary.cpp deleted file mode 100644 index 61567c00..00000000 --- a/src/binary.cpp +++ /dev/null @@ -1,1353 +0,0 @@ -#include -#include -#include -#include - -#include "clickhouse/columns/date.h" -#include "clickhouse/columns/factory.h" -#include "clickhouse/columns/ip4.h" -#include "clickhouse/columns/lowcardinality.h" -#include "clickhouse/columns/nullable.h" -#include -#include -#include - -#if __cplusplus > 199711L -#define register /* Deprecated in C++11. */ -#endif /* #if __cplusplus > 199711L */ - -extern "C" -{ - -#include "postgres.h" - -#include "fmgr.h" -#include "funcapi.h" -#include "internal.h" -#include "pgtime.h" - -#include "access/htup_details.h" -#include "access/tupdesc.h" -#include "catalog/pg_type_d.h" -#include "utils/array.h" -#include "utils/builtins.h" -#include "utils/date.h" -#include "utils/elog.h" -#include "utils/inet.h" -#include "utils/lsyscache.h" -#include "utils/memdebug.h" -#include "utils/palloc.h" -#include "utils/timestamp.h" -#include "utils/uuid.h" - -#include "binary.hh" - - using namespace clickhouse; - -#if defined(__APPLE__) /* Byte ordering on macOS */ -#include -#include -#define HOST_TO_BIG_ENDIAN_64(x) OSSwapHostToBigInt64(x) -#define BIG_ENDIAN_64_TO_HOST(x) OSSwapBigToHostInt64(x) -#else -#include -#define HOST_TO_BIG_ENDIAN_64(x) htobe64(x) -#define BIG_ENDIAN_64_TO_HOST(x) be64toh(x) -#endif - -#define THROW_UNEXPECTED_COLUMN(exp_type, col) \ - throw std::runtime_error("unexpected column type for " + std::string(exp_type) + ": " + col->Type()->GetName()) - - /* palloc which will throw exceptions */ - static void * - exc_palloc(Size size) - { - /* duplicates MemoryContextAlloc to avoid increased overhead */ - void *ret; - MemoryContext context = CurrentMemoryContext; - - Assert(MemoryContextIsValid(context)); - - if (!AllocSizeIsValid(size)) - throw std::bad_alloc(); - - context->isReset = false; - -#if PG_VERSION_NUM >= 170000 - ret = context->methods->alloc(context, size, 0); -#else - ret = context->methods->alloc(context, size); -#endif - if (unlikely(ret == NULL)) - throw std::bad_alloc(); - - VALGRIND_MEMPOOL_ALLOC(context, ret, size); - - return ret; - } - - void * - exc_palloc0(Size size) - { - /* duplicates MemoryContextAllocZero to avoid increased overhead */ - void *ret; - MemoryContext context = CurrentMemoryContext; - - Assert(MemoryContextIsValid(context)); - - if (!AllocSizeIsValid(size)) - throw std::bad_alloc(); - - context->isReset = false; - -#if PG_VERSION_NUM >= 170000 - ret = context->methods->alloc(context, size, 0); -#else - ret = context->methods->alloc(context, size); -#endif - if (unlikely(ret == NULL)) - throw std::bad_alloc(); - - VALGRIND_MEMPOOL_ALLOC(context, ret, size); - - MemSetAligned(ret, 0, size); - - return ret; - } - -#define CLICKHOUSE_SECURE_PORT 9440 - - ch_binary_connection_t * - ch_binary_connect(ch_connection_details *details, char **error) - { - ClientOptions *options = NULL; - ch_binary_connection_t *conn = NULL; - - try - { - options = new ClientOptions(); - options->SetPingBeforeQuery(true); - - if (details->host) - { - options->SetHost(std::string(details->host)); - if (!details->port && ch_is_cloud_host(details->host)) - options->SetPort(CLICKHOUSE_SECURE_PORT); - } - if (details->port) - options->SetPort(details->port); - if (details->dbname) - options->SetDefaultDatabase(std::string(details->dbname)); - if (details->username) - options->SetUser(std::string(details->username)); - if (details->password) - options->SetPassword(std::string(details->password)); - if (options->port == CLICKHOUSE_SECURE_PORT) - options->SetSSLOptions(ClientOptions::SSLOptions()); - - /* options->SetRethrowException(false); */ - conn = new ch_binary_connection_t(); - - Client *client = new Client(*options); - conn->client = client; - conn->options = options; - } - catch (const std::exception &e) - { - if (error) - *error = strdup(e.what()); - - if (conn != NULL) - delete conn; - - if (options != NULL) - delete options; - - conn = NULL; - } - return conn; - } - - static void - set_resp_error(ch_binary_response_t *resp, const char *str) - { - if (resp->error) - return; - - resp->error = strdup(str); - } - - /* - * Converts query->settings to QuerySettings. - */ - static QuerySettings - ch_binary_settings(const Client *client, const ch_query *query) - { - kv_iter iter; - auto res = QuerySettings{}; - - for (iter = new_kv_iter(query->settings); !kv_iter_done(&iter); kv_iter_next(&iter)) - { - res.insert_or_assign(iter.name, QuerySettingsField{ iter.value, 1 }); - } - auto info = client->GetServerInfo(); - if (info.version_major >= 25 || (info.version_major == 24 && info.version_minor >= 10)) - res.insert_or_assign("output_format_native_write_json_as_string", QuerySettingsField{ "1", 1 }); - - return res; - } - - /* - * Converts query->param_values to QueryParams. - */ - static QueryParams - ch_binary_params(const ch_query *query) - { - int i; - auto res = QueryParams{}; - - for (i = 0; i < query->num_params; i++) - res.insert_or_assign(psprintf("p%d", i + 1), QueryParamValue(query->param_values[i])); - - return res; - } - - static void - set_state_error(ch_binary_read_state_t *state, const char *str) - { - assert(state->error == NULL); - state->error = strdup(str); - } - - ch_binary_response_t * - ch_binary_simple_query(ch_binary_connection_t *conn, const ch_query *query, bool (*check_cancel)(void)) - { - Client *client = (Client *)conn->client; - ch_binary_response_t *resp; - std::vector> *values; - - try - { - resp = new ch_binary_response_t(); - values = new std::vector>(); - client->Select(clickhouse::Query(query->sql) - .SetQuerySettings(ch_binary_settings(client, query)) - .SetParams(ch_binary_params(query)) - .OnProgress( - [&check_cancel](const Progress &) - { - if (check_cancel && check_cancel()) - throw std::runtime_error("query was canceled"); - }) - .OnDataCancelable( - [&resp, &values, &check_cancel](const Block &block) - { - if (check_cancel && check_cancel()) - { - set_resp_error(resp, "query was canceled"); - return false; - } - - /* some empty block */ - if (block.GetColumnCount() == 0) - return true; - - auto vec = std::vector(); - - if (resp->columns_count && block.GetColumnCount() != resp->columns_count) - { - set_resp_error(resp, "columns mismatch in blocks"); - return false; - } - - resp->columns_count = block.GetColumnCount(); - resp->blocks_count++; - - for (size_t i = 0; i < resp->columns_count; ++i) - vec.push_back(block[i]); - - values->push_back(std::move(vec)); - return true; - })); - - resp->values = (void *)values; - } - catch (const std::exception &e) - { - client->ResetConnection(); - - values->clear(); - set_resp_error(resp, e.what()); - delete values; - values = NULL; - } - - resp->success = (resp->error == NULL); - return resp; - } - - /* - * Given TypeRef for the value returned by a clickhouse-cpp query and the - * Postgres data type from the corresponding column in the foreign table, - * return the Oid of the type that should be created for the TypeRef. - */ - static Oid - get_corr_postgres_type(const TypeRef &type, Oid pg_type) - { - switch (type->GetCode()) - { - case Type::Code::Int8: - case Type::Code::Int16: - case Type::Code::UInt8: - return INT2OID; - case Type::Code::Int32: - case Type::Code::UInt16: - return INT4OID; - case Type::Code::Int64: - case Type::Code::UInt64: - case Type::Code::UInt32: - return INT8OID; - case Type::Code::Float32: - return FLOAT4OID; - case Type::Code::Float64: - return FLOAT8OID; - case Type::Code::Decimal128: - case Type::Code::Decimal64: - case Type::Code::Decimal32: - case Type::Code::Decimal: - return NUMERICOID; - case Type::Code::FixedString: - case Type::Code::Enum8: - case Type::Code::Enum16: - case Type::Code::String: - return TEXTOID; - case Type::Code::LowCardinality: - return get_corr_postgres_type(type->As()->GetNestedType(), pg_type); - case Type::Code::Date: - case Type::Code::Date32: - return DATEOID; - case Type::Code::DateTime: - return TIMESTAMPTZOID; - case Type::Code::DateTime64: - return TIMESTAMPTZOID; - case Type::Code::UUID: - return UUIDOID; - case Type::Code::Array: - { - /* postgres uses one array type for any number of dimensions, so - * walk past nested Array layers to the leaf element type. */ - auto leaf = type->As()->GetItemType(); - while (leaf->GetCode() == Type::Code::Array) - leaf = leaf->As()->GetItemType(); - - Oid array_type = get_array_type(get_corr_postgres_type(leaf, get_element_type(pg_type))); - if (array_type == InvalidOid) - throw std::runtime_error("pg_clickhouse: could not find array " - " type for column type " - + type->GetName()); - - return array_type; - } - case Type::Code::Tuple: - return RECORDOID; - case Type::Code::Nullable: - return get_corr_postgres_type(type->As()->GetNestedType(), pg_type); - case Type::Code::IPv4: - case Type::Code::IPv6: - return INETOID; - case Type::Code::JSON: - /* We support json and default to jsonb. */ - return pg_type == JSONOID ? JSONOID : JSONBOID; - default: - throw std::runtime_error("pg_clickhouse: unsupported column type " + type->GetName()); - } - } - - void - ch_binary_insert_state_free(void *c) - { - auto *state = (ch_binary_insert_state *)c; - if (state->insert_block) - { - /* Finish the insert to set the proper ClickHouse state */ - delete (Block *)state->insert_block; - Client *client = (Client *)state->conn->client; - try - { - client->EndInsert(); - } - catch (const std::exception &e) - { - client->ResetConnection(); - elog(ERROR, "pg_clickhouse: could not finish INSERT - %s", e.what()); - } - } - } - - void - ch_binary_prepare_insert(void *conn, const ch_query *query, ch_binary_insert_state *state) - { - /* Start the INSERT. */ - Block *block; - Client *client = (Client *)((ch_binary_connection_t *)conn)->client; - try - { - block = new Block(client->BeginInsert(std::string(query->sql) + " VALUES")); - /* XXX https://github.com/ClickHouse/clickhouse-cpp/pull/453/ - block = new Block(client->BeginInsert( - clickhouse::Query(std::string(query->sql)+ " VALUES").SetQuerySettings( - ch_binary_settings(client, query) - ).SetParams( - ch_binary_params(query) - ) - )); - */ - } - catch (const std::exception &e) - { - client->ResetConnection(); - elog(ERROR, "pg_clickhouse: could not prepare insert - %s", e.what()); - } - - /* Setup the column config (or return if no columns). */ - state->len = block->GetColumnCount(); - if (state->len == 0) - { - delete block; - return; - } - state->outdesc = CreateTemplateTupleDesc(state->len); - - /* Iterate over the list of columns returned by ClickHouse. */ - AttrNumber i = 0; - for (Block::Iterator bi(*block); bi.IsValid(); bi.Next()) - { - /* Start with the data type from the foreign table. */ - Oid pg_type = query->tupdesc - ? TupleDescAttr(query->tupdesc, lfirst_int(list_nth_cell(query->attr_nums, i)) - 1)->atttypid - : InvalidOid; - const char *colname = bi.Name().c_str(); - try - { - /* - * Determine the Postgres type to which to convert values so - * that column_append() knows how to append it to a ClickHouse - * column. - */ - pg_type = get_corr_postgres_type(bi.Type(), pg_type); - colname = bi.Name().c_str(); - } - catch (const std::exception &e) - { - elog(ERROR, "pg_clickhouse: could not prepare insert - %s", e.what()); - } - PG_TRY(); - { - TupleDescInitEntry(state->outdesc, ++i, colname, pg_type, -1, 0); - } - PG_CATCH(); - { - /* Clean up and re-throw. */ - client->ResetConnection(); - delete block; - PG_RE_THROW(); - } - PG_END_TRY(); - } - - state->insert_block = (ch_insert_block_h *)block; - } - - /* - * Append val to col. If isnull is true and val is nullable, append a - * null. Otherwise, determine how to convert val, of type valtype, to a - * value appropriate to col, and append that value. Raises an exception if - * valtype is not compatible with col's type. - */ - static void column_append(clickhouse::ColumnRef col, Datum val, Oid valtype, bool isnull); - - /* - * Build a single ColumnArray row's worth of data from a (possibly nested) - * ch_binary_array_t. items_type is the per-row element type of the parent - * ColumnArray: scalar T for Array(T), Array(T) for Array(Array(T)), etc. - * - * For nested types, recurses to build child ColumnArrays and stitches them - * via AppendAsColumn so the outer column's offsets describe the row shape. - */ - static clickhouse::ColumnRef - build_array_row_column(ch_binary_array_t *arr, clickhouse::TypeRef items_type) - { - using namespace clickhouse; - - auto col = CreateColumnByType(items_type->GetName()); - - /* Empty postgres array fits any nesting depth: nothing to walk, so - * skip the dim check and return an empty column at this level. */ - if (arr->len == 0) - return col; - - if (items_type->GetCode() == Type::Code::Array) - { - if (arr->ndim < 2) - throw std::runtime_error("pg_clickhouse: insert array has fewer dimensions than column type"); - - auto inner_arr = col->AsStrict(); - auto inner_items_t = items_type->As()->GetItemType(); - - for (size_t i = 0; i < arr->len; i++) - { - auto child = (ch_binary_array_t *)DatumGetPointer(arr->datums[i]); - auto sub = build_array_row_column(child, inner_items_t); - - inner_arr->AppendAsColumn(sub); - } - return col; - } - - if (arr->ndim != 1) - throw std::runtime_error("pg_clickhouse: insert array has more dimensions than column type"); - - for (size_t i = 0; i < arr->len; i++) - column_append(col, arr->datums[i], arr->item_type, arr->nulls[i]); - return col; - } - - static void - column_append(clickhouse::ColumnRef col, Datum val, Oid valtype, bool isnull) - { - bool nullable = false; - - if (col->Type()->GetCode() == Type::Code::Nullable) - nullable = true; - - /* - * Prevent insertion if the column isn't Nullable. This differs from the - * http engine, which isn't aware of Nullable but just sends values off, - * in which case ClickHouse inserts default values (e.g., 0 for numbers, - * "" for strings, etc.). - * - * XXX Do we want to consider rejecting NULL inserts when the Postgres - * `NOT NULL` constraint exits? And when it doesn't exist and the column - * is not nullable, do we want to send the default value instead? Dropping - * this block should do that. - */ - if (isnull && !nullable) - throw std::runtime_error("cannot append NULL to NOT NULL " + col->Type()->GetName() + " column"); - - if (nullable) - { - auto nullable = col->AsStrict(); - nullable->Append(isnull); - col = nullable->Nested(); - } - - switch (valtype) - { - case INT2OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::UInt8: - col->AsStrict()->Append((uint8_t)val); - break; - case Type::Code::Int8: - col->AsStrict()->Append((int8_t)val); - break; - case Type::Code::Int16: - col->AsStrict()->Append((int16_t)val); - break; - default: - THROW_UNEXPECTED_COLUMN("INT2", col); - } - break; - } - case INT4OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::Int32: - col->AsStrict()->Append((int32_t)val); - break; - case Type::Code::UInt16: - col->AsStrict()->Append((uint16_t)val); - break; - default: - THROW_UNEXPECTED_COLUMN("INT4", col); - } - break; - } - case INT8OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::Int64: - col->AsStrict()->Append((int64_t)val); - break; - case Type::Code::UInt32: - col->AsStrict()->Append((uint32_t)val); - break; - case Type::Code::UInt64: - col->AsStrict()->Append((uint64_t)val); - break; - default: - THROW_UNEXPECTED_COLUMN("INT8", col); - } - break; - } - case FLOAT4OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::Float32: - col->AsStrict()->Append(DatumGetFloat4(val)); - break; - default: - THROW_UNEXPECTED_COLUMN("FLOAT4", col); - } - break; - } - case FLOAT8OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::Float64: - col->AsStrict()->Append(DatumGetFloat8(val)); - break; - default: - THROW_UNEXPECTED_COLUMN("FLOAT8", col); - } - break; - } - case NUMERICOID: - { - /* Convert numeric to string and let ColumnDecimal parse it. */ - switch (col->Type()->GetCode()) - { - case Type::Code::Decimal128: - case Type::Code::Decimal64: - case Type::Code::Decimal32: - case Type::Code::Decimal: - if (isnull) - col->AsStrict()->Append(Int128{}); - else - { - char *s = DatumGetCString(DirectFunctionCall1(numeric_out, val)); - col->AsStrict()->Append(std::string(s)); - pfree(s); - } - break; - default: - THROW_UNEXPECTED_COLUMN("NUMERIC", col); - } - break; - } - case TEXTOID: - { - /* - * ClickHouse allows nuls in strings, and val can be bytea, so - * don't rely on nul termination but copy the full length of the - * data. - */ - std::string s; - if (!isnull) - { - text *string = PG_DETOAST_DATUM(val); - s.assign(VARDATA(string), VARSIZE_ANY_EXHDR(string)); - } - - switch (col->Type()->GetCode()) - { - case Type::Code::FixedString: - col->AsStrict()->Append(s); - break; - case Type::Code::String: - col->AsStrict()->Append(s); - break; - case Type::Code::Enum8: - if (isnull) - col->AsStrict()->Append(0, false); - else - col->AsStrict()->Append(s); - break; - case Type::Code::Enum16: - if (isnull) - col->AsStrict()->Append(0, false); - else - col->AsStrict()->Append(s); - break; - case Type::Code::LowCardinality: - /* XXX Handle LowCardinality(Nullable(x)) */ - if (col->AsStrict()->GetNestedType()->GetCode() == Type::Nullable) - throw std::runtime_error("nested Nullable is not supported"); - col->AsStrict>()->Append(s); - break; - default: - THROW_UNEXPECTED_COLUMN("TEXT", col); - } - - break; - } - case DATEOID: - { - Timestamp t = date2timestamp_no_overflow(DatumGetDateADT(val)); - pg_time_t d = timestamptz_to_time_t(t); - - switch (col->Type()->GetCode()) - { - case Type::Code::Date: - col->AsStrict()->Append(d); - break; - case Type::Code::Date32: - col->AsStrict()->Append(d); - break; - default: - THROW_UNEXPECTED_COLUMN("DATE", col); - } - break; - } - case TIMESTAMPOID: - case TIMESTAMPTZOID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::DateTime: - { - pg_time_t d = timestamptz_to_time_t(DatumGetTimestamp(val)); - col->AsStrict()->Append(d); - break; - } - case Type::Code::DateTime64: - { - auto dt64_col = col->AsStrict(); - Timestamp t = DatumGetTimestamp(val); - Int64 dt64 - = ((1.0 * t) / USECS_PER_SEC + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)) - * pow(10.0, dt64_col->GetPrecision()); - - dt64_col->Append(dt64); - break; - } - default: - THROW_UNEXPECTED_COLUMN("TIMESTAMP", col); - } - break; - } - case ANYARRAYOID: - { - switch (col->Type()->GetCode()) - { - case Type::Array: - { - auto arrcol = col->AsStrict(); - auto items_type = arrcol->GetType().As()->GetItemType(); - auto arr = (ch_binary_array_t *)DatumGetPointer(val); - auto one_row = build_array_row_column(arr, items_type); - - arrcol->AppendAsColumn(one_row); - break; - } - default: - THROW_UNEXPECTED_COLUMN("array", col); - } - break; - } - case UUIDOID: - { - auto uuid_val = DatumGetUUIDP(val)->data; - auto ch_uuid = UUID{}; - if (!isnull) - { - memcpy(&ch_uuid.first, uuid_val, 8); - memcpy(&ch_uuid.second, uuid_val + 8, 8); - ch_uuid.first = BIG_ENDIAN_64_TO_HOST(ch_uuid.first); - ch_uuid.second = BIG_ENDIAN_64_TO_HOST(ch_uuid.second); - } - col->AsStrict()->Append(ch_uuid); - } - break; - case INETOID: - { - if (isnull) - { - switch (col->Type()->GetCode()) - { - case Type::Code::IPv4: - col->AsStrict()->Append(0); - break; - case Type::Code::IPv6: - col->AsStrict()->Append("::"); - break; - default: - THROW_UNEXPECTED_COLUMN("INET", col); - } - } - else - { - char *s = DatumGetCString(DirectFunctionCall1(inet_out, val)); - switch (col->Type()->GetCode()) - { - case Type::Code::IPv4: - col->AsStrict()->Append(s); - break; - case Type::Code::IPv6: - col->AsStrict()->Append(s); - break; - default: - THROW_UNEXPECTED_COLUMN("INET", col); - } - } - } - break; - case JSONBOID: - { - char *s = DatumGetCString(DirectFunctionCall1(jsonb_out, val)); - col->AsStrict()->Append(s); - } - break; - case JSONOID: - { - char *s = DatumGetCString(DirectFunctionCall1(json_out, val)); - col->AsStrict()->Append(s); - } - break; - default: - { - THROW_UNEXPECTED_COLUMN(format_type_extended(valtype, -1, FORMAT_TYPE_ALLOW_INVALID), col); - } - } - } - - /* - * Append state->values[colidx] to state->insert_block[colidx] or raise an - * exception. - */ - void - ch_binary_column_append_data(ch_binary_insert_state *state, size_t colidx) - { - try - { - auto block = (Block *)state->insert_block; - auto col = (*block)[colidx]; - - Datum val = state->values[colidx]; - - Oid valtype = TupleDescAttr(state->outdesc, colidx)->atttypid; - bool isnull = state->nulls[colidx]; - - column_append(col, val, valtype, isnull); - } - catch (const std::exception &e) - { - elog(ERROR, "pg_clickhouse: could not append data to column - %s", e.what()); - } - } - - void - ch_binary_insert_columns(ch_binary_insert_state *state) - { - Client *client = (Client *)state->conn->client; - auto block = (Block *)state->insert_block; - try - { - block->RefreshRowCount(); - client->SendInsertBlock(*block); - block->Clear(); - } - catch (const std::exception &e) - { - client->ResetConnection(); - delete block; - elog(ERROR, "pg_clickhouse: could not insert columns - %s", e.what()); - } - } - - void - ch_binary_close(ch_binary_connection_t *conn) - { - delete (Client *)conn->client; - delete (ClientOptions *)conn->options; - } - - void - ch_binary_response_free(ch_binary_response_t *resp) - { - if (resp->values) - { - auto values = (std::vector> *)resp->values; - values->clear(); - delete values; - } - - if (resp->error) - free(resp->error); - - delete resp; - } - - void - ch_binary_read_state_init(ch_binary_read_state_t *state, ch_binary_response_t *resp, const ch_query *query) - { - state->resp = resp; - state->block = 0; - state->row = 0; - state->done = false; - state->error = NULL; - state->coltypes = NULL; - state->values = NULL; - state->nulls = NULL; - - /* it response was errored just set error in state too */ - if (resp->error) - { - state->done = true; - set_state_error(state, resp->error); - return; - } - - try - { - assert(resp->values); - auto &values = *((std::vector> *)resp->values); - - if (resp->columns_count && values.size() > 0) - { - state->coltypes = new Oid[resp->columns_count]; - state->values = new Datum[resp->columns_count]; - state->nulls = new bool[resp->columns_count]; - } - } - catch (const std::exception &e) - { - set_state_error(state, e.what()); - return; - } - - /* Initialize coltypes to SELECT types, when provided. */ - if (query->tupdesc) - for (size_t i = 0; i < (size_t)list_length(query->attr_nums); i++) - state->coltypes[i] - = TupleDescAttr(query->tupdesc, lfirst_int(list_nth_cell(query->attr_nums, i)) - 1)->atttypid; - else - for (size_t i = 0; i < resp->columns_count; i++) - state->coltypes[i] = InvalidOid; - } - - /* - * This function prepares values for `convert_datum` which is called in - * upper code. - * - * This function calls postgres functions, which can call `palloc` so we can end up - * with elog(ERROR) and longjmp to upper postgres code with leaking c++ memory. - * - * There is not an adequate (without huge overheads) solution, we just consider - * this state unfixable. - * - * Expects `valtype` to already be set to the type of the foreign table - * column for the value, but it may replace it with another type that will - * later be cast to valtype. - * - * Always sets `is_null`. - */ - static Datum - make_datum(clickhouse::ColumnRef col, size_t row, Oid *valtype, bool *is_null) - { - Datum ret = (Datum)0; - - nested_col: - auto type_code = col->Type()->GetCode(); - - *is_null = false; - - switch (type_code) - { - case Type::Code::UInt8: - { - int16 val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT2OID; - } - break; - case Type::Code::UInt16: - { - int16 val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT4OID; - } - break; - case Type::Code::UInt32: - { - int64 val = col->AsStrict()->At(row); - ret = Int64GetDatum(val); - *valtype = INT8OID; - } - break; - case Type::Code::UInt64: - { - uint64 val = col->AsStrict()->At(row); - /* XXX Consider using, e.g., https://pgxn.org/dist/uint128. */ - if (val > LONG_MAX) - throw std::overflow_error("value " + std::to_string(val) + " is out of range of bigint"); - - ret = Int64GetDatum((int64)val); - *valtype = INT8OID; - } - break; - case Type::Code::Int8: - { - int16 val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT2OID; - } - break; - case Type::Code::Int16: - { - int16 val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT2OID; - } - break; - case Type::Code::Int32: - { - int val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT4OID; - } - break; - case Type::Code::Int64: - { - int64 val = col->AsStrict()->At(row); - ret = Int64GetDatum(val); - *valtype = INT8OID; - } - break; - case Type::Code::Float32: - { - float val = col->AsStrict()->At(row); - ret = Float4GetDatum(val); - *valtype = FLOAT4OID; - } - break; - case Type::Code::Float64: - { - double val = col->AsStrict()->At(row); - ret = Float8GetDatum(val); - *valtype = FLOAT8OID; - } - break; - case Type::Code::Decimal128: - case Type::Code::Decimal64: - case Type::Code::Decimal32: - case Type::Code::Decimal: - { - auto decCol = col->AsStrict(); - auto val = decCol->At(row); - - /* Convert the Int128 to a string. */ - std::stringstream ss; - ss << val; - std::string str = ss.str(); - - /* Start a destination string. */ - std::stringstream res; - auto scale = decCol->GetScale(); - - /* Output a dash for negative values */ - if (val < 0) - { - res << '-'; - str.erase(0, 1); - } - - if (scale == 0) - { - /* No decimal point, just output the entire value. */ - res << str; - } - else if (str.length() <= scale) - { - /* Append the entire value prepended with zeros after the decimal. */ - res << "0." << std::string(scale - str.length(), '0') << str; - } - else - { - /* There are digits before the decimal. */ - auto decAt = str.length() - scale; - res << str.substr(0, decAt); - - /* Append any digits after the decimal. */ - if (decAt < str.length()) - { - res << '.' << str.substr(decAt); - } - } - - ret = DirectFunctionCall3(numeric_in, CStringGetDatum(res.str().c_str()), ObjectIdGetDatum(0), - Int32GetDatum(-1)); - *valtype = NUMERICOID; - } - break; - case Type::Code::FixedString: - { - auto s = std::string(col->AsStrict()->At(row)); - /* ClickHouse allows nuls in strings, so copy by full length. */ - ret = PointerGetDatum(cstring_to_text_with_len(s.data(), s.size())); - *valtype = TEXTOID; - } - break; - case Type::Code::String: - { - auto s = std::string(col->AsStrict()->At(row)); - /* ClickHouse allows nuls in strings, so copy by full length. */ - ret = PointerGetDatum(cstring_to_text_with_len(s.data(), s.size())); - *valtype = TEXTOID; - } - break; - case Type::Code::Enum8: - { - auto s = std::string(col->AsStrict()->NameAt(row)); - ret = CStringGetTextDatum(s.c_str()); - *valtype = TEXTOID; - } - break; - case Type::Code::Enum16: - { - auto s = std::string(col->AsStrict()->NameAt(row)); - ret = CStringGetTextDatum(s.c_str()); - *valtype = TEXTOID; - } - break; - case Type::Code::Date: - { - auto val = static_cast(col->AsStrict()->At(row)); - *valtype = DATEOID; - ret = DirectFunctionCall1(timestamp_date, time_t_to_timestamptz(val)); - } - break; - case Type::Code::Date32: - { - auto val = static_cast(col->AsStrict()->At(row)); - *valtype = DATEOID; - ret = DirectFunctionCall1(timestamp_date, time_t_to_timestamptz(val)); - } - break; - case Type::Code::DateTime: - { - auto val = static_cast(col->AsStrict()->At(row)); - *valtype = TIMESTAMPTZOID; - ret = TimestampTzGetDatum(time_t_to_timestamptz(val)); - } - break; - case Type::Code::DateTime64: - { - auto dt_col = col->AsStrict(); - auto val = dt_col->At(row); - int64 power = pow(10, dt_col->GetPrecision()); - *valtype = TIMESTAMPTZOID; - ret = TimestampTzGetDatum(time_t_to_timestamptz(val / power)) + (val % power) * (USECS_PER_SEC / power); - } - break; - case Type::Code::UUID: - { - /* we form char[16] from two uint64 numbers, and they should - * be big endian */ - auto val = col->AsStrict()->At(row); - pg_uuid_t *uuid_val = (pg_uuid_t *)exc_palloc(sizeof(pg_uuid_t)); - - val.first = HOST_TO_BIG_ENDIAN_64(val.first); - val.second = HOST_TO_BIG_ENDIAN_64(val.second); - memcpy(uuid_val->data, &val.first, 8); - memcpy(uuid_val->data + 8, &val.second, 8); - - ret = UUIDPGetDatum(uuid_val); - *valtype = UUIDOID; - } - break; - case Type::Code::Nullable: - { - auto nullable = col->AsStrict(); - if (nullable->IsNull(row)) - { - *is_null = true; - } - else - { - col = nullable->Nested(); - goto nested_col; - } - } - break; - case Type::Code::Array: - { - auto arr = col->AsStrict()->GetAsColumn(row); - size_t len = arr->Size(); - auto slot = (ch_binary_array_t *)exc_palloc(sizeof(ch_binary_array_t)); - - /* find leaf scalar type & nesting depth, since postgres has one - * array type per element type regardless of ndim */ - int ndim = 1; - auto leaf_type = arr->Type(); - while (leaf_type->GetCode() == Type::Code::Array) - { - leaf_type = leaf_type->As()->GetItemType(); - ndim++; - } - - Oid item_type = get_corr_postgres_type(leaf_type, get_element_type(*valtype)); - Oid array_type = get_array_type(item_type); - - if (array_type == InvalidOid) - throw std::runtime_error(std::string("pg_clickhouse: could not") + " find array type for " - + std::to_string(item_type)); - - slot->len = len; - slot->ndim = ndim; - slot->array_type = array_type; - slot->item_type = item_type; - - if (len > 0) - { - slot->datums = (Datum *)exc_palloc0(sizeof(Datum) * len); - slot->nulls = (bool *)exc_palloc0(sizeof(bool) * len); - - /* For ndim==1 inner make_datum returns leaf scalars; for - * ndim>1 it recurses into the Array branch and produces - * nested ch_binary_array_t* values. Use a scratch valtype - * to avoid clobbering slot->item_type. */ - Oid scratch; - for (size_t i = 0; i < len; ++i) - slot->datums[i] = make_datum(arr, i, &scratch, &slot->nulls[i]); - } - - /* this one will need additional work, since we just return raw slot */ - ret = PointerGetDatum(slot); - *valtype = ANYARRAYOID; - } - break; - case Type::Code::Tuple: - { - auto tuple = col->AsStrict(); - auto len = tuple->TupleSize(); - - if (len == 0) - throw std::runtime_error("pg_clickhouse: returned tuple is empty"); - - auto slot = (ch_binary_tuple_t *)exc_palloc(sizeof(ch_binary_tuple_t)); - - slot->datums = (Datum *)exc_palloc(sizeof(Datum) * len); - slot->nulls = (bool *)exc_palloc0(sizeof(bool) * len); - slot->types = (Oid *)exc_palloc0(sizeof(Oid) * len); - slot->len = len; - - for (size_t i = 0; i < len; ++i) - slot->datums[i] = make_datum((*tuple)[i], row, &slot->types[i], &slot->nulls[i]); - - /* this one will need additional work, since we just return raw slot */ - ret = PointerGetDatum(slot); - *valtype = RECORDOID; - } - break; - case Type::Code::LowCardinality: - { - auto item = col->AsStrict()->GetItem(row); - auto data = item.AsBinaryData(); - ret = PointerGetDatum(cstring_to_text_with_len(data.data(), data.size())); - *valtype = TEXTOID; - } - break; - case Type::Code::IPv4: - { - auto item = col->AsStrict()->AsString(row); - ret = DirectFunctionCall1(inet_in, CStringGetDatum(item.c_str())); - *valtype = INETOID; - } - break; - case Type::Code::IPv6: - { - auto item = col->AsStrict()->AsString(row); - ret = DirectFunctionCall1(inet_in, CStringGetDatum(item.c_str())); - *valtype = INETOID; - } - break; - case Type::Code::JSON: - { - if (!*valtype) - *valtype = JSONBOID; - /* Sadly must copy to a null-terminated string as Postgres expects. */ - auto item = std::string(col->AsStrict()->At(row)); - ret = DirectFunctionCall1(*valtype == JSONBOID ? jsonb_in : json_in, CStringGetDatum(item.c_str())); - } - break; - default: - throw std::runtime_error("unsupported type " + std::string(Type::TypeName(type_code)) - + " in binary protocol"); - } - - return ret; - } - - /* - * Read a row from state->block and fill state->values with the - * corresponding values. - */ - bool - ch_binary_read_row(ch_binary_read_state_t *state, TupleDesc tupdesc, List *attrs) - { - /* coltypes is NULL means there are no blocks */ - bool res = false; - - if (state->done || state->coltypes == NULL || state->error) - return false; - - assert(state->resp->values); - auto &values = *((std::vector> *)state->resp->values); - try - { - again: - assert(state->block < state->resp->blocks_count); - auto &block = values[state->block]; - size_t row_count = block[0]->Size(); - - if (row_count == 0) - goto next_row; - - for (size_t i = 0; i < state->resp->columns_count; i++) - { - /* fill value and null arrays */ - state->values[i] = make_datum(block[i], state->row, &state->coltypes[i], &state->nulls[i]); - } - res = true; - - next_row: - state->row++; - if (state->row >= row_count) - { - state->row = 0; - state->block++; - if (state->block >= state->resp->blocks_count) - state->done = true; - else if (row_count == 0) - goto again; - } - } - catch (const std::exception &e) - { - set_state_error(state, e.what()); - } - - return res; - } - - void - ch_binary_read_state_free(ch_binary_read_state_t *state) - { - if (state->coltypes) - { - delete[] state->coltypes; - delete[] state->values; - delete[] state->nulls; - } - - if (state->error) - free(state->error); - } -} diff --git a/src/binary/binary.c b/src/binary/binary.c new file mode 100644 index 00000000..a540e6c4 --- /dev/null +++ b/src/binary/binary.c @@ -0,0 +1,94 @@ +/* + * binary.c + * + * Core glue for the binary driver. Owns the chc_alloc thunks (palloc on + * CurrentMemoryContext, MCXT_ALLOC_HUGE so block buffers can escape the 1GB + * cap), error mapping into ereport, and a couple of small type helpers shared + * across the subdir. + * + * This also holds the CHC_IMPLEMENTATION define for the clickhouse-c + * header-only library; the other .c files in src/binary include the + * same headers without that define and link against the bodies emitted + * here. + * + * API lives in src/include/binary.h. Driver internals shared between + * connection.c / select.c / insert.c live in binary_internal.h + * alongside this file. + */ + +#include "postgres.h" + +#include "utils/palloc.h" + +#define CHC_IMPLEMENTATION +#include "clickhouse.h" +#include "clickhouse-compression.h" +#include "clickhouse-posix-io.h" +#include "clickhouse-openssl.h" +#include "clickhouse-client.h" + +#include "binary_internal.h" + +static void * +pg_chc_palloc(void *ud pg_attribute_unused(), size_t n) +{ + return MemoryContextAllocExtended(CurrentMemoryContext, n, MCXT_ALLOC_HUGE); +} + +static void * +pg_chc_repalloc(void *ud pg_attribute_unused(), void *p, + size_t old_bytes pg_attribute_unused(), size_t new_bytes) +{ + if (!p) + return MemoryContextAllocExtended(CurrentMemoryContext, new_bytes, MCXT_ALLOC_HUGE); + return repalloc_huge(p, new_bytes); +} + +static void +pg_chc_pfree(void *ud pg_attribute_unused(), void *p, size_t bytes pg_attribute_unused()) +{ + if (p) + pfree(p); +} + +const chc_alloc pg_chc_alloc = { + .ud = NULL, + .alloc = pg_chc_palloc, + .realloc = pg_chc_repalloc, + .free = pg_chc_pfree, +}; + +/* power-of-10 lookup; CH bounds DateTime64 / Decimal scale to [0, 9] */ +const int64_t pow10i[10] = { + 1, 10, 100, 1000, 10000, 100000, 1000000, + 10000000, 100000000, 1000000000 +}; + +/* + * output_format_native_write_json_as_string exists on the server from + * 24.10 onwards. Sending it as `important` against an older server would + * fail the query, so gate. + */ +bool +server_supports_json_as_string(const chc_client * c) +{ + const chc_server_info *info = chc_client_server_info(c); + + if (!info) + return false; + if (info->version_major > 24) + return true; + if (info->version_major == 24 && info->version_minor >= 10) + return true; + return false; +} + +void +raise_chc(const chc_err * err, int sqlstate, const char *prefix) +{ + const char *m = (err && err->msg[0]) ? err->msg : "unknown error"; + + ereport(ERROR, + (errcode(sqlstate), + errmsg("pg_clickhouse: %s%s", prefix ? prefix : "", m))); +} diff --git a/src/binary/binary_internal.h b/src/binary/binary_internal.h new file mode 100644 index 00000000..42e7cfba --- /dev/null +++ b/src/binary/binary_internal.h @@ -0,0 +1,210 @@ +/* + * binary_internal.h + * + * Cross-file shared state for the binary driver subdir. Not exposed via + * src/include, anything outside src/binary should use binary.h. + */ + +#ifndef PG_CLICKHOUSE_BINARY_INTERNAL_H +#define PG_CLICKHOUSE_BINARY_INTERNAL_H + +#include "postgres.h" + +#include + +#include "clickhouse.h" +#include "clickhouse-client.h" +#include "clickhouse-posix-io.h" +#include "clickhouse-openssl.h" + +#include "binary.h" +#include "internal.h" + +#if PG_VERSION_NUM < 180000 +#define pg_noreturn pg_attribute_noreturn() +#endif + +typedef struct +{ + Datum *datums; + bool *nulls; + size_t len; + Oid *types; + const char *ch_type_name; +} ch_binary_tuple_t; + +/* + * Holds an array decoded from ClickHouse or built for INSERT. For nested + * arrays (Array(Array(...))) ndim > 1 and datums[i] points to a child + * ch_binary_array_t with ndim-1. item_type is leaf scalar PG type, + * array_type is postgres array type (same across nesting depths since + * postgres uses one array type per element type regardless of ndim). + */ +typedef struct +{ + Datum *datums; + bool *nulls; + size_t len; + int ndim; /* nesting depth, >=1 */ + Oid item_type; /* leaf scalar PG type */ + Oid array_type; /* PG array type (same at every level) */ +} ch_binary_array_t; + +/* Column metadata returned from ch_binary_begin_insert. */ +typedef struct ch_binary_column_info +{ + const char *name; + const chc_type *type; /* type unwrapped of Nullable + LowCardinality */ + bool is_nullable; +} ch_binary_column_info; + +/* + * Pump next non-empty Data block off wire. Returned pointer is borrowed, + * valid until next fetch_next_block or ch_binary_response_free. NULL when + * stream ends (eos, error, canceled), ch_binary_response_error reports + * cause if any. + */ +extern const chc_block *ch_binary_response_fetch_next_block(ch_binary_response_t * resp); + +extern ch_binary_insert_handle * ch_binary_begin_insert(ch_binary_connection_t * conn, + const ch_query * query, + ch_binary_column_info * *out_cols, + size_t * out_n); + +/* + * Tear down handle. Never raises and never talks to server, safe to call + * from a MemoryContext reset callback during transaction abort. Flags + * connection broken if finalize did not run. + */ +extern void ch_binary_release_insert(ch_binary_insert_handle * h); + +/* + * Per-connection state smuggled through ch_binary_connection_t.client. + * + * Lives in own context `cxt` (child of CacheMemoryContext) so connection + * survives transaction boundaries. Result block buffers don't live here: + * pg_chc_alloc routes through CurrentMemoryContext, so blocks land in + * whichever per-query context the caller has switched to. + */ +struct ch_binary_state +{ + /* + * Connection-lifetime context; holds this struct, the chc_client, and + * chc_client's initial read buffer. Deleted by ch_binary_close. + */ + MemoryContext cxt; + + /* + * Registered on cxt; closes fd / SSL on reset so half-built connections + * release OS resources when PG_CATCH deletes cxt. + */ + MemoryContextCallback reset_cb; + + chc_client *client; + /* Transport vtable; backed by posix_state or openssl_state below. */ + chc_io io; + + int fd; /* -1 once closed by reset_cb */ + SSL_CTX *ssl_ctx; + SSL *ssl; + bool tls; + chc_posix_io posix_state; + chc_openssl_io openssl_state; + + /* Per-query cancel callback (no userdata). */ + bool (*check_cancel_fn) (void); + + /* + * Set when an unrecoverable protocol/IO error happened (server raised an + * exception mid-INSERT and closed the socket, write hit EPIPE, etc). + * Cache layer checks via ch_binary_is_broken & drops the entry. + */ + bool broken; +}; + +static inline struct ch_binary_state * +conn_state(ch_binary_connection_t * conn) +{ + return (struct ch_binary_state *) conn->client; +} + +/* chc allocator wired through palloc; defined in binary.c. */ +extern const chc_alloc pg_chc_alloc; + +/* power-of-10 lookup; CH bounds DateTime64 / Decimal scale to [0, 9] */ +extern const int64_t pow10i[10]; + +/* ereport ERROR carrying chc_err->msg with sqlstate / prefix. */ +pg_noreturn extern void raise_chc(const chc_err * err, int sqlstate, + const char *prefix); + +/* True if server advertises output_format_native_write_json_as_string. */ +extern bool server_supports_json_as_string(const chc_client * c); + +/* + * Per-row, per-column append. Set isnull for NULL values (column must be + * Nullable). ereports on type mismatch / NULL-into-NOT-NULL. + */ +extern void ch_binary_append_int(ch_binary_insert_handle * h, size_t col, + int64_t val, bool isnull); +extern void ch_binary_append_uint(ch_binary_insert_handle * h, size_t col, + uint64_t val, bool isnull); +extern void ch_binary_append_bool(ch_binary_insert_handle * h, size_t col, + bool val, bool isnull); +extern void ch_binary_append_double(ch_binary_insert_handle * h, size_t col, + double val, bool isnull); +extern void ch_binary_append_float(ch_binary_insert_handle * h, size_t col, + float val, bool isnull); +extern void ch_binary_append_bytes(ch_binary_insert_handle * h, size_t col, + const void *p, size_t n, bool isnull); +extern void ch_binary_append_decimal(ch_binary_insert_handle * h, size_t col, + const char *digits, bool isnull); +extern void ch_binary_append_uuid(ch_binary_insert_handle * h, size_t col, + const uint8_t bytes[16], bool isnull); + +/* + * IPv4: addr_be is 4 BE bytes (matches PG inet ip_addr layout). + * IPv6: addr_be is 16 BE bytes. Pass NULL with isnull=true. + */ +extern void ch_binary_append_inet(ch_binary_insert_handle * h, size_t col, + const uint8_t * addr_be, size_t addrlen, + bool isnull); + +/* + * Per-row Date/DateTime/DateTime64 sent as seconds-since-epoch (int64). + * For DateTime64 value is wire-level integer at column's scale; + * encode.c does scaling. + */ +extern void ch_binary_append_date_seconds(ch_binary_insert_handle * h, size_t col, + int64_t seconds, bool isnull); +extern void ch_binary_append_datetime_seconds(ch_binary_insert_handle * h, size_t col, + int64_t seconds, bool isnull); +extern void ch_binary_append_datetime64_raw(ch_binary_insert_handle * h, size_t col, + int64_t raw, bool isnull); + +/* + * Array element append. Open with array_begin. All subsequent + * ch_binary_append_* calls target inner items column regardless of + * `col` until ch_binary_array_end. + */ +extern void ch_binary_array_begin(ch_binary_insert_handle * h, size_t col); +extern void ch_binary_array_end(ch_binary_insert_handle * h); + +/* True when handle has an active array context (for assertions). */ +extern bool ch_binary_array_active(const ch_binary_insert_handle * h); + +/* + * Inspect underlying CH column kind. Used by encode.c to + * dispatch on (Oid pg, chc_kind ch). When an array context is open + * (between ch_binary_array_begin/_end) returned kind is element kind, + * not CHC_ARRAY. + */ +extern chc_kind ch_binary_column_kind(const ch_binary_insert_handle * h, + size_t col); +extern uint32_t ch_binary_column_datetime64_precision(const ch_binary_insert_handle * h, + size_t col); + +/* Send buffered rows and clear; ready for next batch. */ +extern void ch_binary_flush_block(ch_binary_insert_handle * h); + +#endif /* PG_CLICKHOUSE_BINARY_INTERNAL_H */ diff --git a/src/binary/connection.c b/src/binary/connection.c new file mode 100644 index 00000000..566049f4 --- /dev/null +++ b/src/binary/connection.c @@ -0,0 +1,261 @@ +/* + * connection.c + * + * Open / probe / close for the binary driver. TCP + optional TLS, then + * chc_client_init handshakes the protocol. Cleanup runs through a + * MemoryContext reset callback so half-built state still releases the + * fd / SSL pair on transaction abort. + */ + +#include "postgres.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/memutils.h" +#include "utils/palloc.h" + +#include +#include + +#include "binary_internal.h" +#include "engine.h" + +#define CLICKHOUSE_SECURE_PORT 9440 + +static bool +cancel_adapter(void *ud) +{ + struct ch_binary_state *s = ud; + + return s->check_cancel_fn ? s->check_cancel_fn() : false; +} + +/* + * Releases OS resources owned by the connection. The chc_client and its + * read buffer live in s->cxt and are freed by the surrounding + * MemoryContextDelete; only fd / SSL need an explicit close. + */ +static void +binary_state_reset_cb(void *arg) +{ + struct ch_binary_state *s = arg; + + if (s->client) + { + chc_client_close(s->client); + s->client = NULL; + } + if (s->ssl) + { + SSL_shutdown(s->ssl); + SSL_free(s->ssl); + s->ssl = NULL; + } + if (s->ssl_ctx) + { + SSL_CTX_free(s->ssl_ctx); + s->ssl_ctx = NULL; + } + if (s->fd >= 0) + { + close(s->fd); + s->fd = -1; + } +} + +static int +tcp_connect(const char *host, int port) +{ + struct addrinfo hints = {}; + struct addrinfo *res = NULL; + char port_s[16]; + + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + snprintf(port_s, sizeof(port_s), "%d", port); + int rc = getaddrinfo(host, port_s, &hints, &res); + + if (rc != 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: getaddrinfo(%s:%d): %s", + host, port, gai_strerror(rc)))); + + int fd = -1; + int save_errno = ECONNREFUSED; + + for (struct addrinfo *ai = res; ai; ai = ai->ai_next) + { + fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) + { + save_errno = errno; + continue; + } + if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) + break; + save_errno = errno; + close(fd); + fd = -1; + } + freeaddrinfo(res); + if (fd < 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: connect(%s:%d): %s", + host, port, strerror(save_errno)))); + + int one = 1; + + (void) setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + (void) setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(one)); + return fd; +} + +static void +tls_connect(struct ch_binary_state *s, const char *host) +{ + static int openssl_init_done = 0; + + if (!openssl_init_done) + { + SSL_library_init(); + SSL_load_error_strings(); + OpenSSL_add_all_algorithms(); + openssl_init_done = 1; + } + + s->ssl_ctx = SSL_CTX_new(TLS_client_method()); + if (!s->ssl_ctx) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize opensssl"), + errdetail("SSL_CTX_new failed"))); + SSL_CTX_set_verify(s->ssl_ctx, SSL_VERIFY_NONE, NULL); + + s->ssl = SSL_new(s->ssl_ctx); + if (!s->ssl) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize opensssl"), + errdetail("SSL_new failed"))); + SSL_set_tlsext_host_name(s->ssl, host); + SSL_set_fd(s->ssl, s->fd); + if (SSL_connect(s->ssl) != 1) + { + unsigned long e = ERR_peek_last_error(); + char ebuf[160]; + + ERR_error_string_n(e, ebuf, sizeof(ebuf)); + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: openssl failed to connect"), + errdetail("%s", ebuf))); + } +} + +ch_binary_connection_t * +ch_binary_connect(ch_connection_details * details) +{ + const char *host = details->host ? details->host : "127.0.0.1"; + int port = details->port; + bool tls = false; + + if (port == 0) + { + if (ch_is_cloud_host(host)) + { + port = CLICKHOUSE_SECURE_PORT; + tls = true; + } + else + port = 9000; + } + else if (port == CLICKHOUSE_SECURE_PORT) + tls = true; + + MemoryContext cxt = AllocSetContextCreate(CacheMemoryContext, + "pg_clickhouse binary connection", + ALLOCSET_SMALL_SIZES); + MemoryContext old = MemoryContextSwitchTo(cxt); + struct ch_binary_state *s; + ch_binary_connection_t *conn; + + PG_TRY(); + { + s = palloc0(sizeof(*s)); + conn = palloc0(sizeof(*conn)); + s->cxt = cxt; + s->fd = -1; + s->tls = tls; + s->reset_cb.func = binary_state_reset_cb; + s->reset_cb.arg = s; + MemoryContextRegisterResetCallback(cxt, &s->reset_cb); + conn->client = s; + + s->fd = tcp_connect(host, port); + if (tls) + { + tls_connect(s, host); + chc_openssl_io_init(&s->openssl_state, &s->io, s->ssl, + cancel_adapter, s); + } + else + { + chc_posix_io_init(&s->posix_state, &s->io, s->fd, + cancel_adapter, s); + } + + chc_client_opts opts = { + .database = details->dbname ? details->dbname : "default", + .user = details->username ? details->username : "default", + .password = details->password ? details->password : "", + }; + chc_err err = {}; + int rc = chc_client_init(&s->client, &opts, &pg_chc_alloc, &s->io, &err); + + if (rc != CHC_OK) + raise_chc(&err, ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION, + "connection error: "); + } + PG_CATCH(); + { + MemoryContextSwitchTo(old); + MemoryContextDelete(cxt); + PG_RE_THROW(); + } + PG_END_TRY(); + + MemoryContextSwitchTo(old); + return conn; +} + +bool +ch_binary_is_broken(const ch_binary_connection_t * conn) +{ + if (!conn) + return false; + const struct ch_binary_state *s = (const struct ch_binary_state *) conn->client; + + return s ? s->broken : false; +} + +void +ch_binary_close(ch_binary_connection_t * conn) +{ + if (!conn) + return; + struct ch_binary_state *s = conn_state(conn); + + if (s) + MemoryContextDelete(s->cxt); + /* conn itself was palloc'd inside s->cxt; freed by the delete */ +} diff --git a/src/convert.c b/src/binary/convert.c similarity index 94% rename from src/convert.c rename to src/binary/convert.c index c056447b..e277c5e3 100644 --- a/src/convert.c +++ b/src/binary/convert.c @@ -1,6 +1,4 @@ -/* - * Conversion functions for the binary driver. -*/ +/* Conversion functions for the binary driver. */ #include "postgres.h" @@ -19,7 +17,7 @@ #include "executor/tuptable.h" #include "fdw.h" -#include "binary.hh" +#include "binary_internal.h" #include typedef struct ch_convert_state ch_convert_state; @@ -232,8 +230,8 @@ convert_array(ch_convert_state * state, Datum val) } else { - int dims[MAXDIM]; - int lbs[MAXDIM]; + int dims[MAXDIM] = {}; + int lbs[MAXDIM] = {}; size_t total = 1; size_t idx = 0; Datum *flat; @@ -319,12 +317,6 @@ convert_bool(ch_convert_state * state, Datum val) return BoolGetDatum(dat); } -inline static Datum -convert_bool_to_int16(ch_convert_output_state * state, Datum val) -{ - return Int16GetDatum(DatumGetBool(val) ? 1 : 0); -} - Datum ch_binary_convert_datum(void *state, Datum val) { @@ -416,7 +408,8 @@ ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype) if (typentry->tupDesc == NULL) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("type %s is not composite", + errmsg("pg_clickhouse: cannot return %s as %s", + slot->ch_type_name ? slot->ch_type_name : "?", format_type_be(outtype)))); tupdesc = typentry->tupDesc; @@ -514,21 +507,6 @@ init_output_convert_state(ch_convert_output_state * state) if (state->intype == BYTEAOID && state->outtype == TEXTOID) return; - /* column_append() handles both JSON and JSONB, so no need to convert. */ - if ( - (state->intype == JSONBOID && state->outtype == JSONOID) - || (state->intype == JSONOID && state->outtype == JSONBOID) - ) - return; - - /* Postgres has no cast from bool to INT16, so provide our own. */ - if (state->outtype == INT2OID && state->intype == BOOLOID) - { - state->func = convert_bool_to_int16; - state->ctype = COERCION_PATH_FUNC; - return; - } - state->func = convert_out_generic; state->ctype = find_coercion_pathway(state->outtype, state->intype, @@ -672,10 +650,10 @@ build_nested_binary_array(int level, int ndim, int *dims, Oid item_type, } /* - * For each value to be output, convert it, if necessary, from the Postgres - * Datum type defined for the foreign table to a Datum that column_append() in - * binary.cpp knows how to convert to a ClickHouse type. No conversion for - * binary-compatible types; other types require a CAST. + * For each output value, convert it, if necessary, from the Postgres Datum + * type defined for the foreign table to a Datum that the binary INSERT + * path in encode.c knows how to convert to a ClickHouse type. No conversion + * for binary-compatible types; other types require a CAST. * ch_binary_make_tuple_map() makes this determination for each type, stored * in insert_state->conversion_states) */ diff --git a/src/binary/decode.c b/src/binary/decode.c new file mode 100644 index 00000000..6f24c0c2 --- /dev/null +++ b/src/binary/decode.c @@ -0,0 +1,888 @@ +/* + * decode.c + * + * Build PG Datums from a chc_column produced by clickhouse-c. Walks the + * wire-shaped column accessors directly; per-row transforms (decimal to + * text, IPv4/IPv6 text, UUID byteswap, enum name lookup, LowCardinality + * key->dict deref, Nullable strip) happen inline at read time. + */ + +#include "postgres.h" + +#include +#include /* AF_INET, expanded by PG inet macros */ + +#include "catalog/pg_type_d.h" +#include "fmgr.h" +#include "port/pg_bswap.h" +#include "utils/builtins.h" +#include "utils/date.h" +#include "utils/inet.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/numeric.h" +#include "utils/timestamp.h" +#include "utils/uuid.h" + +#include "binary_internal.h" + +/* CH Date / Date32 epoch is unix; offset to PG epoch (2000-01-01) */ +#define CH_TO_PG_DATE_OFFSET (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) + +/* Little-endian fixed-width reads at row offset. */ +static inline int8_t +rd_i8(const uint8_t * p, uint64_t row) +{ + return (int8_t) p[row]; +} + +static inline uint8_t +rd_u8(const uint8_t * p, uint64_t row) +{ + return p[row]; +} + +static inline bool +rd_bool(const bool *p, uint64_t row) +{ + return (bool) p[row]; +} + +static inline int16_t +rd_i16(const uint8_t * p, uint64_t row) +{ + int16_t v; + + memcpy(&v, p + row * sizeof(int16_t), sizeof(int16_t)); + return v; +} + +static inline uint16_t +rd_u16(const uint8_t * p, uint64_t row) +{ + uint16_t v; + + memcpy(&v, p + row * sizeof(uint16_t), sizeof(uint16_t)); + return v; +} + +static inline int32_t +rd_i32(const uint8_t * p, uint64_t row) +{ + int32_t v; + + memcpy(&v, p + row * sizeof(int32_t), sizeof(int32_t)); + return v; +} + +static inline uint32_t +rd_u32(const uint8_t * p, uint64_t row) +{ + uint32_t v; + + memcpy(&v, p + row * sizeof(uint32_t), sizeof(uint32_t)); + return v; +} + +static inline int64_t +rd_i64(const uint8_t * p, uint64_t row) +{ + int64_t v; + + memcpy(&v, p + row * sizeof(int64_t), sizeof(int64_t)); + return v; +} + +static inline uint64_t +rd_u64(const uint8_t * p, uint64_t row) +{ + uint64_t v; + + memcpy(&v, p + row * sizeof(uint64_t), sizeof(uint64_t)); + return v; +} + +static inline float +rd_f32(const uint8_t * p, uint64_t row) +{ + float v; + + memcpy(&v, p + row * sizeof(float), sizeof(float)); + return v; +} + +static inline double +rd_f64(const uint8_t * p, uint64_t row) +{ + double v; + + memcpy(&v, p + row * sizeof(double), sizeof(double)); + return v; +} + +static inline void +slice_str(const chc_column * col, uint64_t row, + const char **out_ptr, size_t * out_len) +{ + const uint64_t *offs = chc_column_string_offsets(col); + const uint8_t *data = chc_column_string_data(col); + uint64_t start = row == 0 ? 0 : offs[row - 1]; + uint64_t end = offs[row]; + + *out_ptr = (const char *) data + start; + *out_len = (size_t) (end - start); +} + +static Oid +ch_kind_to_pg_oid(const chc_type * type) +{ + switch (chc_type_kind(type)) + { + case CHC_VOID: + case CHC_NOTHING: + return InvalidOid; + case CHC_NULLABLE: + case CHC_LOW_CARDINALITY: + return ch_kind_to_pg_oid(chc_type_child(type, 0)); + case CHC_INT8: + case CHC_INT16: + case CHC_UINT8: + return INT2OID; + case CHC_BOOL: + return BOOLOID; + case CHC_INT32: + case CHC_UINT16: + return INT4OID; + case CHC_INT64: + case CHC_UINT32: + case CHC_UINT64: + return INT8OID; + case CHC_FLOAT32: + return FLOAT4OID; + case CHC_FLOAT64: + return FLOAT8OID; + case CHC_DECIMAL32: + case CHC_DECIMAL64: + case CHC_DECIMAL128: + case CHC_DECIMAL256: + return NUMERICOID; + case CHC_STRING: + case CHC_FIXED_STRING: + case CHC_ENUM8: + case CHC_ENUM16: + return TEXTOID; + case CHC_JSON: + case CHC_OBJECT: + return JSONBOID; + case CHC_DATE: + case CHC_DATE32: + return DATEOID; + case CHC_DATETIME: + case CHC_DATETIME64: + return TIMESTAMPTZOID; + case CHC_UUID: + return UUIDOID; + case CHC_IPV4: + case CHC_IPV6: + return INETOID; + case CHC_ARRAY: + return ANYARRAYOID; + case CHC_TUPLE: + return RECORDOID; + default: + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported column type"))); + } + /* unreachable */ + return InvalidOid; +} + +static Datum read_value(const chc_column * col, const chc_type * type, + uint64_t row, Oid * valtype, bool *is_null); + +/* + * Format a ClickHouse Decimal (two's-complement signed integer in LE bytes of + * width 4/8/16/32 for Decimal32/64/128/256, with `scale` fractional digits + * carried on the column type) into `out`. Returns bytes written, -1 on overflow. + */ +static int +format_decimal_text(const uint8_t * bytes, size_t width, uint32_t scale, + char *out, size_t out_cap) +{ + uint32_t mag[8]; + size_t nwords = width / 4; + bool neg = false; + + if (width == 0 || width > 32 || width % 4 != 0) + return -1; + memcpy(mag, bytes, width); + /* top bit of MSW is sign; negate two's-complement to get magnitude */ + if (mag[nwords - 1] & 0x80000000u) + { + neg = true; + for (size_t i = 0; i < nwords; i++) + mag[i] = ~mag[i]; + uint64_t carry = 1; + + for (size_t i = 0; i < nwords && carry; i++) + { + uint64_t v = (uint64_t) mag[i] + carry; + + mag[i] = (uint32_t) v; + carry = v >> 32; + } + } + + char buf[80]; + int n = 0; + bool nonzero; + + /* base-10 division of mag yields digits LSB-first */ + do + { + uint64_t rem = 0; + + nonzero = false; + for (ssize_t i = (ssize_t) nwords - 1; i >= 0; i--) + { + uint64_t v = (rem << 32) | mag[i]; + + mag[i] = (uint32_t) (v / 10); + rem = v % 10; + if (mag[i]) + nonzero = true; + } + buf[n++] = (char) ('0' + (uint32_t) rem); + } while (nonzero && n < (int) sizeof(buf)); + + /* pad leading zeros so digit count covers fractional portion */ + while (n <= (int) scale) + buf[n++] = '0'; + + size_t need = (size_t) neg + (size_t) n + (scale ? 1 : 0) + 1; + + if (need > out_cap) + return -1; + char *p = out; + + if (neg) + *p++ = '-'; + + /* emit MSD-first, inserting '.' before `scale` trailing digits */ + for (int i = n - 1; i >= 0; i--) + { + if (i + 1 == scale) + *p++ = '.'; + *p++ = buf[i]; + } + *p = '\0'; + return (int) (p - out); +} + +static Datum +read_decimal(const chc_column * col, const chc_type * type, uint64_t row) +{ + size_t es; + const uint8_t *p = (const uint8_t *) chc_column_fixed_data(col, &es); + uint32_t scale = (uint32_t) chc_type_decimal_scale(type); + char buf[80]; + int rc; + +#if PG_VERSION_NUM >= 140000 + /* Decimal32/64 fit in int64; skip the byte-array text path. */ + if (es == 4) + return NumericGetDatum(int64_div_fast_to_numeric(rd_i32(p, row), (int) scale)); + if (es == 8) + return NumericGetDatum(int64_div_fast_to_numeric(rd_i64(p, row), (int) scale)); +#endif + + rc = format_decimal_text(p + row * es, es, scale, buf, sizeof(buf)); + if (rc < 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: decimal too wide"))); + return DirectFunctionCall3(numeric_in, CStringGetDatum(buf), + ObjectIdGetDatum(0), Int32GetDatum(-1)); +} + +static Datum +read_string_as_text(const chc_column * col, uint64_t row) +{ + const char *p; + size_t len; + + slice_str(col, row, &p, &len); + return PointerGetDatum(cstring_to_text_with_len(p, len)); +} + +static Datum +read_fixedstring_as_text(const chc_column * col, uint64_t row) +{ + size_t width; + const uint8_t *base = chc_column_fixed_data(col, &width); + + return PointerGetDatum(cstring_to_text_with_len((const char *) base + row * width, + width)); +} + +static Datum +read_uuid(const chc_column * col, uint64_t row) +{ + pg_uuid_t *u = (pg_uuid_t *) palloc(sizeof(pg_uuid_t)); + const uint8_t *p = (uint8_t *) chc_column_fixed_data(col, NULL) + row * 16; + uint64_t a, + b; + + memcpy(&a, p, 8); + memcpy(&b, p + 8, 8); + a = pg_hton64(a); + b = pg_hton64(b); + memcpy(u->data, &a, 8); + memcpy(u->data + 8, &b, 8); + return UUIDPGetDatum(u); +} + +/* + * IPv4 wire format: native uint32 (LE on supported hosts). PG inet wants + * BE bytes of dotted-quad, so pg_hton32 the value into ip_addr directly. + */ +static Datum +read_ipv4(const chc_column * col, uint64_t row) +{ + inet *res = (inet *) palloc0(sizeof(inet)); + uint32_t addr; + + memcpy(&addr, (const uint8_t *) chc_column_fixed_data(col, NULL) + row * 4, 4); + addr = pg_hton32(addr); + ip_family(res) = PGSQL_AF_INET; + ip_bits(res) = 32; + memcpy(ip_addr(res), &addr, 4); + SET_INET_VARSIZE(res); + return InetPGetDatum(res); +} + +/* IPv6 wire is already network order; same layout as PG inet ip_addr. */ +static Datum +read_ipv6(const chc_column * col, uint64_t row) +{ + inet *res = (inet *) palloc0(sizeof(inet)); + + ip_family(res) = PGSQL_AF_INET6; + ip_bits(res) = 128; + memcpy(ip_addr(res), (const uint8_t *) chc_column_fixed_data(col, NULL) + row * 16, 16); + SET_INET_VARSIZE(res); + return InetPGetDatum(res); +} + +static Datum +read_enum_as_text(const chc_column * col, const chc_type * type, uint64_t row) +{ + size_t es; + const uint8_t *p = (const uint8_t *) chc_column_fixed_data(col, &es); + int64_t v = 0; + + if (es == 1) + v = (int8_t) p[row]; + else + { + int16_t t; + + memcpy(&t, p + row * 2, 2); + v = t; + } + + size_t n = chc_type_enum_count(type); + + for (size_t i = 0; i < n; i++) + { + const char *en; + size_t el; + int64_t ev; + + chc_type_enum_at(type, i, &en, &el, &ev); + if (ev == v) + return PointerGetDatum(cstring_to_text_with_len(en ? en : "", el)); + } + return PointerGetDatum(cstring_to_text_with_len("", 0)); +} + +/* + * JSON body bytes are STRING-serialized JSON document text. Run them + * through json_in / jsonb_in depending on the caller's target valtype + * (set by ch_kind_to_pg_oid to JSONBOID, overridable when the foreign + * table declares the column as `json`). + */ +static Datum +read_json(const chc_column * col, uint64_t row, Oid valtype) +{ + const char *p; + size_t len; + char *cstr; + Datum ret; + + slice_str(col, row, &p, &len); + cstr = palloc(len + 1); + memcpy(cstr, p, len); + cstr[len] = '\0'; + ret = DirectFunctionCall1(valtype == JSONOID ? json_in : jsonb_in, + CStringGetDatum(cstr)); + pfree(cstr); + return ret; +} + +/* + * LowCardinality(String) or LowCardinality(Nullable(String)). Key for + * row indexes into the dict column; the dict's first slot is the null + * sentinel for the Nullable variant. + */ +static Datum +read_lc_string(const chc_column * col, const chc_type * type, uint64_t row, + Oid * valtype, bool *is_null) +{ + int ks = chc_column_lc_key_size(col); + const uint8_t *kp = (const uint8_t *) chc_column_lc_keys(col) + (size_t) row * ks; + uint64_t k = 0; + + switch (ks) + { + case 1: + k = kp[0]; + break; + case 2: + { + uint16_t v; + + memcpy(&v, kp, 2); + k = v; + break; + } + case 4: + { + uint32_t v; + + memcpy(&v, kp, 4); + k = v; + break; + } + case 8: + memcpy(&k, kp, 8); + break; + default: + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: unexpected LowCardinality key size %d", ks))); + } + + const chc_column *dict = chc_column_lc_dict(col); + const chc_type *inner_t = chc_type_child(type, 0); + + if (chc_type_kind(inner_t) == CHC_NULLABLE + && chc_column_layout(dict) == CHC_COL_NULLABLE) + { + const uint8_t *dnm = chc_column_null_map(dict); + + if (dnm && dnm[k]) + { + *valtype = TEXTOID; + *is_null = true; + return (Datum) 0; + } + dict = chc_column_nullable_inner(dict); + } + + *valtype = TEXTOID; + *is_null = false; + if (chc_column_layout(dict) != CHC_COL_STRING) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported LowCardinality inner type"))); + return read_string_as_text(dict, k); +} + +static Datum +read_array(const chc_column * col, const chc_type * type, uint64_t row, + Oid * valtype, bool *is_null) +{ + const uint64_t *offs = chc_column_array_offsets(col); + uint64_t start = row == 0 ? 0 : offs[row - 1]; + uint64_t end = offs[row]; + uint64_t len = end - start; + const chc_type *inner_t = chc_type_child(type, 0); + const chc_column *inner = chc_column_array_values(col); + ch_binary_array_t *slot = (ch_binary_array_t *) palloc(sizeof(ch_binary_array_t)); + const chc_type *leaf = type; + int ndim = 0; + + /* + * postgres uses one array type per element type regardless of nesting, so + * walk past nested Array layers to the leaf scalar type. + */ + while (chc_type_kind(leaf) == CHC_ARRAY) + { + ndim++; + leaf = chc_type_child(leaf, 0); + } + + slot->len = len; + slot->ndim = ndim; + slot->item_type = ch_kind_to_pg_oid(leaf); + slot->array_type = get_array_type(slot->item_type); + if (slot->array_type == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: could not find array type for column type \"%s\"", + chc_type_name(leaf, NULL)))); + + if (len > 0) + { + Oid scratch = slot->item_type; + + slot->datums = (Datum *) palloc0(sizeof(Datum) * len); + slot->nulls = (bool *) palloc0(sizeof(bool) * len); + + /* + * For ndim==1 read_value returns leaf scalars; for ndim>1 inner_t is + * itself CHC_ARRAY so recursion produces nested ch_binary_array_t*. + * Use a scratch valtype to avoid clobbering slot->item_type. + */ + for (uint64_t i = 0; i < len; ++i) + slot->datums[i] = read_value(inner, inner_t, start + i, + &scratch, &slot->nulls[i]); + } + else + { + slot->datums = NULL; + slot->nulls = NULL; + } + + *valtype = ANYARRAYOID; + *is_null = false; + return PointerGetDatum(slot); +} + +static Datum +read_tuple(const chc_column * col, const chc_type * type, uint64_t row, + Oid * valtype, bool *is_null) +{ + size_t n = chc_type_n_children(type); + ch_binary_tuple_t *slot; + + if (n == 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: returned tuple is empty"))); + + slot = (ch_binary_tuple_t *) palloc(sizeof(ch_binary_tuple_t)); + slot->datums = (Datum *) palloc(sizeof(Datum) * n); + slot->nulls = (bool *) palloc0(sizeof(bool) * n); + slot->types = (Oid *) palloc0(sizeof(Oid) * n); + slot->len = n; + slot->ch_type_name = chc_type_name(type, NULL); + + for (size_t i = 0; i < n; ++i) + { + const chc_type *ft = chc_type_child(type, i); + const chc_column *fc = chc_column_tuple_child(col, i); + + slot->datums[i] = read_value(fc, ft, row, &slot->types[i], &slot->nulls[i]); + } + + *valtype = RECORDOID; + *is_null = false; + return PointerGetDatum(slot); +} + +static Datum +read_value(const chc_column * col, const chc_type * type, uint64_t row, + Oid * valtype, bool *is_null) +{ + /* Unwrap outer Nullable, handling nulls here. */ + if (chc_type_kind(type) == CHC_NULLABLE) + { + const chc_type *inner_t = chc_type_child(type, 0); + + if (chc_column_layout(col) == CHC_COL_NULLABLE) + { + const uint8_t *nm = chc_column_null_map(col); + + if (nm && nm[row]) + { + *valtype = ch_kind_to_pg_oid(inner_t); + *is_null = true; + return (Datum) 0; + } + col = chc_column_nullable_inner(col); + } + type = inner_t; + } + *is_null = false; + + switch (chc_type_kind(type)) + { + case CHC_VOID: + case CHC_NOTHING: + *valtype = InvalidOid; + *is_null = true; + return (Datum) 0; + case CHC_UINT8: + *valtype = INT2OID; + return (Datum) rd_u8((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_BOOL: + *valtype = BOOLOID; + return (Datum) rd_bool((const bool *) chc_column_fixed_data(col, NULL), row); + case CHC_INT8: + *valtype = INT2OID; + return (Datum) rd_i8((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_INT16: + *valtype = INT2OID; + return (Datum) rd_i16((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_UINT16: + *valtype = INT4OID; + return (Datum) rd_u16((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_INT32: + *valtype = INT4OID; + return (Datum) rd_i32((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_UINT32: + *valtype = INT8OID; + return Int64GetDatum((int64) rd_u32((const uint8_t *) chc_column_fixed_data(col, NULL), row)); + case CHC_INT64: + *valtype = INT8OID; + return Int64GetDatum(rd_i64((const uint8_t *) chc_column_fixed_data(col, NULL), row)); + case CHC_UINT64: + { + uint64_t v = rd_u64((const uint8_t *) chc_column_fixed_data(col, NULL), row); + + if (v > (uint64_t) PG_INT64_MAX) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("value " UINT64_FORMAT " is out of range of bigint", + v))); + *valtype = INT8OID; + return Int64GetDatum((int64) v); + } + case CHC_FLOAT32: + *valtype = FLOAT4OID; + return Float4GetDatum(rd_f32((const uint8_t *) chc_column_fixed_data(col, NULL), row)); + case CHC_FLOAT64: + *valtype = FLOAT8OID; + return Float8GetDatum(rd_f64((const uint8_t *) chc_column_fixed_data(col, NULL), row)); + case CHC_DECIMAL32: + case CHC_DECIMAL64: + case CHC_DECIMAL128: + case CHC_DECIMAL256: + *valtype = NUMERICOID; + return read_decimal(col, type, row); + case CHC_STRING: + *valtype = TEXTOID; + return read_string_as_text(col, row); + case CHC_ENUM8: + case CHC_ENUM16: + *valtype = TEXTOID; + return read_enum_as_text(col, type, row); + case CHC_JSON: + case CHC_OBJECT: + { + /* + * *valtype arrives set to JSONBOID by default (from + * ch_kind_to_pg_oid) but the caller's foreign-table + * declaration may have requested JSONOID; honor it. + */ + Oid target = (*valtype == JSONOID) ? JSONOID : JSONBOID; + + *valtype = target; + return read_json(col, row, target); + } + case CHC_FIXED_STRING: + *valtype = TEXTOID; + return read_fixedstring_as_text(col, row); + case CHC_DATE: + *valtype = DATEOID; + return DateADTGetDatum((DateADT) rd_u16((const uint8_t *) chc_column_fixed_data(col, NULL), row) + - CH_TO_PG_DATE_OFFSET); + case CHC_DATE32: + *valtype = DATEOID; + return DateADTGetDatum((DateADT) rd_i32((const uint8_t *) chc_column_fixed_data(col, NULL), row) + - CH_TO_PG_DATE_OFFSET); + case CHC_DATETIME: + { + uint32_t secs = rd_u32((const uint8_t *) chc_column_fixed_data(col, NULL), row); + + *valtype = TIMESTAMPTZOID; + return TimestampTzGetDatum(time_t_to_timestamptz((pg_time_t) secs)); + } + case CHC_DATETIME64: + { + int64 raw = rd_i64((const uint8_t *) chc_column_fixed_data(col, NULL), row); + uint32_t scale = chc_type_datetime64_scale(type); + int64 power = pow10i[scale]; + + *valtype = TIMESTAMPTZOID; + return TimestampTzGetDatum(time_t_to_timestamptz(raw / power) + + (raw % power) * (USECS_PER_SEC / power)); + } + case CHC_UUID: + *valtype = UUIDOID; + return read_uuid(col, row); + case CHC_IPV4: + *valtype = INETOID; + return read_ipv4(col, row); + case CHC_IPV6: + *valtype = INETOID; + return read_ipv6(col, row); + case CHC_LOW_CARDINALITY: + return read_lc_string(col, type, row, valtype, is_null); + case CHC_ARRAY: + return read_array(col, type, row, valtype, is_null); + case CHC_TUPLE: + return read_tuple(col, type, row, valtype, is_null); + default: + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported type in binary protocol"))); + } + /* unreachable */ + return (Datum) 0; +} + +/* ---- read state ----------------------------------------------------- */ + +static bool +load_block(ch_binary_read_state_t * state) +{ + state->cur = ch_binary_response_fetch_next_block(state->resp); + if (state->cur == NULL) + { + const char *resp_err = ch_binary_response_error(state->resp); + + if (resp_err) + state->error = pstrdup(resp_err); + state->done = true; + return false; + } + return true; +} + +void +ch_binary_read_state_init(ch_binary_read_state_t * state, ch_binary_response_t * resp) +{ + const char *resp_err; + size_t ncols; + + state->resp = resp; + state->block = 0; + state->row = 0; + state->done = false; + state->error = NULL; + state->coltypes = NULL; + state->values = NULL; + state->nulls = NULL; + state->cur = NULL; + + resp_err = ch_binary_response_error(resp); + if (resp_err) + { + state->done = true; + state->error = pstrdup(resp_err); + return; + } + + ncols = ch_binary_response_columns(resp); + if (ncols == 0) + { + state->done = true; + return; + } + + state->coltypes = palloc0(sizeof(Oid) * ncols); + state->values = palloc0(sizeof(Datum) * ncols); + state->nulls = palloc0(sizeof(bool) * ncols); + + if (!load_block(state)) + return; + + for (size_t i = 0; i < ncols; i++) + state->coltypes[i] = ch_kind_to_pg_oid(chc_block_column_type(state->cur, i)); +} + +bool +ch_binary_read_row(ch_binary_read_state_t * state) +{ + size_t ncols; + + if (state->done || state->coltypes == NULL || state->error) + return false; + + ncols = ch_binary_response_columns(state->resp); + +again: + if (state->cur == NULL) + { + if (!load_block(state)) + return false; + } + + if (state->row >= chc_block_n_rows(state->cur)) + { + state->row = 0; + state->cur = NULL; + goto again; + } + + PG_TRY(); + { + for (size_t i = 0; i < ncols; i++) + { + /* + * Currently, read_value overwrites *valtype for most types + * canonical PG type for that CH kind. For CHC_JSON we honor the + * incoming value so callers (binary_fetch_row) can pin the Datum + * type to JSONOID for `data json` foreign columns and avoid the + * jsonb_in -> jsonb_out round-trip that would strip CH's verbatim + * STRING- serialized formatting. + */ + Oid t = state->coltypes[i]; + const chc_column *col = chc_block_column(state->cur, i); + const chc_type *ct = chc_block_column_type(state->cur, i); + + state->values[i] = read_value(col, ct, state->row, &t, &state->nulls[i]); + } + } + PG_CATCH(); + { + MemoryContext mcxt = GetMemoryChunkContext(state); + MemoryContext oldcxt; + ErrorData *edata; + const char *msg; + static const char prefix[] = "pg_clickhouse: "; + + oldcxt = MemoryContextSwitchTo(mcxt); + edata = CopyErrorData(); + msg = edata->message ? edata->message : "unknown error"; + + /* + * binary_fetch_row re-prefixes with "pg_clickhouse: error while + * reading row:"; drop a leading "pg_clickhouse: " here so the final + * message doesn't carry it twice. + */ + if (strncmp(msg, prefix, sizeof(prefix) - 1) == 0) + msg += sizeof(prefix) - 1; + state->error = pstrdup(msg); + FlushErrorState(); + FreeErrorData(edata); + MemoryContextSwitchTo(oldcxt); + state->done = true; + return false; + } + PG_END_TRY(); + + state->row++; + return true; +} + +void +ch_binary_read_state_free(ch_binary_read_state_t * state) +{ + /* state->error is palloc'd; freed with surrounding memory context. */ + state->error = NULL; +} diff --git a/src/binary/encode.c b/src/binary/encode.c new file mode 100644 index 00000000..c21d50b6 --- /dev/null +++ b/src/binary/encode.c @@ -0,0 +1,428 @@ +/* + * encode.c + * + * PG-side INSERT path. Reads PG Datums and dispatches into the typed + * ch_binary_append_* shims exposed by insert.c. + */ + +#include "postgres.h" + +#include +#include /* AF_INET, expanded by PG inet macros */ + +#include "access/tupdesc.h" +#include "catalog/pg_type_d.h" +#include "nodes/pg_list.h" +#include "fmgr.h" +#include "utils/builtins.h" +#include "utils/date.h" +#include "utils/inet.h" +#include "utils/lsyscache.h" +#include "utils/palloc.h" +#include "utils/timestamp.h" +#include "utils/uuid.h" + +#include "binary_internal.h" + +/* CH Date epoch is unix; offset to PG epoch (2000-01-01) */ +#define CH_TO_PG_DATE_OFFSET (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) + +/* + * Map a CH column kind to the PG type our import path uses. Used when + * constructing the TupleDesc for INSERT...VALUES. + */ +static Oid +ch_kind_to_pg_oid_for_insert(const chc_type * type, const char *colname) +{ + switch (chc_type_kind(type)) + { + case CHC_INT8: + case CHC_INT16: + case CHC_UINT8: + return INT2OID; + case CHC_BOOL: + return BOOLOID; + case CHC_INT32: + case CHC_UINT16: + return INT4OID; + case CHC_INT64: + case CHC_UINT32: + case CHC_UINT64: + return INT8OID; + case CHC_FLOAT32: + return FLOAT4OID; + case CHC_FLOAT64: + return FLOAT8OID; + case CHC_DECIMAL32: + case CHC_DECIMAL64: + case CHC_DECIMAL128: + case CHC_DECIMAL256: + return NUMERICOID; + case CHC_STRING: + case CHC_FIXED_STRING: + case CHC_ENUM8: + case CHC_ENUM16: + return TEXTOID; + case CHC_JSON: + case CHC_OBJECT: + return JSONBOID; + case CHC_DATE: + case CHC_DATE32: + return DATEOID; + case CHC_DATETIME: + case CHC_DATETIME64: + return TIMESTAMPTZOID; + case CHC_UUID: + return UUIDOID; + case CHC_IPV4: + case CHC_IPV6: + return INETOID; + case CHC_ARRAY: + { + /* + * postgres uses one array type per element type regardless of + * nesting, so unwrap nested Array layers before looking up. + */ + const chc_type *leaf = type; + Oid item_type; + Oid array_type; + + while (chc_type_kind(leaf) == CHC_ARRAY) + leaf = chc_type_child(leaf, 0); + item_type = ch_kind_to_pg_oid_for_insert(leaf, colname); + array_type = get_array_type(item_type); + + if (array_type == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: could not find array type for column \"%s\"", + colname ? colname : "?"))); + return array_type; + } + case CHC_TUPLE: + return RECORDOID; + case CHC_NULLABLE: + case CHC_LOW_CARDINALITY: + return ch_kind_to_pg_oid_for_insert(chc_type_child(type, 0), colname); + default: + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported column type \"%s\" for \"%s\"", + chc_type_name(type, NULL), + colname ? colname : "?"))); + } + return InvalidOid; +} + +void +ch_binary_prepare_insert(void *conn, const ch_query * query, + ch_binary_insert_state * state) +{ + ch_binary_column_info *cols = NULL; + size_t n = 0; + ch_binary_insert_handle *h; + + h = ch_binary_begin_insert((ch_binary_connection_t *) conn, query, &cols, &n); + + state->len = n; + state->insert_block = h; + + if (n == 0) + return; + + state->outdesc = CreateTemplateTupleDesc(n); + for (size_t i = 0; i < n; i++) + { + Oid pg_type = ch_kind_to_pg_oid_for_insert(cols[i].type, cols[i].name); + + /* + * CHC_JSON defaults to JSONBOID; honor foreign-table's JSONOID + * declaration when present so outdesc matches the source slot and no + * json↔jsonb conversion is needed. + */ + if (pg_type == JSONBOID && query->tupdesc && query->attr_nums) + { + AttrNumber attnum = list_nth_int((List *) query->attr_nums, i); + + if (attnum >= 1 && attnum <= query->tupdesc->natts && + TupleDescAttr(query->tupdesc, attnum - 1)->atttypid == JSONOID) + pg_type = JSONOID; + } + + TupleDescInitEntry(state->outdesc, (AttrNumber) (i + 1), + cols[i].name ? cols[i].name : "", + pg_type, -1, 0); + } +} + +/* + * Append a single value (already extracted from a Datum + isnull) into the + * current column, dispatching on (PG Oid, CH kind). ereports on mismatch. + */ +static void +append_one(ch_binary_insert_handle * h, size_t colidx, + chc_kind kind, Datum val, Oid valtype, bool isnull) +{ + switch (valtype) + { + case INT2OID: + case INT4OID: + case INT8OID: + { + int64_t v = 0; + + /* Support mixing integer types. */ + if (!(kind == CHC_BOOL + || (kind >= CHC_INT8 && kind <= CHC_INT64) + || (kind >= CHC_UINT8 && kind <= CHC_UINT64))) + goto type_mismatch; + if (!isnull) + { + if (valtype == INT2OID) + v = (int64_t) DatumGetInt16(val); + else if (valtype == INT4OID) + v = (int64_t) DatumGetInt32(val); + else + v = DatumGetInt64(val); + } + ch_binary_append_int(h, colidx, v, isnull); + return; + } + case BOOLOID: + if (kind != CHC_BOOL && kind != CHC_UINT8) + goto type_mismatch; + ch_binary_append_bool(h, colidx, (bool) val, isnull); + return; + case FLOAT4OID: + if (kind != CHC_FLOAT32) + goto type_mismatch; + ch_binary_append_float(h, colidx, DatumGetFloat4(val), isnull); + return; + case FLOAT8OID: + if (kind != CHC_FLOAT64) + goto type_mismatch; + ch_binary_append_double(h, colidx, DatumGetFloat8(val), isnull); + return; + case NUMERICOID: + { + char *s = NULL; + + if (kind != CHC_DECIMAL32 && kind != CHC_DECIMAL64 + && kind != CHC_DECIMAL128 && kind != CHC_DECIMAL256) + goto type_mismatch; + if (!isnull) + s = DatumGetCString(DirectFunctionCall1(numeric_out, val)); + ch_binary_append_decimal(h, colidx, s, isnull); + if (s) + pfree(s); + return; + } + case TEXTOID: + { + const char *p = NULL; + size_t len = 0; + text *string = NULL; + + if (!isnull) + { + string = PG_DETOAST_DATUM(val); + p = VARDATA(string); + len = VARSIZE_ANY_EXHDR(string); + } + switch (kind) + { + case CHC_FIXED_STRING: + case CHC_STRING: + case CHC_ENUM8: + case CHC_ENUM16: + ch_binary_append_bytes(h, colidx, p, len, isnull); + return; + default: + goto type_mismatch; + } + } + case DATEOID: + { + int64_t seconds = 0; + + if (kind != CHC_DATE && kind != CHC_DATE32) + goto type_mismatch; + if (!isnull) + seconds = ((int64_t) DatumGetDateADT(val) + + CH_TO_PG_DATE_OFFSET) * SECS_PER_DAY; + ch_binary_append_date_seconds(h, colidx, seconds, isnull); + return; + } + case TIMESTAMPOID: + case TIMESTAMPTZOID: + { + switch (kind) + { + case CHC_DATETIME: + { + int64_t seconds = isnull + ? 0 + : (int64_t) timestamptz_to_time_t(DatumGetTimestamp(val)); + + ch_binary_append_datetime_seconds(h, colidx, seconds, isnull); + } break; + case CHC_DATETIME64: + { + int64_t raw = 0; + + if (!isnull) + { + uint32_t prec = ch_binary_column_datetime64_precision(h, colidx); + Timestamp t = DatumGetTimestamp(val); + int64 power = pow10i[prec]; + int64 secs = t / USECS_PER_SEC; + int64 us_rem = t % USECS_PER_SEC; + + /* + * floor-divide; C trunc-to-zero leaves + * negative remainder + */ + if (us_rem < 0) + { + secs -= 1; + us_rem += USECS_PER_SEC; + } + secs += CH_TO_PG_DATE_OFFSET * SECS_PER_DAY; + raw = secs * power + us_rem * power / USECS_PER_SEC; + } + ch_binary_append_datetime64_raw(h, colidx, raw, isnull); + } break; + default: + goto type_mismatch; + } + return; + } + case ANYARRAYOID: + { + ch_binary_array_t *arr; + chc_kind item_kind; + Oid child_valtype; + + if (kind != CHC_ARRAY) + goto type_mismatch; + + arr = (ch_binary_array_t *) DatumGetPointer(val); + ch_binary_array_begin(h, colidx); + + /* + * While array_begin is active ch_binary_column_kind targets + * the inner element kind. For nested arrays the children are + * themselves ch_binary_array_t* so recurse with ANYARRAYOID; + * at the leaf level use the scalar item_type. + */ + item_kind = ch_binary_column_kind(h, colidx); + child_valtype = (arr->ndim > 1) ? ANYARRAYOID : arr->item_type; + for (size_t i = 0; i < arr->len; i++) + append_one(h, 0, item_kind, arr->datums[i], child_valtype, arr->nulls[i]); + + ch_binary_array_end(h); + return; + } + case UUIDOID: + { + uint8_t bytes[16]; + + if (kind != CHC_UUID) + goto type_mismatch; + if (!isnull) + memcpy(bytes, DatumGetUUIDP(val)->data, 16); + else + memset(bytes, 0, 16); + ch_binary_append_uuid(h, colidx, bytes, isnull); + return; + } + case INETOID: + { + const uint8_t *addr = NULL; + size_t addrlen = 0; + + if (kind != CHC_IPV4 && kind != CHC_IPV6) + goto type_mismatch; + if (!isnull) + { + inet *ipa = DatumGetInetPP(val); + int fam = ip_family(ipa); + int expected = (kind == CHC_IPV4) ? PGSQL_AF_INET : PGSQL_AF_INET6; + + if (fam != expected) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: inet family mismatch for column %zu", + colidx))); + addr = ip_addr(ipa); + addrlen = ip_addrsize(ipa); + } + else + addrlen = (kind == CHC_IPV4) ? 4 : 16; + ch_binary_append_inet(h, colidx, addr, addrlen, isnull); + return; + } + case JSONOID: + case JSONBOID: + { + char *s = NULL; + size_t len = 0; + + if (kind != CHC_JSON && kind != CHC_OBJECT) + goto type_mismatch; + if (!isnull) + { + s = DatumGetCString(DirectFunctionCall1(valtype == JSONBOID + ? jsonb_out + : json_out, + val)); + len = strlen(s); + } + ch_binary_append_bytes(h, colidx, s, len, isnull); + if (s) + pfree(s); + return; + } + default: + goto type_mismatch; + } + +type_mismatch: + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: unexpected PG/CH type pair for column %zu", colidx))); +} + +void +ch_binary_column_append_data(ch_binary_insert_state * state, size_t colidx) +{ + Datum val = state->values[colidx]; + Oid valtype = TupleDescAttr(state->outdesc, colidx)->atttypid; + bool isnull = state->nulls[colidx]; + chc_kind kind = ch_binary_column_kind(state->insert_block, colidx); + + append_one(state->insert_block, colidx, kind, val, valtype, isnull); +} + +void +ch_binary_insert_columns(ch_binary_insert_state * state) +{ + ch_binary_flush_block(state->insert_block); +} + +void +ch_binary_insert_state_free(void *c) +{ + ch_binary_insert_state *state = (ch_binary_insert_state *) c; + + if (state->insert_block == NULL) + return; + + /* + * Reset-callback context: cannot ereport. Finalize runs from the FDW + * happy path before we get here; if it did not (mid-query abort), the + * release call flags the connection broken so the next query rebuilds. + */ + ch_binary_release_insert(state->insert_block); + state->insert_block = NULL; +} diff --git a/src/binary/insert.c b/src/binary/insert.c new file mode 100644 index 00000000..f202920e --- /dev/null +++ b/src/binary/insert.c @@ -0,0 +1,1475 @@ +/* + * insert.c + * + * INSERT path for the binary driver. begin_insert sends the query, takes + * the server's empty Data block as schema, classifies each column into + * one of a handful of accumulator layouts (fixed / string / LC / array + * variants), and exposes typed ch_binary_append_* shims. flush_block + * materialises the accumulators into a chc_block_builder and ships one + * Data packet; finalize_insert sends the closing empty Data and drains. + * release_insert is reset-callback safe: it never talks to the wire. + */ + +#include "postgres.h" + +#include + +#include "common/hashfn.h" +#include "port/pg_bswap.h" +#include "utils/memutils.h" +#include "utils/palloc.h" + +#include "binary_internal.h" + +/* dynamic buffer for raw bytes; palloc-backed, freed by context delete */ +typedef struct dynbuf +{ + uint8_t *data; + size_t len; + size_t cap; +} dynbuf; + +static void +dynbuf_reserve(dynbuf * b, size_t need) +{ + if (need <= b->cap) + return; + size_t ncap = b->cap ? b->cap : 64; + + while (ncap < need) + ncap *= 2; + b->data = b->data + ? repalloc_huge(b->data, ncap) + : MemoryContextAllocExtended(CurrentMemoryContext, ncap, MCXT_ALLOC_HUGE); + b->cap = ncap; +} + +static void +dynbuf_append(dynbuf * b, const void *src, size_t n) +{ + dynbuf_reserve(b, b->len + n); + if (src && n) + memcpy(b->data + b->len, src, n); + b->len += n; +} + +static void +dynbuf_append_zero(dynbuf * b, size_t n) +{ + dynbuf_reserve(b, b->len + n); + memset(b->data + b->len, 0, n); + b->len += n; +} + +static void +dynbuf_reset(dynbuf * b) +{ + b->len = 0; +} + +/* dynamic array of u64 */ +typedef struct u64buf +{ + uint64_t *data; + size_t len; + size_t cap; +} u64buf; + +static void +u64buf_push(u64buf * b, uint64_t v) +{ + if (b->len + 1 > b->cap) + { + size_t ncap = b->cap ? b->cap * 2 : 16; + size_t bytes = ncap * sizeof(uint64_t); + + b->data = b->data + ? repalloc_huge(b->data, bytes) + : MemoryContextAllocExtended(CurrentMemoryContext, bytes, MCXT_ALLOC_HUGE); + b->cap = ncap; + } + b->data[b->len++] = v; +} + +static void +u64buf_reset(u64buf * b) +{ + b->len = 0; +} + +/* + * Insert column layout decided at begin_insert from the column's chc_type. + * Each typed append_* appends into one of three storage groups: + * - body / body_offs: fixed-width or string values for the top-level + * column or, when an array context is open, the inner items. + * - arr_offs: cumulative outer ends for Array(*) columns. + * - nulls: one byte per top-level row for Nullable(*). + */ +typedef enum +{ + IC_FIXED, + IC_STRING, + IC_LC_STRING, + IC_ARRAY_FIXED, + IC_ARRAY_STRING, + IC_ARRAY_NESTED_FIXED, + IC_ARRAY_NESTED_STRING, + IC_JSON_STRING, +} ic_layout; + +static inline bool +ic_layout_array_fixed(ic_layout l) +{ + return l == IC_ARRAY_FIXED || l == IC_ARRAY_NESTED_FIXED; +} + +static inline bool +ic_layout_array_string(ic_layout l) +{ + return l == IC_ARRAY_STRING || l == IC_ARRAY_NESTED_STRING; +} + +typedef struct ic_col +{ + const chc_type *t; /* full column type (incl. Nullable wrapper) */ + const chc_type *inner_t; /* possibly unwrapped Nullable */ + const chc_type *elem_t; /* Array element type, with Nullable + * unwrapped */ + ic_layout layout; + bool is_nullable; + int array_depth; /* current open begin/end nesting */ + int ndim; /* >=1 for array layouts; 1 for top-level + * Array */ + bool array_inner_is_string; + size_t elem_size; /* fixed elem size (top-level FIXED or array + * element FIXED) */ + uint32_t dt64_precision; + dynbuf body; /* row-aligned for FIXED, byte-flat for + * STRING/LC */ + u64buf body_offs; /* offsets per row for STRING/LC inner */ + dynbuf nulls; /* per top-level row */ + u64buf arr_offs; /* cumulative ends for outermost array layer + * (or only layer when ndim==1) */ + u64buf *arr_offs_inner; /* for ndim>1: ndim-1 inner-layer offsets, + * [0]=just inside outer, [ndim-2]=adjacent to + * leaves */ + size_t n_rows; /* committed top-level rows */ + + /* + * Cached column info exposed to callers. info.type borrows from the + * initial_block's chc_type tree, unwrapping Nullable + outer LC. + */ + ch_binary_column_info info; +} ic_col; + +struct ch_binary_insert_handle +{ + MemoryContext cxt; + chc_client *client; + struct ch_binary_state *state; /* parent connection; used to flag broken + * state on error */ + chc_block *initial_block; /* schema source (server's empty Data) */ + size_t ncols; + ic_col *cols; + bool array_active; + size_t array_col_idx; + bool started; + bool finalized; /* finalize_insert has run (success or raised) */ +}; + +static void +classify_column(ic_col * ic, const chc_type * t) +{ + ic->t = t; + if (chc_type_kind(t) == CHC_NULLABLE) + { + ic->is_nullable = true; + t = chc_type_child(t, 0); + } + ic->inner_t = t; + + chc_kind k = chc_type_kind(t); + + if (k == CHC_LOW_CARDINALITY) + { + const chc_type *inner = chc_type_child(t, 0); + bool inner_nullable = chc_type_kind(inner) == CHC_NULLABLE; + const chc_type *base = inner_nullable ? chc_type_child(inner, 0) : inner; + + if (chc_type_kind(base) != CHC_STRING) + { + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported LowCardinality variant: %s", + chc_type_name(base, NULL)))); + } + ic->layout = IC_LC_STRING; + ic->elem_t = base; + return; + } + if (k == CHC_ARRAY) + { + /* + * Walk through nested Array layers to the leaf element type so + * Array(Array(...)) maps to a single ic_col with ndim levels. + */ + const chc_type *base = t; + int ndim = 0; + + while (chc_type_kind(base) == CHC_ARRAY) + { + ndim++; + base = chc_type_child(base, 0); + } + bool elem_nullable = chc_type_kind(base) == CHC_NULLABLE; + + /* Array(Nullable(T)) not supported yet. */ + if (elem_nullable) + { + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: %s not currently supported", + chc_type_name(base, NULL)))); + } + chc_kind ek = chc_type_kind(base); + + ic->elem_t = base; + ic->ndim = ndim; + if (ek == CHC_STRING || ek == CHC_FIXED_STRING) + { + ic->layout = ndim == 1 ? IC_ARRAY_STRING : IC_ARRAY_NESTED_STRING; + ic->array_inner_is_string = true; + } + else if (chc_type_elem_size(base) > 0) + { + ic->layout = ndim == 1 ? IC_ARRAY_FIXED : IC_ARRAY_NESTED_FIXED; + ic->elem_size = chc_type_elem_size(base); + } + else + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported Array element type: %s", chc_type_name(base, NULL)))); + if (ndim > 1) + ic->arr_offs_inner = palloc0((size_t) (ndim - 1) * sizeof(u64buf)); + return; + } + if (k == CHC_STRING) + { + ic->layout = IC_STRING; + return; + } + if (k == CHC_JSON) + { + ic->layout = IC_JSON_STRING; + return; + } + size_t es = chc_type_elem_size(t); + + if (es > 0) + { + ic->layout = IC_FIXED; + ic->elem_size = es; + if (k == CHC_DATETIME64) + ic->dt64_precision = (uint32_t) chc_type_datetime64_scale(t); + return; + } + if (k == CHC_FIXED_STRING) + { + ic->layout = IC_FIXED; + ic->elem_size = (size_t) chc_type_fixed_size(t); + return; + } + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: could not prepare insert - unsupported column type: %s", + chc_type_name(t, NULL)))); +} + +static void +recv_initial_block(struct ch_binary_state *s, ch_binary_insert_handle * h) +{ + for (;;) + { + chc_packet pkt = {}; + chc_err err = {}; + int rc = chc_client_recv_packet(s->client, &pkt, &err); + + if (rc != CHC_OK) + { + s->broken = true; + raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); + } + if (pkt.kind == CHC_PKT_EXCEPTION) + { + const char *msg = "server exception"; + + if (pkt.exception && pkt.exception->display_text) + msg = pkt.exception->display_text; + else if (pkt.exception && pkt.exception->name) + msg = pkt.exception->name; + char *msg_copy = pstrdup(msg); + + s->broken = true; + chc_packet_clear(s->client, &pkt); + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: could not prepare insert - %s", msg_copy))); + } + if (pkt.kind == CHC_PKT_DATA && pkt.block && + chc_block_n_columns(pkt.block) > 0) + { + h->initial_block = pkt.block; + pkt.block = NULL; + chc_packet_clear(s->client, &pkt); + return; + } + chc_packet_clear(s->client, &pkt); + } +} + +/* + * After classify_column ereports the server is mid-INSERT awaiting our + * Data; send empty Data + drain so the connection stays usable. + */ +static void +drain_aborted_insert(struct ch_binary_state *s) +{ + chc_err ce = {}; + + (void) chc_client_send_data(s->client, NULL, &ce); + for (;;) + { + chc_packet drain = {}; + + ce = (chc_err) + { + 0 + }; + int drc = chc_client_recv_packet(s->client, &drain, &ce); + bool eos = (drc == CHC_OK && + (drain.kind == CHC_PKT_END_OF_STREAM || + drain.kind == CHC_PKT_EXCEPTION)); + + chc_packet_clear(s->client, &drain); + if (drc != CHC_OK || eos) + break; + } +} + +ch_binary_insert_handle * +ch_binary_begin_insert(ch_binary_connection_t * conn, const ch_query * query, + ch_binary_column_info * *out_cols, size_t * out_n) +{ + struct ch_binary_state *s = conn_state(conn); + + /* + * Parent h's cxt to the connection's cxt, not CurrentMemoryContext. The + * caller registers a reset callback on a sibling context that drains the + * insert via end_insert(h); if h lived under that sibling, MemoryContext + * tree teardown would free h before the callback fired. + */ + MemoryContext cxt = AllocSetContextCreate(s->cxt, + "pg_clickhouse binary insert", + ALLOCSET_DEFAULT_SIZES); + MemoryContext old = MemoryContextSwitchTo(cxt); + ch_binary_insert_handle *h; + volatile bool need_drain = false; + + PG_TRY(); + { + h = palloc0(sizeof(*h)); + h->cxt = cxt; + h->client = s->client; + h->state = s; + + /* Append " VALUES" so server enters insert mode. */ + size_t sql_len = strlen(query->sql); + char *sql = palloc(sql_len + 8); + + memcpy(sql, query->sql, sql_len); + memcpy(sql + sql_len, " VALUES", 7); + sql[sql_len + 7] = '\0'; + + /* + * On servers that support it (24.10+), tell server to serialize any + * JSON columns using STRING wire format. INSERT path doesn't need + * this, server reads the per-column version prefix the builder + * writes, but we set it on the same packet for symmetry with the + * SELECT path and so any RETURNING-style projection on top still + * decodes. + */ + chc_query_setting json_setting = { + .name = "output_format_native_write_json_as_string", + .value = "1", + .important = true, + }; + chc_query_opts insert_opts = {}; + const chc_query_opts *opts_ptr = NULL; + + if (server_supports_json_as_string(s->client)) + { + insert_opts.settings = &json_setting; + insert_opts.n_settings = 1; + opts_ptr = &insert_opts; + } + + chc_err err = {}; + int rc = chc_client_send_query_ex(s->client, sql, sql_len + 7, opts_ptr, &err); + + if (rc != CHC_OK) + { + s->broken = true; + raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); + } + + recv_initial_block(s, h); + + /* + * Server is now waiting our Data; failures past this point need an + * empty-Data + drain so the connection stays usable. + */ + need_drain = true; + h->started = true; + + size_t nc = chc_block_n_columns(h->initial_block); + + h->ncols = nc; + h->cols = palloc0(nc * sizeof(ic_col)); + for (size_t i = 0; i < nc; i++) + { + ic_col *c = &h->cols[i]; + const chc_type *ct = chc_block_column_type(h->initial_block, i); + size_t nlen; + const char *nm; + + classify_column(c, ct); + nm = chc_block_column_name(h->initial_block, i, &nlen); + c->info.name = pnstrdup(nm ? nm : "", nlen); + c->info.is_nullable = c->is_nullable; + + /* + * inner_t already unwrapped Nullable; unwrap LowCardinality and + * perhaps its inner Nullable to expose innermost type. + */ + const chc_type *vt = c->inner_t; + + if (chc_type_kind(vt) == CHC_LOW_CARDINALITY) + { + vt = chc_type_child(vt, 0); + if (chc_type_kind(vt) == CHC_NULLABLE) + vt = chc_type_child(vt, 0); + } + c->info.type = vt; + } + + *out_cols = NULL; + if (nc) + { + ch_binary_column_info *arr = palloc(nc * sizeof(*arr)); + + for (size_t i = 0; i < nc; i++) + arr[i] = h->cols[i].info; + *out_cols = arr; + } + *out_n = nc; + } + PG_CATCH(); + { + if (need_drain) + drain_aborted_insert(s); + MemoryContextSwitchTo(old); + MemoryContextDelete(cxt); + PG_RE_THROW(); + } + PG_END_TRY(); + + MemoryContextSwitchTo(old); + return h; +} + +/* + * Resolve the storage to receive a value for column `col_idx`, accounting + * for an active array context. Returns the ic_col* whose body buffer is + * the target. For Nullable wrappers, records the null bit. ereports on + * NULL-into-NOT-NULL. + */ +static ic_col * +resolve_col(ch_binary_insert_handle * h, size_t col_idx, bool isnull) +{ + ic_col *c = h->array_active ? &h->cols[h->array_col_idx] : &h->cols[col_idx]; + + if (isnull && !h->array_active && !c->is_nullable) + { + size_t tnlen; + const char *tname = chc_type_name(c->t, &tnlen); + + ereport(ERROR, + (errcode(ERRCODE_NOT_NULL_VIOLATION), + errmsg("pg_clickhouse: cannot append NULL to NOT NULL %.*s column", + (int) tnlen, tname ? tname : "?"))); + } + if (!h->array_active && c->is_nullable) + { + uint8_t b = isnull ? 1 : 0; + + dynbuf_append(&c->nulls, &b, 1); + } + return c; +} + +static void +append_fixed_bytes(ic_col * c, const void *p, size_t n) +{ + dynbuf_append(&c->body, p, n); +} + +static void +append_fixed_zero(ic_col * c, size_t n) +{ + dynbuf_append_zero(&c->body, n); +} + +static void +append_string_row(ic_col * c, const void *p, size_t n) +{ + if (n) + dynbuf_append(&c->body, p, n); + u64buf_push(&c->body_offs, c->body.len); +} + +/* + * Convert decimal text "[-]digits[.frac]" into a ClickHouse Decimal wire value: + * two's-complement signed integer in `width` LE bytes (4/8/16/32 for + * Decimal32/64/128/256), with `scale` fractional digits folded into the value. + */ +static void +decimal_text_to_bytes(const char *s, uint32_t scale, size_t width, uint8_t * out) +{ + bool neg = false; + + if (!s) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("pg_clickhouse: decimal parse failure"))); + if (*s == '-') + { + neg = true; + s++; + } + else if (*s == '+') + s++; + + /* find offsets, going to iterate digits, skipping non-digits */ + const char *dot = strchr(s, '.'); + size_t ilen = dot ? (size_t) (dot - s) : strlen(s); + const char *frac = dot ? dot + 1 : ""; + size_t flen = strlen(frac); + size_t ndig = ilen + scale; + + uint32_t mag[8] = {}; + size_t nwords = width / 4; + + /* accumulate digits (padded/truncated to scale) into mag */ + for (size_t i = 0; i < ndig; i++) + { + char c = i < ilen ? s[i] + : i - ilen < flen ? frac[i - ilen] + : '0'; + uint64_t carry = (uint64_t) (c - '0'); + + for (size_t b = 0; b < nwords; b++) + { + uint64_t v = (uint64_t) mag[b] * 10 + carry; + + mag[b] = (uint32_t) v; + carry = v >> 32; + } + } + /* two's-complement negation */ + if (neg) + { + for (size_t b = 0; b < nwords; b++) + mag[b] = ~mag[b]; + uint64_t carry = 1; + + for (size_t b = 0; b < nwords && carry; b++) + { + uint64_t v = (uint64_t) mag[b] + carry; + + mag[b] = (uint32_t) v; + carry = v >> 32; + } + } + + memcpy(out, mag, width); +} + +static void +append_int_kind(ic_col * c, int64_t val) +{ + chc_kind k = ic_layout_array_fixed(c->layout) + ? chc_type_kind(c->elem_t) + : chc_type_kind(c->inner_t); + + switch (k) + { + case CHC_INT8: + case CHC_UINT8: + case CHC_BOOL: + { + int8_t v = (int8_t) val; + + append_fixed_bytes(c, &v, 1); + return; + } + case CHC_INT16: + case CHC_UINT16: + { + int16_t v = (int16_t) val; + + append_fixed_bytes(c, &v, 2); + return; + } + case CHC_INT32: + case CHC_UINT32: + { + int32_t v = (int32_t) val; + + append_fixed_bytes(c, &v, 4); + return; + } + case CHC_INT64: + case CHC_UINT64: + { + append_fixed_bytes(c, &val, 8); + return; + } + default: + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: int value into non-integer column"))); + } +} + +void +ch_binary_append_int(ch_binary_insert_handle * h, size_t col, int64_t val, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + + if (isnull) + append_fixed_zero(c, c->elem_size); + else + append_int_kind(c, val); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_uint(ch_binary_insert_handle * h, size_t col, uint64_t val, bool isnull) +{ + ch_binary_append_int(h, col, (int64_t) val, isnull); +} + +void +ch_binary_append_bool(ch_binary_insert_handle * h, size_t col, bool val, bool isnull) +{ + ch_binary_append_int(h, col, val, isnull); +} + +void +ch_binary_append_double(ch_binary_insert_handle * h, size_t col, double val, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + + if (isnull) + append_fixed_zero(c, 8); + else + append_fixed_bytes(c, &val, 8); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_float(ch_binary_insert_handle * h, size_t col, float val, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + + if (isnull) + append_fixed_zero(c, 4); + else + append_fixed_bytes(c, &val, 4); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_bytes(ch_binary_insert_handle * h, size_t col, const void *p, + size_t n, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + chc_kind k = ic_layout_array_string(c->layout) + ? chc_type_kind(c->elem_t) + : chc_type_kind(c->inner_t); + + if (c->layout == IC_LC_STRING) + { + /* Accumulate via body+body_offs; dict materialized at flush. */ + if (isnull) + append_string_row(c, NULL, 0); + else + append_string_row(c, p, n); + MemoryContextSwitchTo(old); + return; + } + + if (k == CHC_FIXED_STRING) + { + size_t w = (size_t) chc_type_fixed_size(ic_layout_array_string(c->layout) ? c->elem_t : c->inner_t); + + if (isnull) + append_fixed_zero(c, w); + else + { + size_t take = n < w ? n : w; + + if (take) + dynbuf_append(&c->body, p, take); + if (take < w) + dynbuf_append_zero(&c->body, w - take); + } + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_ENUM8 || k == CHC_ENUM16) + { + size_t ew = (k == CHC_ENUM8) ? 1 : 2; + + if (isnull) + { + append_fixed_zero(c, ew); + MemoryContextSwitchTo(old); + return; + } + const chc_type *et = ic_layout_array_string(c->layout) ? c->elem_t : c->inner_t; + size_t nenum = chc_type_enum_count(et); + int64_t val = 0; + bool found = false; + + for (size_t i = 0; i < nenum; i++) + { + const char *en; + size_t el; + int64_t ev; + + chc_type_enum_at(et, i, &en, &el, &ev); + if (el == n && memcmp(en, p, n) == 0) + { + val = ev; + found = true; + break; + } + } + if (!found) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("pg_clickhouse: enum value '%.*s' not found", + (int) n, (const char *) p))); + if (k == CHC_ENUM8) + { + int8_t v = (int8_t) val; + + append_fixed_bytes(c, &v, 1); + } + else + { + int16_t v = (int16_t) val; + + append_fixed_bytes(c, &v, 2); + } + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_STRING || k == CHC_JSON || k == CHC_OBJECT) + { + if (isnull) + append_string_row(c, NULL, 0); + else + append_string_row(c, p, n); + MemoryContextSwitchTo(old); + return; + } + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: bytes into non-text column"))); +} + +void +ch_binary_append_decimal(ch_binary_insert_handle * h, size_t col, + const char *digits, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + size_t w; + + switch (k) + { + case CHC_DECIMAL32: + w = 4; + break; + case CHC_DECIMAL64: + w = 8; + break; + case CHC_DECIMAL128: + w = 16; + break; + case CHC_DECIMAL256: + w = 32; + break; + default: + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: decimal into non-decimal column"))); + } + uint8_t raw[32] = {}; + + if (!isnull && digits) + { + uint32_t scale = (uint32_t) chc_type_decimal_scale(t); + + decimal_text_to_bytes(digits, scale, w, raw); + } + append_fixed_bytes(c, raw, w); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_uuid(ch_binary_insert_handle * h, size_t col, + const uint8_t bytes[16], bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + uint8_t wire[16] = {}; + + if (!isnull) + { + uint64_t a, + b; + + memcpy(&a, bytes, 8); + memcpy(&b, bytes + 8, 8); + a = pg_ntoh64(a); + b = pg_ntoh64(b); + memcpy(wire, &a, 8); + memcpy(wire + 8, &b, 8); + } + append_fixed_bytes(c, wire, 16); + MemoryContextSwitchTo(old); +} + +/* + * addr_be is BE bytes (PG inet ip_addr layout). For IPv4 CH wire is a + * host-order uint32; pg_ntoh32 turns BE bytes into the right host value. + * For IPv6 CH wire matches PG byte order. + */ +void +ch_binary_append_inet(ch_binary_insert_handle * h, size_t col, + const uint8_t * addr_be, size_t addrlen, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + + if (k == CHC_IPV4 && addrlen == 4) + { + uint32_t addr = 0; + + if (!isnull && addr_be) + { + uint32_t be; + + memcpy(&be, addr_be, 4); + addr = pg_ntoh32(be); + } + append_fixed_bytes(c, &addr, 4); + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_IPV6 && addrlen == 16) + { + uint8_t raw[16] = {}; + + if (!isnull && addr_be) + memcpy(raw, addr_be, 16); + append_fixed_bytes(c, raw, 16); + MemoryContextSwitchTo(old); + return; + } + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: cannot insert inet into non-inet column"))); +} + +void +ch_binary_append_date_seconds(ch_binary_insert_handle * h, size_t col, + int64_t seconds, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + + if (k == CHC_DATE) + { + uint16_t days = isnull ? 0 : (uint16_t) (seconds / 86400); + + append_fixed_bytes(c, &days, 2); + } + else if (k == CHC_DATE32) + { + int32_t days = isnull ? 0 : (int32_t) (seconds / 86400); + + append_fixed_bytes(c, &days, 4); + } + else + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: date into non-date column"))); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_datetime_seconds(ch_binary_insert_handle * h, size_t col, + int64_t seconds, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + uint32_t v = isnull ? 0 : (uint32_t) seconds; + + append_fixed_bytes(c, &v, 4); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_datetime64_raw(ch_binary_insert_handle * h, size_t col, + int64_t raw, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + int64_t v = isnull ? 0 : raw; + + append_fixed_bytes(c, &v, 8); + MemoryContextSwitchTo(old); +} + +void +ch_binary_array_begin(ch_binary_insert_handle * h, size_t col) +{ + /* + * Nested arrays recurse via append_one with col=0, so once an array is + * open the caller's col is meaningless, resolve from array_col_idx. + */ + size_t idx = h->array_active ? h->array_col_idx : col; + + if (idx >= h->ncols) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: col out of range"))); + ic_col *c = &h->cols[idx]; + + if (c->layout != IC_ARRAY_FIXED && c->layout != IC_ARRAY_STRING + && c->layout != IC_ARRAY_NESTED_FIXED && c->layout != IC_ARRAY_NESTED_STRING) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: column is not Array"))); + if (c->array_depth >= c->ndim) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: nesting exceeds column ndim"))); + if (c->array_depth == 0) + { + if (c->is_nullable) + { + uint8_t b = 0; + MemoryContext old = MemoryContextSwitchTo(h->cxt); + + dynbuf_append(&c->nulls, &b, 1); + MemoryContextSwitchTo(old); + } + h->array_active = true; + h->array_col_idx = idx; + } + c->array_depth++; +} + +void +ch_binary_array_end(ch_binary_insert_handle * h) +{ + if (!h->array_active) + return; + ic_col *c = &h->cols[h->array_col_idx]; + + if (c->array_depth == 0) + return; + + uint64_t end; + + /* + * At innermost depth, count is leaf-element count; at outer levels the + * count is total children written so far at the level below. + */ + if (c->array_depth == c->ndim) + { + if (c->layout == IC_ARRAY_FIXED || c->layout == IC_ARRAY_NESTED_FIXED) + end = c->elem_size ? (uint64_t) (c->body.len / c->elem_size) : 0; + else + end = c->body_offs.len; + } + else + end = c->arr_offs_inner[c->array_depth - 1].len; + + MemoryContext old = MemoryContextSwitchTo(h->cxt); + + if (c->array_depth == 1) + { + u64buf_push(&c->arr_offs, end); + c->n_rows++; + h->array_active = false; + } + else + u64buf_push(&c->arr_offs_inner[c->array_depth - 2], end); + MemoryContextSwitchTo(old); + c->array_depth--; +} + +bool +ch_binary_array_active(const ch_binary_insert_handle * h) +{ + return h && h->array_active; +} + +chc_kind +ch_binary_column_kind(const ch_binary_insert_handle * h, size_t col) +{ + if (col >= h->ncols) + return CHC_VOID; + const ic_col *c = &h->cols[col]; + + if (h->array_active) + { + c = &h->cols[h->array_col_idx]; + + /* + * While nested, surface CHC_ARRAY until the innermost layer is open; + * at that point return the leaf kind so encode targets scalars. + */ + if (c->array_depth < c->ndim) + return CHC_ARRAY; + return chc_type_kind(c->elem_t); + } + if (c->layout == IC_ARRAY_FIXED || c->layout == IC_ARRAY_STRING + || c->layout == IC_ARRAY_NESTED_FIXED || c->layout == IC_ARRAY_NESTED_STRING) + return CHC_ARRAY; + if (c->layout == IC_LC_STRING) + return CHC_STRING; /* PG side targets TEXT */ + return chc_type_kind(c->inner_t); +} + +uint32_t +ch_binary_column_datetime64_precision(const ch_binary_insert_handle * h, size_t col) +{ + if (col >= h->ncols) + return 0; + return h->cols[col].dt64_precision; +} + +/* Dedup map for LowCardinality dict */ +typedef struct lcd_key +{ + const uint8_t *bytes; + size_t len; +} lcd_key; + +typedef struct lcd_entry +{ + uint32 status; + lcd_key key; + uint32 idx; +} lcd_entry; + +#define SH_PREFIX lcd +#define SH_ELEMENT_TYPE lcd_entry +#define SH_KEY_TYPE lcd_key +#define SH_KEY key +#define SH_HASH_KEY(tb, key) hash_bytes((key).bytes, (int) (key).len) +#define SH_EQUAL(tb, a, b) \ + ((a).len == (b).len && memcmp((a).bytes, (b).bytes, (a).len) == 0) +#define SH_SCOPE static inline +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" + +/* Build LC dict (collect unique strings in insertion order) */ +static void +build_lc_dict(ic_col * c, bool nullable, + uint64_t * *out_dict_offs, uint8_t * *out_dict_data, + size_t * out_dict_n, void **out_keys, int *out_key_size, + size_t * out_n_rows) +{ + size_t n_rows = c->body_offs.len; + uint64_t *dict_offs = NULL; + uint8_t *dict_data = NULL; + uint32_t *keys = n_rows ? palloc(n_rows * sizeof(uint32_t)) : NULL; + size_t dict_n = 0; + size_t dict_cap = 0; + size_t data_len = 0; + lcd_hash *ht = n_rows + ? lcd_create(CurrentMemoryContext, + (uint32) Min(n_rows, (size_t) PG_UINT32_MAX), NULL) + : NULL; + + if (nullable) + { + /* dict[0] = "" sentinel. */ + dict_cap = 8; + dict_offs = palloc(dict_cap * sizeof(uint64_t)); + dict_offs[0] = 0; + dict_n = 1; + } + + for (size_t i = 0; i < n_rows; i++) + { + uint64_t start = i == 0 ? 0 : c->body_offs.data[i - 1]; + uint64_t end = c->body_offs.data[i]; + size_t len = (size_t) (end - start); + const uint8_t *bytes = c->body.data + start; + lcd_key k = {bytes, len}; + lcd_entry *entry; + bool found; + + /* If nullable and this row was null (signalled by null bit), key 0. */ + if (nullable && c->nulls.data[i]) + { + keys[i] = 0; + continue; + } + + entry = lcd_insert(ht, k, &found); + if (found) + { + keys[i] = entry->idx; + continue; + } + + if (dict_n == dict_cap) + { + dict_cap = dict_cap ? dict_cap * 2 : 64; + dict_offs = dict_offs + ? repalloc(dict_offs, dict_cap * sizeof(uint64_t)) + : palloc(dict_cap * sizeof(uint64_t)); + } + data_len += len; + dict_offs[dict_n] = data_len; + entry->idx = (uint32) dict_n; + keys[i] = (uint32) dict_n; + dict_n++; + } + + if (data_len) + { + lcd_iterator it; + lcd_entry *e; + + dict_data = MemoryContextAllocExtended(CurrentMemoryContext, data_len, + MCXT_ALLOC_HUGE); + lcd_start_iterate(ht, &it); + while ((e = lcd_iterate(ht, &it)) != NULL) + { + uint64_t s = e->idx == 0 ? 0 : dict_offs[e->idx - 1]; + + memcpy(dict_data + s, e->key.bytes, e->key.len); + } + } + if (ht) + lcd_destroy(ht); + *out_dict_offs = dict_offs; + *out_dict_data = dict_data; + *out_dict_n = dict_n; + *out_keys = keys; + *out_key_size = 4; + *out_n_rows = n_rows; +} + +void +ch_binary_flush_block(ch_binary_insert_handle * h) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + chc_err err = {}; + chc_block_builder *bb = NULL; + int rc = chc_block_builder_init(&bb, &pg_chc_alloc, &err); + + if (rc != CHC_OK) + raise_chc(&err, ERRCODE_FDW_ERROR, "could not append data to column - "); + + for (size_t i = 0; i < h->ncols; i++) + { + ic_col *c = &h->cols[i]; + const char *name = c->info.name ? c->info.name : ""; + size_t nlen = strlen(name); + + switch (c->layout) + { + case IC_FIXED: + { + size_t n_rows = c->elem_size ? c->body.len / c->elem_size : 0; + + if (c->is_nullable) + rc = chc_block_builder_append_nullable_fixed(bb, name, nlen, c->t, + c->nulls.data, c->body.data, + n_rows, &err); + else + rc = chc_block_builder_append_fixed(bb, name, nlen, c->t, + c->body.data, n_rows, &err); + break; + } + case IC_STRING: + { + size_t n_rows = c->body_offs.len; + + if (c->is_nullable) + rc = chc_block_builder_append_nullable_string(bb, name, nlen, c->t, + c->nulls.data, + c->body_offs.data, + c->body.data, + n_rows, &err); + else + rc = chc_block_builder_append_string(bb, name, nlen, + c->body_offs.data, + c->body.data, + n_rows, &err); + break; + } + case IC_JSON_STRING: + { + /* + * JSON columns share IC_STRING's per-row accumulator; the + * library emits the 8-byte SerializationObject::STRING + * prefix and writes rows as String-Binary. + */ + size_t n_rows = c->body_offs.len; + + rc = chc_block_builder_append_json_string(bb, name, nlen, c->t, + c->body_offs.data, + c->body.data, + n_rows, &err); + break; + } + case IC_LC_STRING: + { + bool nullable_inner = chc_type_kind(chc_type_child(c->inner_t, 0)) == CHC_NULLABLE; + size_t dict_n, + n_rows; + int key_size; + uint64_t *lc_offs; + uint8_t *lc_data; + void *lc_keys; + + build_lc_dict(c, nullable_inner, &lc_offs, &lc_data, + &dict_n, &lc_keys, &key_size, &n_rows); + rc = chc_block_builder_append_low_cardinality_string(bb, name, nlen, c->t, + key_size, lc_keys, + lc_offs, lc_data, + dict_n, n_rows, &err); + break; + } + case IC_ARRAY_FIXED: + { + size_t n_rows = c->arr_offs.len; + + rc = chc_block_builder_append_array_fixed(bb, name, nlen, c->t, + c->arr_offs.data, + c->body.data, + n_rows, &err); + break; + } + case IC_ARRAY_STRING: + { + size_t n_rows = c->arr_offs.len; + + rc = chc_block_builder_append_array_string(bb, name, nlen, c->t, + c->arr_offs.data, + c->body_offs.data, + c->body.data, + n_rows, &err); + break; + } + case IC_ARRAY_NESTED_FIXED: + case IC_ARRAY_NESTED_STRING: + { + size_t n_rows = c->arr_offs.len; + const uint64_t **lvl_offsets = palloc((size_t) c->ndim * sizeof(*lvl_offsets)); + size_t *lvl_lens = palloc((size_t) c->ndim * sizeof(*lvl_lens)); + + lvl_offsets[0] = c->arr_offs.data; + lvl_lens[0] = c->arr_offs.len; + for (int lvl = 1; lvl < c->ndim; lvl++) + { + lvl_offsets[lvl] = c->arr_offs_inner[lvl - 1].data; + lvl_lens[lvl] = c->arr_offs_inner[lvl - 1].len; + } + if (c->layout == IC_ARRAY_NESTED_FIXED) + rc = chc_block_builder_append_array_nested_fixed( + bb, name, nlen, c->t, c->ndim, + lvl_offsets, lvl_lens, + c->body.data, n_rows, &err); + else + rc = chc_block_builder_append_array_nested_string( + bb, name, nlen, c->t, c->ndim, + lvl_offsets, lvl_lens, + c->body_offs.data, c->body.data, + n_rows, &err); + break; + } + } + if (rc != CHC_OK) + raise_chc(&err, ERRCODE_FDW_ERROR, + "could not append data to column - "); + } + + rc = chc_client_send_data(h->client, bb, &err); + chc_block_builder_destroy(bb); + if (rc != CHC_OK) + { + if (h->state) + h->state->broken = true; + raise_chc(&err, ERRCODE_FDW_ERROR, "could not insert columns - "); + } + + /* Reset per-column buffers for next batch. */ + for (size_t i = 0; i < h->ncols; i++) + { + ic_col *c = &h->cols[i]; + + dynbuf_reset(&c->body); + dynbuf_reset(&c->nulls); + u64buf_reset(&c->body_offs); + u64buf_reset(&c->arr_offs); + if (c->arr_offs_inner) + { + for (int lvl = 0; lvl + 1 < c->ndim; lvl++) + u64buf_reset(&c->arr_offs_inner[lvl]); + } + c->n_rows = 0; + } + MemoryContextSwitchTo(old); +} + +/* + * Send final empty Data + drain. May ereport on server exception or + * transport failure. Idempotent via h->finalized. Leaves h->cxt alive; + * ch_binary_release_insert deletes it. + */ +void +ch_binary_finalize_insert(ch_binary_insert_handle * h) +{ + if (!h || h->finalized) + return; + + /* + * Set early so an ereport(ERROR) below still leaves h in the "do not + * touch the wire" state for the release callback. + */ + h->finalized = true; + + if (!(h->started && h->client)) + return; + + MemoryContext old = MemoryContextSwitchTo(h->cxt); + char *exc_msg = NULL; + bool broke = false; + chc_err err = {}; + int rc = chc_client_send_data(h->client, NULL, &err); + + if (rc != CHC_OK) + { + broke = true; + exc_msg = pstrdup(err.msg[0] ? err.msg : "send_data failed"); + } + else + { + /* Drain until EOS or exception. */ + for (;;) + { + chc_packet pkt = {}; + + err = (chc_err) + { + 0 + }; + rc = chc_client_recv_packet(h->client, &pkt, &err); + if (rc != CHC_OK) + { + broke = true; + exc_msg = pstrdup(err.msg[0] ? err.msg : "recv_packet failed"); + chc_packet_clear(h->client, &pkt); + break; + } + if (pkt.kind == CHC_PKT_EXCEPTION) + { + const char *msg = "server exception"; + + if (pkt.exception && pkt.exception->display_text) + msg = pkt.exception->display_text; + else if (pkt.exception && pkt.exception->name) + msg = pkt.exception->name; + exc_msg = pstrdup(msg); + broke = true; + chc_packet_clear(h->client, &pkt); + break; + } + chc_packet_clear(h->client, &pkt); + if (pkt.kind == CHC_PKT_END_OF_STREAM) + break; + } + } + + /* + * Server raised mid-INSERT and typically closes the socket; the next op + * would EPIPE. Mark broken so the cache rebuilds. + */ + if (broke && h->state) + h->state->broken = true; + + MemoryContextSwitchTo(old); + + if (exc_msg) + { + /* exc_msg lives in h->cxt; copy into parent before raising. */ + char *parent_msg = pstrdup(exc_msg); + + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: could not finish INSERT - %s", parent_msg))); + } +} + +/* + * Teardown counterpart to finalize. Safe from a MemoryContext reset + * callback: never raises, never talks to the server. If finalize did not + * run (mid-query abort), flags the connection broken so it rebuilds on + * next use. + */ +void +ch_binary_release_insert(ch_binary_insert_handle * h) +{ + if (!h) + return; + + if (!h->finalized && h->started && h->state) + h->state->broken = true; + + MemoryContextDelete(h->cxt); +} diff --git a/src/binary/select.c b/src/binary/select.c new file mode 100644 index 00000000..78386438 --- /dev/null +++ b/src/binary/select.c @@ -0,0 +1,462 @@ +/* + * select.c + * + * Submit a SELECT (chc_client_send_query_ex) and pump Data packets one block + * at a time. decode.c pulls the next block via + * ch_binary_response_fetch_next_block when it exhausts the current one, so + * peak memory is bounded by one block plus what postgres holds for the + * current row. Settings come from the foreign-table KV list; we also add + * output_format_native_write_json_as_string=1 against servers that understand + * it. Cancel polling drives chc_io's per-read callback; server-side + * exceptions flag the connection as broken so the cache drops it. Premature + * close (eg LIMIT, error in decode, transaction abort) sends Cancel + * + drains in ch_binary_response_free so the connection is reusable. + */ + +#include "postgres.h" + +#include +#include +#include + +#include "utils/memutils.h" +#include "utils/palloc.h" + +#include "binary_internal.h" +#include "kv_list.h" + +/* Wall-clock cap on drain_until_eos. A well-behaved server acknowledges + * Cancel by flushing the in-flight block + EndOfStream within a second. */ +#define DRAIN_DEADLINE_US (5 * 1000 * 1000) + +struct ch_binary_response_t +{ + MemoryContext cxt; + struct ch_binary_state *state; /* parent connection state */ + chc_client *client; /* recv_packet target */ + bool (*check_cancel) (void); + + char *error; /* NULL on success */ + bool success; + bool eos; /* END_OF_STREAM or exception seen */ + + size_t columns_count; + chc_block *staged; /* next block to hand out; owned */ + chc_block *prev; /* last block handed out; freed at next fetch */ +}; + +static void +resp_set_error(ch_binary_response_t * resp, const char *msg) +{ + if (resp->error) + return; + resp->error = pstrdup(msg && *msg ? msg : "?"); +} + +static void +resp_set_exception(ch_binary_response_t * resp, const chc_exception * ex) +{ + if (resp->error) + return; + const char *msg = NULL; + + if (ex) + { + if (ex->display_text && ex->display_text[0]) + msg = ex->display_text; + else if (ex->name && ex->name[0]) + msg = ex->name; + } + resp->error = pstrdup(msg ? msg : "server exception"); +} + +/* + * Read one wire packet under resp->cxt and fold it into the response. Sets + * resp->staged (for non-empty Data), resp->columns_count (on first Data with + * columns), resp->error (cancel/exception/transport), or resp->eos + * (END_OF_STREAM / exception / transport failure). Other packet kinds + * (progress, log, profile, ...) are silently consumed. + */ +static void +pump_one(ch_binary_response_t * resp) +{ + MemoryContext old = MemoryContextSwitchTo(resp->cxt); + chc_packet pkt = {}; + chc_err err = {}; + int rc = chc_client_recv_packet(resp->client, &pkt, &err); + + if (rc != CHC_OK) + { + resp_set_error(resp, err.msg); + resp->state->broken = true; + resp->eos = true; + MemoryContextSwitchTo(old); + return; + } + + if (resp->check_cancel && resp->check_cancel()) + resp_set_error(resp, "query was canceled"); + + switch (pkt.kind) + { + case CHC_PKT_DATA: + if (pkt.block && chc_block_n_columns(pkt.block) > 0) + { + if (resp->columns_count == 0) + resp->columns_count = chc_block_n_columns(pkt.block); + if (chc_block_n_rows(pkt.block) > 0 && resp->staged == NULL) + { + size_t ncols = chc_block_n_columns(pkt.block); + chc_err verr = {}; + int vrc = CHC_OK; + + for (size_t i = 0; i < ncols; i++) + { + vrc = chc_column_validate(chc_block_column(pkt.block, i), &verr); + if (vrc != CHC_OK) + break; + } + if (vrc != CHC_OK) + { + resp_set_error(resp, verr.msg); + resp->state->broken = true; + resp->eos = true; + } + else + { + resp->staged = pkt.block; + pkt.block = NULL; + } + } + } + chc_packet_clear(resp->client, &pkt); + break; + + case CHC_PKT_EXCEPTION: + resp_set_exception(resp, pkt.exception); + chc_packet_clear(resp->client, &pkt); + + /* + * Older servers (and some protocol states) close the socket after + * raising an exception, so reusing this connection for a + * follow-up query risks EPIPE. Match the legacy C++ driver (which + * always called Client::ResetConnection) and treat the connection + * as broken. + */ + resp->state->broken = true; + resp->eos = true; + break; + + case CHC_PKT_END_OF_STREAM: + chc_packet_clear(resp->client, &pkt); + resp->eos = true; + break; + + default: + chc_packet_clear(resp->client, &pkt); + break; + } + + MemoryContextSwitchTo(old); +} + +static int64_t +drain_now_us(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return (int64_t) ts.tv_sec * 1000000 + ts.tv_nsec / 1000; +} + +static void +drain_set_deadline(struct ch_binary_state *s, int64_t deadline_us) +{ + if (s->tls) + chc_openssl_io_set_deadline(&s->openssl_state, deadline_us); + else + chc_posix_io_set_deadline(&s->posix_state, deadline_us); +} + +/* + * Best-effort drain of any unconsumed stream so the connection is left clean + * for the next query. Disables cancel polling while draining so + * QueryCancelPending doesn't short-circuit every refill, and caps total wait + * with an IO deadline so a peer that ignores Cancel doesn't cause hang. Sets + * state->broken on transport failure / timeout / send_cancel failure so the + * cache drops the connection. + */ +static void +drain_until_eos(ch_binary_response_t * resp) +{ + if (resp->eos) + return; + + MemoryContext old = MemoryContextSwitchTo(resp->cxt); + chc_err ce = {}; + + if (chc_client_send_cancel(resp->client, &ce) != CHC_OK) + { + resp->state->broken = true; + resp->eos = true; + MemoryContextSwitchTo(old); + return; + } + resp->state->check_cancel_fn = NULL; + resp->check_cancel = NULL; + + drain_set_deadline(resp->state, drain_now_us() + DRAIN_DEADLINE_US); + + for (;;) + { + chc_packet pkt = {}; + int rc; + + ce = (chc_err) + { + 0 + }; + rc = chc_client_recv_packet(resp->client, &pkt, &ce); + + if (rc != CHC_OK) + { + resp->state->broken = true; + resp->eos = true; + break; + } + bool stop = pkt.kind == CHC_PKT_END_OF_STREAM || + pkt.kind == CHC_PKT_EXCEPTION; + + chc_packet_clear(resp->client, &pkt); + if (stop) + { + resp->eos = true; + break; + } + } + + drain_set_deadline(resp->state, 0); + MemoryContextSwitchTo(old); +} + +ch_binary_response_t * +ch_binary_simple_query(ch_binary_connection_t * conn, const ch_query * query, + bool (*check_cancel) (void)) +{ + struct ch_binary_state *s = conn_state(conn); + MemoryContext cxt = AllocSetContextCreate(CurrentMemoryContext, + "pg_clickhouse binary response", + ALLOCSET_DEFAULT_SIZES); + MemoryContext old = MemoryContextSwitchTo(cxt); + ch_binary_response_t *resp = palloc0(sizeof(*resp)); + + resp->cxt = cxt; + resp->state = s; + resp->client = s->client; + resp->check_cancel = check_cancel; + s->check_cancel_fn = check_cancel; + + size_t n_user_settings = 0; + size_t n_params = (size_t) (query->num_params > 0 ? query->num_params : 0); + chc_query_setting *settings = NULL; + chc_query_param *params = NULL; + bool want_json_as_string = server_supports_json_as_string(s->client); + + { + const kv_list *kv = query->settings; + + if (kv) + n_user_settings = (size_t) kv->length; + } + size_t n_settings = n_user_settings + (want_json_as_string ? 1 : 0); + + if (n_settings) + { + settings = palloc0(n_settings * sizeof(*settings)); + size_t i = 0; + + for (kv_iter it = new_kv_iter(query->settings); !kv_iter_done(&it); kv_iter_next(&it), i++) + { + settings[i].name = it.name; + settings[i].value = it.value; + settings[i].important = true; + } + if (want_json_as_string) + { + settings[i].name = "output_format_native_write_json_as_string"; + settings[i].value = "1"; + settings[i].important = true; + } + } + if (n_params) + { + params = palloc0(n_params * sizeof(*params)); + for (size_t i = 0; i < n_params; i++) + { + char nm[32]; + + snprintf(nm, sizeof(nm), "p%zu", i + 1); + params[i].name = pstrdup(nm); + + /* + * Quote & escape the value the way clickhouse-cpp's + * WriteQuotedString did: wrap in single quotes, replace inner + * specials with backslash-escapes the server's + * Field::restoreFromDump understands. Without escaping inner + * quotes the server stops parsing at the first `'` inside the + * value, which breaks Array(String) parameters whose CH literal + * already contains quoted elements. + */ + const char *raw = query->param_values[i]; + + if (raw) + { + size_t rlen = strlen(raw); + size_t cap = rlen * 4 + 3; + char *dst = palloc(cap); + size_t o = 0; + + dst[o++] = '\''; + for (size_t j = 0; j < rlen; j++) + { + unsigned char ch = (unsigned char) raw[j]; + + switch (ch) + { + case '\0': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '0'; + dst[o++] = '0'; + break; + case '\b': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '0'; + dst[o++] = '8'; + break; + case '\t': + dst[o++] = '\\'; + dst[o++] = 't'; + break; + case '\n': + dst[o++] = '\\'; + dst[o++] = 'n'; + break; + case '\'': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '2'; + dst[o++] = '7'; + break; + case '\\': + dst[o++] = '\\'; + dst[o++] = '\\'; + break; + default: + dst[o++] = (char) ch; + } + } + dst[o++] = '\''; + dst[o] = '\0'; + params[i].value = dst; + } + else + params[i].value = "'\\N'"; + } + } + + chc_query_opts opts = { + .settings = settings, + .n_settings = n_settings, + .params = params, + .n_params = n_params, + }; + chc_err err = {}; + int rc = chc_client_send_query_ex(s->client, query->sql, + strlen(query->sql), &opts, &err); + + if (rc != CHC_OK) + { + resp_set_error(resp, err.msg); + s->broken = true; + resp->eos = true; + goto done; + } + + /* + * Pump until schema is known so callers can call + * ch_binary_response_columns immediately. Empty result sets exit on eos + * with columns_count still set from the header Data block. + */ + while (resp->columns_count == 0 && !resp->eos && !resp->error) + pump_one(resp); + +done: + resp->success = (resp->error == NULL); + MemoryContextSwitchTo(old); + return resp; +} + +void +ch_binary_response_free(ch_binary_response_t * resp) +{ + if (!resp) + return; + + /* Avoid raising from a MemoryContextResetCallback. */ + PG_TRY(); + { + drain_until_eos(resp); + } + PG_CATCH(); + { + FlushErrorState(); + resp->state->broken = true; + } + PG_END_TRY(); + + resp->state->check_cancel_fn = NULL; + MemoryContextDelete(resp->cxt); +} + +const char * +ch_binary_response_error(const ch_binary_response_t * resp) +{ + return resp ? resp->error : NULL; +} + +bool +ch_binary_response_success(const ch_binary_response_t * resp) +{ + return resp && resp->success; +} + +size_t +ch_binary_response_columns(const ch_binary_response_t * resp) +{ + return resp ? resp->columns_count : 0; +} + +const chc_block * +ch_binary_response_fetch_next_block(ch_binary_response_t * resp) +{ + if (!resp) + return NULL; + + if (resp->prev) + { + chc_block_destroy(resp->prev, &pg_chc_alloc); + resp->prev = NULL; + } + + while (resp->staged == NULL && !resp->eos && !resp->error) + pump_one(resp); + + if (resp->staged == NULL) + return NULL; + + resp->prev = resp->staged; + resp->staged = NULL; + return resp->prev; +} diff --git a/src/connection.c b/src/connection.c index 86959382..2032431f 100644 --- a/src/connection.c +++ b/src/connection.c @@ -123,10 +123,20 @@ chfdw_get_connection(UserMapping * user) } /* - * We don't check the health of cached connection here, because it would - * require some overhead. Broken connection will be detected when the - * connection is actually used. + * Drop connections that hit an unrecoverable protocol/IO error on the + * previous statement (server raised mid-INSERT and closed the socket, + * write hit EPIPE, etc). Without this, a subsequent statement would write + * to the dead socket and surface a useless "Broken pipe" instead of the + * real error from the next request. */ + if (entry->gate.conn != NULL && + entry->gate.methods->is_broken != NULL && + entry->gate.methods->is_broken(entry->gate.conn)) + { + elog(LOG, "closing broken pg_clickhouse connection"); + entry->gate.methods->disconnect(entry->gate.conn); + entry->gate.conn = NULL; + } /* * If cache entry doesn't have a connection, we have to establish a new diff --git a/src/fdw.c b/src/fdw.c index 20ca5ed6..5a798b88 100644 --- a/src/fdw.c +++ b/src/fdw.c @@ -2,16 +2,18 @@ A PostgreSQL extension for connecting to ClickHouse servers. */ +#include + /* PostgreSQL includes. */ #include "postgres.h" #include "catalog/pg_class_d.h" #include "commands/defrem.h" -#include "commands/explain.h" #include "foreign/fdwapi.h" #include "funcapi.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#include "nodes/parsenodes.h" #include "optimizer/clauses.h" #include "optimizer/cost.h" #include "optimizer/pathnode.h" @@ -32,12 +34,12 @@ #if PG_VERSION_NUM >= 180000 #include "commands/explain_format.h" #include "commands/explain_state.h" +#else +#include "commands/explain.h" #endif /* extension includes. */ #include "utils/builtins.h" -#include "binary.hh" -#include "internal.h" #include "fdw.h" #include "version.h" @@ -1420,6 +1422,15 @@ clickhouseEndForeignInsert(EState * estate, /* flush */ oldcontext = MemoryContextSwitchTo(fmstate->temp_cxt); fmstate->conn.methods->insert_tuple(fmstate->state, NULL); + + /* + * Finalize on the happy path so the binary driver can ereport on a + * server-side INSERT exception without raising from inside the + * MemoryContext reset callback that fires during abort. + */ + if (fmstate->conn.methods->finalize_insert) + fmstate->conn.methods->finalize_insert(fmstate->state); + MemoryContextSwitchTo(oldcontext); MemoryContextReset(fmstate->temp_cxt); diff --git a/src/include/binary.h b/src/include/binary.h new file mode 100644 index 00000000..3ca9b7bd --- /dev/null +++ b/src/include/binary.h @@ -0,0 +1,116 @@ +/* + * binary.h + * + * API exposed by the binary driver. Allocates against caller's + * CurrentMemoryContext at every public entry point and ereports on + * error. Returned objects own a dedicated MemoryContext; explicit + * *_free / *_close calls do MemoryContextDelete. + */ + +#ifndef CLICKHOUSE_BINARY_H +#define CLICKHOUSE_BINARY_H + +#include "postgres.h" + +#include +#include +#include + +#include "access/tupdesc.h" +#include "executor/tuptable.h" +#include "utils/palloc.h" + +#include "clickhouse.h" +#include "engine.h" + +typedef struct ch_binary_connection_t ch_binary_connection_t; +typedef struct ch_binary_response_t ch_binary_response_t; +typedef struct ch_binary_insert_handle ch_binary_insert_handle; + +/* Connection. */ +extern ch_binary_connection_t * ch_binary_connect(ch_connection_details * details); +extern void ch_binary_close(ch_binary_connection_t * conn); + +/* + * Returns true if connection encountered an unrecoverable error + * (server exception, IO failure, mid-protocol break). Callers should + * drop cached connection instead of reusing it. + */ +extern bool ch_binary_is_broken(const ch_binary_connection_t * conn); + +/* SELECT. */ +extern ch_binary_response_t * ch_binary_simple_query(ch_binary_connection_t * conn, + const ch_query * query, + bool (*check_cancel) (void)); +extern void ch_binary_response_free(ch_binary_response_t * resp); +extern const char *ch_binary_response_error(const ch_binary_response_t * resp); +extern bool ch_binary_response_success(const ch_binary_response_t * resp); +extern size_t ch_binary_response_columns(const ch_binary_response_t * resp); + +/* INSERT. */ + +/* + * Finish the insert: send final empty Data, drain the response, ereport + * if the server raised. Idempotent; call exactly once from the FDW happy + * path before tearing down the handle. + */ +extern void ch_binary_finalize_insert(ch_binary_insert_handle * h); + +/* PG-typed surface follows. */ + +typedef struct +{ + ch_binary_response_t *resp; + Oid *coltypes; + Datum *values; + bool *nulls; + + size_t block; /* current block */ + size_t row; /* row in current block */ + const chc_block *cur; /* borrowed from resp; NULL when unloaded */ + void *gc; /* allocated objects while reading */ + char *error; + bool done; +} ch_binary_read_state_t; + +typedef struct +{ + MemoryContext memcxt; /* used for cleanup */ + MemoryContextCallback callback; + + TupleDesc outdesc; + ch_binary_insert_handle *insert_block; + size_t len; + void *conversion_states; + char *table_name; + Oid relid; /* foreign table relid, for column_name + * lookups */ + + Datum *values; + bool *nulls; + bool success; + + ch_binary_connection_t *conn; +} ch_binary_insert_state; + +/* SELECT helpers (decode.c). */ +extern void ch_binary_read_state_init(ch_binary_read_state_t * state, ch_binary_response_t * resp); +extern void ch_binary_read_state_free(ch_binary_read_state_t * state); +extern bool ch_binary_read_row(ch_binary_read_state_t * state); + +/* SELECT/INSERT type conversion (convert.c). */ +extern Datum ch_binary_convert_datum(void *state, Datum val); +extern void *ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype); +extern void ch_binary_free_convert_state(void *state); + +/* INSERT helpers (encode.c). */ +extern void ch_binary_prepare_insert(void *conn, const ch_query * query, + ch_binary_insert_state * state); +extern void ch_binary_insert_columns(ch_binary_insert_state * state); +extern void ch_binary_column_append_data(ch_binary_insert_state * state, size_t colidx); +extern void *ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid); +extern void ch_binary_insert_state_free(void *c); +extern void ch_binary_do_output_conversion(ch_binary_insert_state * state, + TupleTableSlot * slot); + +#endif /* CLICKHOUSE_BINARY_H */ diff --git a/src/include/binary.hh b/src/include/binary.hh deleted file mode 100644 index e3ce0d91..00000000 --- a/src/include/binary.hh +++ /dev/null @@ -1,110 +0,0 @@ -#ifndef CLICKHOUSE_BINARY_H -#define CLICKHOUSE_BINARY_H - -#include "engine.h" - -#ifdef __cplusplus -extern "C" -{ -#endif - - typedef struct ch_binary_connection_t ch_binary_connection_t; - typedef struct ch_insert_block_h ch_insert_block_h; - typedef struct ch_binary_response_t - { - void *values; - size_t columns_count; - size_t blocks_count; - char *error; - bool success; - } ch_binary_response_t; - - typedef struct - { - ch_binary_response_t *resp; - Oid *coltypes; - Datum *values; - bool *nulls; - - size_t block; /* current block */ - size_t row; /* row in current block */ - void *gc; /* allocated objects while reading */ - char *error; - bool done; - } ch_binary_read_state_t; - - typedef struct - { - Datum *datums; - bool *nulls; - size_t len; - Oid *types; - } ch_binary_tuple_t; - - /* - * Holds an array read from ClickHouse. For nested arrays - * (Array(Array(...))) ndim > 1 and datums[i] points to a child - * ch_binary_array_t with ndim-1. item_type is the leaf scalar type, - * array_type is the postgres array type (same at every level since - * postgres uses one array type per element type regardless of - * dimensionality). - */ - typedef struct - { - Datum *datums; - bool *nulls; - size_t len; - int ndim; /* nesting depth, ≥1 (used on selects) */ - Oid item_type; /* leaf element type (used on selects) */ - Oid array_type; /* used on selects */ - } ch_binary_array_t; - - typedef struct - { - MemoryContext memcxt; /* used for cleanup */ - MemoryContextCallback callback; - - TupleDesc outdesc; - ch_insert_block_h *insert_block; /* clickhouse::Block */ - size_t len; - void *conversion_states; - char *table_name; - Oid relid; /* foreign table relid, for column_name - * lookups */ - - Datum *values; - bool *nulls; - bool success; - - ch_binary_connection_t *conn; - } ch_binary_insert_state; - - extern ch_binary_connection_t * ch_binary_connect(ch_connection_details * details, char **error); - extern void ch_binary_close(ch_binary_connection_t * conn); - extern ch_binary_response_t * ch_binary_simple_query(ch_binary_connection_t * conn, - const ch_query * query, bool (*check_cancel) (void)); - extern void ch_binary_response_free(ch_binary_response_t * resp); - -/* reading */ - void ch_binary_read_state_init(ch_binary_read_state_t * state, ch_binary_response_t * resp, const ch_query * query); - void ch_binary_read_state_free(ch_binary_read_state_t * state); - bool ch_binary_read_row(ch_binary_read_state_t * state, TupleDesc tupdesc, List * attrs); - Datum ch_binary_convert_datum(void *state, Datum val); - void *ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype); - void ch_binary_free_convert_state(void *); - -/* insertion */ - void ch_binary_prepare_insert(void *conn, const ch_query * query, - ch_binary_insert_state * state); - void ch_binary_insert_columns(ch_binary_insert_state * state); - void ch_binary_column_append_data(ch_binary_insert_state * state, size_t colidx); - void *ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid); - void ch_binary_insert_state_free(void *c); - void ch_binary_do_output_conversion(ch_binary_insert_state * insert_state, - TupleTableSlot * slot); - -#ifdef __cplusplus -} -#endif - -#endif /* CLICKHOUSE_BINARY_H */ diff --git a/src/include/fdw.h b/src/include/fdw.h index 5fecfc3d..4b711658 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -75,9 +75,11 @@ typedef Datum * (*cursor_fetch_row_method) (ChFdwScanRowContext * ctx); typedef void *(*prepare_insert_method) (void *conn, ResultRelInfo *, List *, const ch_query *, char *); typedef void (*insert_tuple_method) (void *state, TupleTableSlot * slot); +typedef void (*finalize_insert_method) (void *state); typedef ch_cursor * (*streaming_query_method) (void *conn, const ch_query * query, int32 fetch_size); +typedef bool (*is_broken_method) (const void *conn); typedef struct { @@ -86,8 +88,11 @@ typedef struct cursor_fetch_row_method fetch_row; prepare_insert_method prepare_insert; insert_tuple_method insert_tuple; + finalize_insert_method finalize_insert; /* NULL if no explicit finalize + * step needed */ streaming_query_method streaming_query; /* NULL if not supported */ cursor_fetch_row_method streaming_fetch_row; /* NULL if not supported */ + is_broken_method is_broken; /* NULL means connection is never broken */ } libclickhouse_methods; typedef struct diff --git a/src/option.c b/src/option.c index 2ef946b0..f70c9b15 100644 --- a/src/option.c +++ b/src/option.c @@ -303,7 +303,14 @@ chfdw_extract_options(List * defelems, char **driver, char **host, int *port, if (host && strcmp(def->defname, "host") == 0) *host = defGetString(def); else if (port && strcmp(def->defname, "port") == 0) - *port = atoi(defGetString(def)); + { + char *endptr = NULL; + long val = strtol(defGetString(def), &endptr, 10); + + /* Just ignore invalid values. */ + if (val <= INT_MAX && val > 0 && (!*endptr || *endptr == 0)) + *port = val; + } else if (username && strcmp(def->defname, "user") == 0) *username = defGetString(def); else if (password && strcmp(def->defname, "password") == 0) diff --git a/src/pglink.c b/src/pglink.c index 274098d9..56a04c24 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -19,7 +19,7 @@ #include "fdw.h" #include "http.h" #include "http_streaming.h" -#include "binary.hh" +#include "binary.h" #include #include @@ -58,10 +58,12 @@ static libclickhouse_methods http_methods = static void binary_disconnect(void *conn); static ch_cursor * binary_simple_query(void *conn, const ch_query * query); static void binary_cursor_free(void *cursor); +static bool binary_is_broken(const void *conn); /* static void binary_simple_insert(void *conn, const char *query); */ static Datum * binary_fetch_row(ChFdwScanRowContext * ctx); static void binary_insert_tuple(void *, TupleTableSlot * slot); +static void binary_finalize_insert(void *istate); static void *binary_prepare_insert(void *, ResultRelInfo *, List *, const ch_query * query, char *table_name); static char *ch_escape_string(const char *s, size_t len); @@ -76,8 +78,10 @@ static libclickhouse_methods binary_methods = .fetch_row = binary_fetch_row, .prepare_insert = binary_prepare_insert, .insert_tuple = binary_insert_tuple, + .finalize_insert = binary_finalize_insert, .streaming_query = NULL, .streaming_fetch_row = NULL, + .is_broken = binary_is_broken, }; static int @@ -818,31 +822,9 @@ http_insert_tuple(void *istate, TupleTableSlot * slot) ch_connection chfdw_binary_connect(ch_connection_details * details) { - char *ch_error = NULL; ch_connection res; - ch_binary_connection_t *conn = ch_binary_connect(details, &ch_error); - if (conn == NULL) - { - if (ch_error == NULL) - { - ereport(ERROR, - (errcode(ERRCODE_FDW_OUT_OF_MEMORY), - errmsg("out of memory"))); - } - else - { - char *error = pstrdup(ch_error); - - free(ch_error); - - ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: connection error: %s", error))); - } - } - - res.conn = conn; + res.conn = ch_binary_connect(details); res.methods = &binary_methods; res.is_binary = true; return res; @@ -855,6 +837,12 @@ binary_disconnect(void *conn) ch_binary_close((ch_binary_connection_t *) conn); } +static bool +binary_is_broken(const void *conn) +{ + return ch_binary_is_broken((const ch_binary_connection_t *) conn); +} + static ch_cursor * binary_simple_query(void *conn, const ch_query * query) { @@ -865,9 +853,9 @@ binary_simple_query(void *conn, const ch_query * query) ch_binary_response_t *resp = ch_binary_simple_query(conn, query, &is_canceled); - if (!resp->success) + if (!ch_binary_response_success(resp)) { - char *error = pstrdup(resp->error); + char *error = pstrdup(ch_binary_response_error(resp)); ch_binary_response_free(resp); ereport(ERROR, ( @@ -887,10 +875,33 @@ binary_simple_query(void *conn, const ch_query * query) state = (ch_binary_read_state_t *) palloc0(sizeof(ch_binary_read_state_t)); cursor->query = pstrdup(query->sql); cursor->read_state = state; - cursor->columns_count = resp->columns_count; - ch_binary_read_state_init(cursor->read_state, resp, query); + cursor->columns_count = ch_binary_response_columns(resp); + ch_binary_read_state_init(cursor->read_state, resp); cursor->conversion_states = palloc0(sizeof(uintptr_t) * cursor->columns_count); + /* + * CH JSON columns default to JSONBOID in state->coltypes. When foreign + * table column is declared `json` (JSONOID), override so + * binary_make_datum returns json Datum from CH's STRING bytes, skipping + * jsonb_in / jsonb_out round-trip that would reformat CH's emit and break + * expected outputs that pin CH's exact formatting. + */ + if (query->tupdesc && state->coltypes) + { + ListCell *lc; + size_t j = 0; + + foreach(lc, query->attr_nums) + { + int i = lfirst_int(lc); + + if (state->coltypes[j] == JSONBOID && + TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) + state->coltypes[j] = JSONOID; + j++; + } + } + cursor->memcxt = tempcxt; cursor->callback.func = binary_cursor_free; cursor->callback.arg = cursor; @@ -898,12 +909,10 @@ binary_simple_query(void *conn, const ch_query * query) MemoryContextSwitchTo(oldcxt); if (state->error) - { ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: could not initialize read state: %s", - state->error))); - } + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", state->error), + errdetail_internal("Remote Query: %.64000s", query->sql))); return cursor; } @@ -923,6 +932,14 @@ binary_simple_query(void *conn, const ch_query * query) * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, * which only cares about text. */ +static void +binary_fetch_row_errcb(void *arg) +{ + const char *sql = (const char *) arg; + + errdetail_internal("Remote Query: %.64000s", sql); +} + static Datum * binary_fetch_row(ChFdwScanRowContext * ctx) { @@ -933,7 +950,14 @@ binary_fetch_row(ChFdwScanRowContext * ctx) Datum *values = ctx->values; bool *nulls = ctx->nulls; ch_binary_read_state_t *state = cursor->read_state; - bool have_data = ch_binary_read_row(state, tupdesc, attrs); + ErrorContextCallback errcallback; + + errcallback.callback = binary_fetch_row_errcb; + errcallback.arg = (void *) cursor->query; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + bool have_data = ch_binary_read_row(state); size_t attcount = list_length(attrs); if (state->error) @@ -943,11 +967,14 @@ binary_fetch_row(ChFdwScanRowContext * ctx) state->error))); if (!have_data) + { + error_context_stack = errcallback.previous; return NULL; + } if (attcount == 0) { - if (state->resp->columns_count == 1 && state->nulls[0]) + if (ch_binary_response_columns(state->resp) == 1 && state->nulls[0]) { nulls[0] = true; goto ok; @@ -958,14 +985,12 @@ binary_fetch_row(ChFdwScanRowContext * ctx) errmsg("pg_clickhouse: unexpected state: attributes " "count == 0 and haven't got NULL in the response"))); } - else if (attcount != state->resp->columns_count) + else if (attcount != ch_binary_response_columns(state->resp)) { ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal("pg_clickhouse: columns mismatch"), - errdetail("Number of returned columns (%lu) does not match " - "expected column count (%lu).", - state->resp->columns_count, attcount))); + errmsg_internal("pg_clickhouse: returned %lu columns, expected %lu", + ch_binary_response_columns(state->resp), attcount))); } if (tupdesc) @@ -1039,6 +1064,7 @@ binary_fetch_row(ChFdwScanRowContext * ctx) } ok: + error_context_stack = errcallback.previous; return state->values; } @@ -1111,10 +1137,10 @@ binary_insert_tuple(void *istate, TupleTableSlot * slot) /* * Set up conversion states so that Postgres Datums of types * defined by the foreign table can be converted to a well-known - * Postgres type that column_append() in binary.cpp knows how to - * convert to the appropriate ClickHouse type. Binary compatible - * types don't convert; others require a CAST. Fails if there is - * no cast. + * Postgres type that the binary INSERT path knows how to convert + * to the appropriate ClickHouse type. Binary compatible types + * don't convert; others require a CAST. Fails if there is no + * cast. */ state->conversion_states = ch_binary_make_tuple_map(slot->tts_tupleDescriptor, state->outdesc, state->relid); MemoryContextSwitchTo(old_mcxt); @@ -1131,6 +1157,15 @@ binary_insert_tuple(void *istate, TupleTableSlot * slot) } } +static void +binary_finalize_insert(void *istate) +{ + ch_binary_insert_state *state = istate; + + if (state && state->insert_block) + ch_binary_finalize_insert(state->insert_block); +} + /* * Query to generate table for doc/pg_clickhouse.md. Keep in sync with * str_types_map below. On change, re-run and paste the output into @@ -1325,16 +1360,36 @@ chfdw_construct_create_tables(ImportForeignSchemaStmt * stmt, ForeignServer * se NULL }; + /* + * Drain the outer query into private strings before opening the per-table + * column queries: both use the same connection, and binary streaming only + * permits one in-flight response at a time. + */ + List *tables = NIL; + while ((row_values = conn.methods->fetch_row(&tables_ctx)) != NULL) { + List *triple = list_make3(pstrdup(TextDatumGetCString(row_values[0])), + pstrdup(TextDatumGetCString(row_values[1])), + pstrdup(TextDatumGetCString(row_values[2]))); + + CHECK_FOR_INTERRUPTS(); + tables = lappend(tables, triple); + } + MemoryContextDelete(cursor->memcxt); + + ListCell *tlc; + + foreach(tlc, tables) + { + List *triple = (List *) lfirst(tlc); + char *table_name = (char *) linitial(triple); + char *engine = (char *) lsecond(triple); + char *engine_full = (char *) lthird(triple); StringInfoData buf; - char *table_name = TextDatumGetCString(row_values[0]); - char *engine = TextDatumGetCString(row_values[1]); - char *engine_full = TextDatumGetCString(row_values[2]); Datum *dvalues; bool first = true; - CHECK_FOR_INTERRUPTS(); if (table_name == NULL) continue; @@ -1454,7 +1509,6 @@ chfdw_construct_create_tables(ImportForeignSchemaStmt * stmt, ForeignServer * se MemoryContextDelete(cols_ctx.cursor->memcxt); } - MemoryContextDelete(cursor->memcxt); return result; } diff --git a/test/expected/binary.out b/test/expected/binary.out index d73583dc..4185be2b 100644 --- a/test/expected/binary.out +++ b/test/expected/binary.out @@ -309,6 +309,7 @@ SELECT * FROM farrays ORDER BY c1; SELECT * FROM farrays2 ORDER BY c1; ERROR: pg_clickhouse: could not cast value from integer[] to bigint[] +DETAIL: Remote Query: SELECT c1, c2 FROM binary_test.arrays ORDER BY c1 ASC NULLS LAST -- nested arrays SELECT * FROM fnested_arrays ORDER BY c1; c1 | c2 | c3 @@ -319,7 +320,7 @@ SELECT * FROM fnested_arrays ORDER BY c1; SELECT * FROM fragged_arrays ORDER BY c1; ERROR: malformed array literal: "{{"1","2","3"},{"4"}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +DETAIL: Remote Query: SELECT c1, c2 FROM binary_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- tuples SELECT * FROM ftuples ORDER BY c1; c1 | c2 | c3 diff --git a/test/expected/binary_inserts.out b/test/expected/binary_inserts.out index 9dd46567..5620558f 100644 --- a/test/expected/binary_inserts.out +++ b/test/expected/binary_inserts.out @@ -298,7 +298,7 @@ SELECT * FROM nested_arrays ORDER BY c1; /* shape mismatch with column type must error rather than silently corrupt */ INSERT INTO nested_arrays VALUES (4, ARRAY[1,2,3], ARRAY['x']); -ERROR: pg_clickhouse: could not append data to column - pg_clickhouse: insert array has fewer dimensions than column type +ERROR: pg_clickhouse: unexpected PG/CH type pair for column 0 /* Check UUIDs and IPs */ \d addr Foreign table "binary_inserts_test.addr" @@ -490,7 +490,7 @@ INSERT INTO default_vals VALUES( NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); -ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL UInt8 column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL UInt8 column SELECT * FROM default_vals; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c18 | c19 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 ----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- @@ -498,7 +498,7 @@ SELECT * FROM default_vals; -- Test unsupported Nullables. INSERT INTO not_nullable VALUES (1, 'x', 'x', NULL); -ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column DROP USER MAPPING FOR CURRENT_USER SERVER binary_inserts_loopback; SELECT clickhouse_raw_query('DROP DATABASE binary_inserts_test'); clickhouse_raw_query diff --git a/test/expected/binary_inserts_1.out b/test/expected/binary_inserts_1.out index dce76e48..e8b81ed4 100644 --- a/test/expected/binary_inserts_1.out +++ b/test/expected/binary_inserts_1.out @@ -289,7 +289,7 @@ SELECT * FROM nested_arrays ORDER BY c1; /* shape mismatch with column type must error rather than silently corrupt */ INSERT INTO nested_arrays VALUES (4, ARRAY[1,2,3], ARRAY['x']); -ERROR: pg_clickhouse: could not append data to column - pg_clickhouse: insert array has fewer dimensions than column type +ERROR: pg_clickhouse: unexpected PG/CH type pair for column 0 /* Check UUIDs and IPs */ \d addr Foreign table "binary_inserts_test.addr" @@ -481,7 +481,7 @@ INSERT INTO default_vals VALUES( NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); -ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL UInt8 column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL UInt8 column SELECT * FROM default_vals; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c18 | c19 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 ----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- @@ -489,7 +489,7 @@ SELECT * FROM default_vals; -- Test unsupported Nullables. INSERT INTO not_nullable VALUES (1, 'x', 'x', NULL); -ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column DROP USER MAPPING FOR CURRENT_USER SERVER binary_inserts_loopback; SELECT clickhouse_raw_query('DROP DATABASE binary_inserts_test'); clickhouse_raw_query diff --git a/test/expected/binary_queries_5.out b/test/expected/binary_queries_5.out index fa3e305e..8243455e 100644 --- a/test/expected/binary_queries_5.out +++ b/test/expected/binary_queries_5.out @@ -124,7 +124,7 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: DB::Exception: Database `no such database` does not exist +ERROR: pg_clickhouse: connection error: DB::Exception: Database `no such database` doesn't exist ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail ERROR: pg_clickhouse: connection error: DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. diff --git a/test/expected/binary_queries_6.out b/test/expected/binary_queries_6.out deleted file mode 100644 index 7a365603..00000000 --- a/test/expected/binary_queries_6.out +++ /dev/null @@ -1,747 +0,0 @@ -SET datestyle = 'ISO'; -CREATE SERVER binary_queries_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_queries_test', driver 'binary'); -CREATE SERVER binary_queries_loopback2 FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_queries_test', driver 'binary'); -CREATE USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback; -CREATE USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback2; -SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t1 - (c1 Int, c2 Int, c3 String, c4 Date, c5 Date, c6 String, c7 String, c8 String) - ENGINE = MergeTree PARTITION BY c4 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t2 (c1 Int, c2 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t3 (c1 Int, c3 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t4 (c1 Int, c2 Int, c3 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -CREATE SCHEMA binary_queries_test; -SET search_path = binary_queries_test, public; -CREATE FOREIGN TABLE ft1 ( - c0 int, - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER binary_queries_loopback OPTIONS (table_name 't1'); -ALTER FOREIGN TABLE ft1 DROP COLUMN c0; -CREATE FOREIGN TABLE ft2 ( - c1 int NOT NULL, - c2 text NOT NULL -) SERVER binary_queries_loopback OPTIONS (table_name 't2'); -CREATE FOREIGN TABLE ft4 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback OPTIONS (table_name 't4'); -CREATE FOREIGN TABLE ft5 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback OPTIONS (table_name 't4'); -CREATE FOREIGN TABLE ft6 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback2 OPTIONS (table_name 't4'); -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t1 - SELECT number, - number % 10, - toString(number), - toDate('1990-01-01'), - toDate('1990-01-01'), - number % 10, - number % 10, - 'foo' - FROM numbers(1, 110);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t2 - SELECT number, - concat('AAA', toString(number)) - FROM numbers(1, 100);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t4 - SELECT number, - number + 1, - concat('AAA', toString(number)) - FROM numbers(1, 100);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -\set VERBOSITY terse -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work - c3 | c4 -----+------------ - 1 | 1990-01-01 -(1 row) - -ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'no such database'); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: DB::Exception: Database `no such database` doesn't exist -ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (ADD user 'no such user'); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: connection error: DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. -ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'binary_queries_test'); -ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (DROP user); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work again - c3 | c4 -----+------------ - 1 | 1990-01-01 -(1 row) - -\set VERBOSITY default -ANALYZE ft1; -EXPLAIN (COSTS OFF) SELECT * FROM ft1 ORDER BY c1 OFFSET 100 LIMIT 10; - QUERY PLAN ---------------------- - Foreign Scan on ft1 -(1 row) - -SELECT * FROM ft1 ORDER BY c1 OFFSET 100 LIMIT 10; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 ORDER BY t1.c1, t1.tableoid OFFSET 100 LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------------------------------- - Limit - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - -> Sort - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - Sort Key: t1.c1, t1.tableoid - -> Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 -(8 rows) - -SELECT * FROM ft1 t1 ORDER BY t1.c1, t1.tableoid OFFSET 100 LIMIT 10; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1 FROM ft1 t1 ORDER BY t1.c1 OFFSET 100 LIMIT 10; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: t1.*, c1 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 ORDER BY c1 ASC NULLS LAST LIMIT 10 OFFSET 100 -(3 rows) - -SELECT t1 FROM ft1 t1 ORDER BY t1.c1 OFFSET 100 LIMIT 10; - t1 -------------------------------------------- - (101,1,101,1990-01-01,1990-01-01,1,1,foo) - (102,2,102,1990-01-01,1990-01-01,2,2,foo) - (103,3,103,1990-01-01,1990-01-01,3,3,foo) - (104,4,104,1990-01-01,1990-01-01,4,4,foo) - (105,5,105,1990-01-01,1990-01-01,5,5,foo) - (106,6,106,1990-01-01,1990-01-01,6,6,foo) - (107,7,107,1990-01-01,1990-01-01,7,7,foo) - (108,8,108,1990-01-01,1990-01-01,8,8,foo) - (109,9,109,1990-01-01,1990-01-01,9,9,foo) - (110,0,110,1990-01-01,1990-01-01,0,0,foo) -(10 rows) - -SELECT * FROM ft1 WHERE false; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 101 AND t1.c6 = '1' AND t1.c7 >= '1'; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c7 >= '1')) AND ((c1 = 101)) AND ((c6 = '1')) -(3 rows) - -SELECT COUNT(*) FROM ft1 t1; - count -------- - 110 -(1 row) - -SELECT * FROM ft1 t1 WHERE t1.c3 IN (SELECT c2 FROM ft2 t2 WHERE c1 <= 10) ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -SELECT * FROM ft1 t1 WHERE t1.c3 = (SELECT MAX(c2) FROM ft2 t2) ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -WITH t1 AS (SELECT * FROM ft1 WHERE c1 <= 10) SELECT t2.c1, t2.c2, t2.c2 FROM t1, ft2 t2 WHERE t1.c1 = t2.c1 ORDER BY t1.c1; - c1 | c2 | c2 -----+-------+------- - 1 | AAA1 | AAA1 - 2 | AAA2 | AAA2 - 3 | AAA3 | AAA3 - 4 | AAA4 | AAA4 - 5 | AAA5 | AAA5 - 6 | AAA6 | AAA6 - 7 | AAA7 | AAA7 - 8 | AAA8 | AAA8 - 9 | AAA9 | AAA9 - 10 | AAA10 | AAA10 -(10 rows) - -SELECT 'fixed', NULL FROM ft1 t1 WHERE c1 = 1; - ?column? | ?column? -----------+---------- - fixed | -(1 row) - -SET enable_hashjoin TO false; -SET enable_nestloop TO false; -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c1 FROM ft2 t1 JOIN ft1 t2 ON (t1.c1 = t2.c1) OFFSET 100 LIMIT 10; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: t1.c1, t2.c1 - Relations: (ft2 t1) INNER JOIN (ft1 t2) - Remote SQL: SELECT r1.c1, r2.c1 FROM binary_queries_test.t2 r1 ALL INNER JOIN binary_queries_test.t1 r2 ON (((r1.c1 = r2.c1))) LIMIT 10 OFFSET 100 -(4 rows) - -SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - c1 | c1 -----+---- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) OFFSET 100 LIMIT 10; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: t1.c1, t2.c1 - Relations: (ft2 t1) LEFT JOIN (ft1 t2) - Remote SQL: SELECT r1.c1, r2.c1 FROM binary_queries_test.t2 r1 ALL LEFT JOIN binary_queries_test.t1 r2 ON (((r1.c1 = r2.c1))) LIMIT 10 OFFSET 100 -(4 rows) - -EXPLAIN SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - QUERY PLAN ----------------------------------------------------------------- - Limit (cost=1.00..1.05 rows=10 width=32) - -> Unique (cost=1.00..5.60 rows=1000 width=32) - -> Foreign Scan (cost=1.00..0.60 rows=1000 width=32) - Relations: (ft2 t1) LEFT JOIN (ft1 t2) -(4 rows) - -SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - c1 | c1 -----+---- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -RESET enable_hashjoin; -RESET enable_nestloop; -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 1; -- Var, OpExpr(b), Const - QUERY PLAN --------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = 1)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 100 AND t1.c2 = 0; -- BoolExpr - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = 100)) AND ((c2 = 0)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NULL; -- NullTest - QUERY PLAN ------------------------------------------- - Result - Output: c1, c2, c3, c4, c5, c6, c7, c8 - One-Time Filter: false -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL; -- NullTest - QUERY PLAN ---------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE round(abs(c1), 0) = 1; -- FuncExpr - QUERY PLAN ------------------------------------------------------------------------------------------------------------------ - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((round(abs(c1), 0) = 1)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = -c1; -- OpExpr(l) - QUERY PLAN -------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = (- c1))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE 1 = factorial(c1); -- OpExpr(r) - QUERY PLAN -------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((1 = factorial(c1))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE (c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL); -- DistinctExpr - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE (((c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]); -- ScalarArrayOpExpr - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((has([c2, 1, (c1 + 0)],c1))) -(3 rows) - -SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]) ORDER BY c1; -- ScalarArrayOpExpr - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 1 | 1 | 1 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 2 | 2 | 2 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 3 | 3 | 3 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 4 | 4 | 4 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 5 | 5 | 5 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 6 | 6 | 6 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 7 | 7 | 7 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 8 | 8 | 8 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 9 | 9 | 9 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 10 | 0 | 10 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 11 | 1 | 11 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 12 | 2 | 12 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 13 | 3 | 13 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 14 | 4 | 14 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 15 | 5 | 15 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 16 | 6 | 16 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 17 | 7 | 17 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 18 | 8 | 18 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 19 | 9 | 19 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 20 | 0 | 20 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 21 | 1 | 21 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 22 | 2 | 22 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 23 | 3 | 23 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 24 | 4 | 24 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 25 | 5 | 25 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 26 | 6 | 26 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 27 | 7 | 27 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 28 | 8 | 28 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 29 | 9 | 29 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 30 | 0 | 30 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 31 | 1 | 31 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 32 | 2 | 32 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 33 | 3 | 33 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 34 | 4 | 34 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 35 | 5 | 35 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 36 | 6 | 36 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 37 | 7 | 37 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 38 | 8 | 38 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 39 | 9 | 39 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 40 | 0 | 40 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 41 | 1 | 41 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 42 | 2 | 42 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 43 | 3 | 43 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 44 | 4 | 44 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 45 | 5 | 45 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 46 | 6 | 46 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 47 | 7 | 47 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 48 | 8 | 48 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 49 | 9 | 49 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 50 | 0 | 50 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 51 | 1 | 51 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 52 | 2 | 52 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 53 | 3 | 53 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 54 | 4 | 54 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 55 | 5 | 55 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 56 | 6 | 56 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 57 | 7 | 57 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 58 | 8 | 58 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 59 | 9 | 59 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 60 | 0 | 60 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 61 | 1 | 61 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 62 | 2 | 62 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 63 | 3 | 63 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 64 | 4 | 64 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 65 | 5 | 65 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 66 | 6 | 66 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 67 | 7 | 67 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 68 | 8 | 68 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 69 | 9 | 69 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 70 | 0 | 70 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 71 | 1 | 71 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 72 | 2 | 72 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 73 | 3 | 73 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 74 | 4 | 74 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 75 | 5 | 75 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 76 | 6 | 76 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 77 | 7 | 77 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 78 | 8 | 78 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 79 | 9 | 79 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 80 | 0 | 80 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 81 | 1 | 81 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 82 | 2 | 82 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 83 | 3 | 83 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 84 | 4 | 84 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 85 | 5 | 85 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 86 | 6 | 86 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 87 | 7 | 87 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 88 | 8 | 88 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 89 | 9 | 89 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 90 | 0 | 90 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 91 | 1 | 91 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 92 | 2 | 92 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 93 | 3 | 93 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 94 | 4 | 94 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 95 | 5 | 95 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 96 | 6 | 96 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 97 | 7 | 97 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 98 | 8 | 98 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 99 | 9 | 99 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 100 | 0 | 100 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(110 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1]; -- ArrayRef - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = (([c1, c2, 3])[1]))) -(3 rows) - -SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1] ORDER BY c1; -- ArrayRef - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 1 | 1 | 1 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 2 | 2 | 2 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 3 | 3 | 3 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 4 | 4 | 4 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 5 | 5 | 5 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 6 | 6 | 6 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 7 | 7 | 7 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 8 | 8 | 8 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 9 | 9 | 9 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 10 | 0 | 10 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 11 | 1 | 11 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 12 | 2 | 12 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 13 | 3 | 13 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 14 | 4 | 14 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 15 | 5 | 15 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 16 | 6 | 16 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 17 | 7 | 17 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 18 | 8 | 18 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 19 | 9 | 19 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 20 | 0 | 20 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 21 | 1 | 21 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 22 | 2 | 22 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 23 | 3 | 23 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 24 | 4 | 24 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 25 | 5 | 25 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 26 | 6 | 26 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 27 | 7 | 27 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 28 | 8 | 28 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 29 | 9 | 29 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 30 | 0 | 30 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 31 | 1 | 31 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 32 | 2 | 32 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 33 | 3 | 33 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 34 | 4 | 34 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 35 | 5 | 35 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 36 | 6 | 36 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 37 | 7 | 37 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 38 | 8 | 38 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 39 | 9 | 39 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 40 | 0 | 40 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 41 | 1 | 41 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 42 | 2 | 42 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 43 | 3 | 43 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 44 | 4 | 44 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 45 | 5 | 45 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 46 | 6 | 46 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 47 | 7 | 47 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 48 | 8 | 48 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 49 | 9 | 49 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 50 | 0 | 50 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 51 | 1 | 51 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 52 | 2 | 52 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 53 | 3 | 53 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 54 | 4 | 54 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 55 | 5 | 55 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 56 | 6 | 56 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 57 | 7 | 57 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 58 | 8 | 58 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 59 | 9 | 59 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 60 | 0 | 60 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 61 | 1 | 61 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 62 | 2 | 62 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 63 | 3 | 63 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 64 | 4 | 64 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 65 | 5 | 65 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 66 | 6 | 66 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 67 | 7 | 67 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 68 | 8 | 68 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 69 | 9 | 69 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 70 | 0 | 70 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 71 | 1 | 71 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 72 | 2 | 72 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 73 | 3 | 73 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 74 | 4 | 74 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 75 | 5 | 75 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 76 | 6 | 76 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 77 | 7 | 77 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 78 | 8 | 78 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 79 | 9 | 79 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 80 | 0 | 80 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 81 | 1 | 81 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 82 | 2 | 82 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 83 | 3 | 83 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 84 | 4 | 84 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 85 | 5 | 85 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 86 | 6 | 86 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 87 | 7 | 87 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 88 | 8 | 88 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 89 | 9 | 89 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 90 | 0 | 90 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 91 | 1 | 91 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 92 | 2 | 92 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 93 | 3 | 93 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 94 | 4 | 94 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 95 | 5 | 95 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 96 | 6 | 96 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 97 | 7 | 97 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 98 | 8 | 98 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 99 | 9 | 99 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 100 | 0 | 100 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(110 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c6 = E'foo''s\\bar'; -- check special chars - QUERY PLAN --------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c6 = 'foo''s\\bar')) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c8 = 'foo'; -- can't be sent to remote - QUERY PLAN ------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c8 = 'foo')) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT (CASE WHEN c1 < 10 THEN 1 WHEN c1 < 50 THEN 2 ELSE 3 END) a, - sum(length(c2)) FROM ft2 GROUP BY a ORDER BY a; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: (CASE WHEN (c1 < 10) THEN 1 WHEN (c1 < 50) THEN 2 ELSE 3 END), (sum(length(c2))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END, sum(lengthUTF8(c2)) FROM binary_queries_test.t2 GROUP BY (CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END) ORDER BY CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END ASC NULLS LAST -(4 rows) - -SELECT (CASE WHEN c1 < 10 THEN 1 WHEN c1 < 50 THEN 2 ELSE 3 END) a, - sum(length(c2)) FROM ft2 GROUP BY a ORDER BY a; - a | sum ----+----- - 1 | 36 - 2 | 200 - 3 | 256 -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT SUM(c1) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN ------------------------------------------------------------------------------- - Foreign Scan - Output: (sum(c1) FILTER (WHERE (c1 < 20))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT sumIf(c1,(((c1 < 20)) > 0)) FROM binary_queries_test.t2 -(4 rows) - -SELECT SUM(c1) FILTER (WHERE c1 < 20) FROM ft2; - sum ------ - 190 -(1 row) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(DISTINCT c1) FROM ft2; - QUERY PLAN ---------------------------------------------------------------------- - Foreign Scan - Output: (count(DISTINCT c1)) - Relations: Aggregate on (ft2) - Remote SQL: SELECT count(DISTINCT c1) FROM binary_queries_test.t2 -(4 rows) - -SELECT COUNT(DISTINCT c1) FROM ft2; - count -------- - 100 -(1 row) - -/* DISTINCT with IF */ -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(DISTINCT c1) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN --------------------------------------------------------------------------------------- - Aggregate - Output: count(DISTINCT c1) FILTER (WHERE (c1 < 20)) - -> Foreign Scan on binary_queries_test.ft2 - Output: c1, c2 - Remote SQL: SELECT c1 FROM binary_queries_test.t2 ORDER BY c1 ASC NULLS LAST -(5 rows) - -SELECT COUNT(DISTINCT c1) FILTER (WHERE c1 < 20) FROM ft2; - count -------- - 19 -(1 row) - -/* https://github.com/ClickHouse/pg_clickhouse/issues/25 */ -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(*) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN ------------------------------------------------------------------------------ - Foreign Scan - Output: (count(*) FILTER (WHERE (c1 < 20))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT countIf((((c1 < 20)) > 0)) FROM binary_queries_test.t2 -(4 rows) - -SELECT COUNT(*) FILTER (WHERE c1 < 10) FROM ft2; - count -------- - 9 -(1 row) - -SELECT clickhouse_raw_query('DROP DATABASE binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -DROP USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback2; -DROP USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback; -DROP SERVER binary_queries_loopback2 CASCADE; -NOTICE: drop cascades to foreign table ft6 -DROP SERVER binary_queries_loopback CASCADE; -NOTICE: drop cascades to 4 other objects -DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft2 -drop cascades to foreign table ft4 -drop cascades to foreign table ft5 diff --git a/test/expected/binary_queries_7.out b/test/expected/binary_queries_7.out deleted file mode 100644 index 6ba4bc34..00000000 --- a/test/expected/binary_queries_7.out +++ /dev/null @@ -1,745 +0,0 @@ -SET datestyle = 'ISO'; -CREATE SERVER binary_queries_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_queries_test', driver 'binary'); -CREATE SERVER binary_queries_loopback2 FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_queries_test', driver 'binary'); -CREATE USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback; -CREATE USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback2; -SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t1 - (c1 Int, c2 Int, c3 String, c4 Date, c5 Date, c6 String, c7 String, c8 String) - ENGINE = MergeTree PARTITION BY c4 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t2 (c1 Int, c2 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t3 (c1 Int, c3 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t4 (c1 Int, c2 Int, c3 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -CREATE FOREIGN TABLE ft1 ( - c0 int, - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER binary_queries_loopback OPTIONS (table_name 't1'); -ALTER FOREIGN TABLE ft1 DROP COLUMN c0; -CREATE FOREIGN TABLE ft2 ( - c1 int NOT NULL, - c2 text NOT NULL -) SERVER binary_queries_loopback OPTIONS (table_name 't2'); -CREATE FOREIGN TABLE ft4 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback OPTIONS (table_name 't4'); -CREATE FOREIGN TABLE ft5 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback OPTIONS (table_name 't4'); -CREATE FOREIGN TABLE ft6 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback2 OPTIONS (table_name 't4'); -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t1 - SELECT number, - number % 10, - toString(number), - toDate('1990-01-01'), - toDate('1990-01-01'), - number % 10, - number % 10, - 'foo' - FROM numbers(1, 110);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t2 - SELECT number, - concat('AAA', toString(number)) - FROM numbers(1, 100);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t4 - SELECT number, - number + 1, - concat('AAA', toString(number)) - FROM numbers(1, 100);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -\set VERBOSITY terse -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work - c3 | c4 -----+------------ - 1 | 1990-01-01 -(1 row) - -ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'no such database'); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: DB::Exception: Database `no such database` doesn't exist -ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (ADD user 'no such user'); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: connection error: DB::Exception: no such user: Authentication failed: password is incorrect or there is no user with such name -ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'binary_queries_test'); -ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (DROP user); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work again - c3 | c4 -----+------------ - 1 | 1990-01-01 -(1 row) - -\set VERBOSITY default -ANALYZE ft1; -EXPLAIN (COSTS OFF) SELECT * FROM ft1 ORDER BY c1 OFFSET 100 LIMIT 10; - QUERY PLAN ---------------------- - Foreign Scan on ft1 -(1 row) - -SELECT * FROM ft1 ORDER BY c1 OFFSET 100 LIMIT 10; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 ORDER BY t1.c1, t1.tableoid OFFSET 100 LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------------------------------- - Limit - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - -> Sort - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - Sort Key: t1.c1, t1.tableoid - -> Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 -(8 rows) - -SELECT * FROM ft1 t1 ORDER BY t1.c1, t1.tableoid OFFSET 100 LIMIT 10; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1 FROM ft1 t1 ORDER BY t1.c1 OFFSET 100 LIMIT 10; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: t1.*, c1 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 ORDER BY c1 ASC NULLS LAST LIMIT 10 OFFSET 100 -(3 rows) - -SELECT t1 FROM ft1 t1 ORDER BY t1.c1 OFFSET 100 LIMIT 10; - t1 -------------------------------------------- - (101,1,101,1990-01-01,1990-01-01,1,1,foo) - (102,2,102,1990-01-01,1990-01-01,2,2,foo) - (103,3,103,1990-01-01,1990-01-01,3,3,foo) - (104,4,104,1990-01-01,1990-01-01,4,4,foo) - (105,5,105,1990-01-01,1990-01-01,5,5,foo) - (106,6,106,1990-01-01,1990-01-01,6,6,foo) - (107,7,107,1990-01-01,1990-01-01,7,7,foo) - (108,8,108,1990-01-01,1990-01-01,8,8,foo) - (109,9,109,1990-01-01,1990-01-01,9,9,foo) - (110,0,110,1990-01-01,1990-01-01,0,0,foo) -(10 rows) - -SELECT * FROM ft1 WHERE false; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 101 AND t1.c6 = '1' AND t1.c7 >= '1'; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c7 >= '1')) AND ((c1 = 101)) AND ((c6 = '1')) -(3 rows) - -SELECT COUNT(*) FROM ft1 t1; - count -------- - 110 -(1 row) - -SELECT * FROM ft1 t1 WHERE t1.c3 IN (SELECT c2 FROM ft2 t2 WHERE c1 <= 10) ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -SELECT * FROM ft1 t1 WHERE t1.c3 = (SELECT MAX(c2) FROM ft2 t2) ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -WITH t1 AS (SELECT * FROM ft1 WHERE c1 <= 10) SELECT t2.c1, t2.c2, t2.c2 FROM t1, ft2 t2 WHERE t1.c1 = t2.c1 ORDER BY t1.c1; - c1 | c2 | c2 -----+-------+------- - 1 | AAA1 | AAA1 - 2 | AAA2 | AAA2 - 3 | AAA3 | AAA3 - 4 | AAA4 | AAA4 - 5 | AAA5 | AAA5 - 6 | AAA6 | AAA6 - 7 | AAA7 | AAA7 - 8 | AAA8 | AAA8 - 9 | AAA9 | AAA9 - 10 | AAA10 | AAA10 -(10 rows) - -SELECT 'fixed', NULL FROM ft1 t1 WHERE c1 = 1; - ?column? | ?column? -----------+---------- - fixed | -(1 row) - -SET enable_hashjoin TO false; -SET enable_nestloop TO false; -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c1 FROM ft2 t1 JOIN ft1 t2 ON (t1.c1 = t2.c1) OFFSET 100 LIMIT 10; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: t1.c1, t2.c1 - Relations: (ft2 t1) INNER JOIN (ft1 t2) - Remote SQL: SELECT r1.c1, r2.c1 FROM binary_queries_test.t2 r1 ALL INNER JOIN binary_queries_test.t1 r2 ON (((r1.c1 = r2.c1))) LIMIT 10 OFFSET 100 -(4 rows) - -SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - c1 | c1 -----+---- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) OFFSET 100 LIMIT 10; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: t1.c1, t2.c1 - Relations: (ft2 t1) LEFT JOIN (ft1 t2) - Remote SQL: SELECT r1.c1, r2.c1 FROM binary_queries_test.t2 r1 ALL LEFT JOIN binary_queries_test.t1 r2 ON (((r1.c1 = r2.c1))) LIMIT 10 OFFSET 100 -(4 rows) - -EXPLAIN SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - QUERY PLAN ----------------------------------------------------------------- - Limit (cost=1.00..1.05 rows=10 width=32) - -> Unique (cost=1.00..5.60 rows=1000 width=32) - -> Foreign Scan (cost=1.00..0.60 rows=1000 width=32) - Relations: (ft2 t1) LEFT JOIN (ft1 t2) -(4 rows) - -SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - c1 | c1 -----+---- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -RESET enable_hashjoin; -RESET enable_nestloop; -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 1; -- Var, OpExpr(b), Const - QUERY PLAN --------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = 1)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 100 AND t1.c2 = 0; -- BoolExpr - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = 100)) AND ((c2 = 0)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NULL; -- NullTest - QUERY PLAN ------------------------------------------- - Result - Output: c1, c2, c3, c4, c5, c6, c7, c8 - One-Time Filter: false -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL; -- NullTest - QUERY PLAN ---------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE round(abs(c1), 0) = 1; -- FuncExpr - QUERY PLAN ------------------------------------------------------------------------------------------------------------------ - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((round(abs(c1), 0) = 1)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = -c1; -- OpExpr(l) - QUERY PLAN -------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = (- c1))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE 1 = factorial(c1); -- OpExpr(r) - QUERY PLAN -------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((1 = factorial(c1))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE (c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL); -- DistinctExpr - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE (((c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]); -- ScalarArrayOpExpr - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((has([c2, 1, (c1 + 0)],c1))) -(3 rows) - -SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]) ORDER BY c1; -- ScalarArrayOpExpr - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 1 | 1 | 1 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 2 | 2 | 2 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 3 | 3 | 3 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 4 | 4 | 4 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 5 | 5 | 5 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 6 | 6 | 6 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 7 | 7 | 7 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 8 | 8 | 8 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 9 | 9 | 9 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 10 | 0 | 10 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 11 | 1 | 11 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 12 | 2 | 12 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 13 | 3 | 13 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 14 | 4 | 14 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 15 | 5 | 15 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 16 | 6 | 16 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 17 | 7 | 17 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 18 | 8 | 18 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 19 | 9 | 19 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 20 | 0 | 20 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 21 | 1 | 21 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 22 | 2 | 22 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 23 | 3 | 23 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 24 | 4 | 24 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 25 | 5 | 25 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 26 | 6 | 26 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 27 | 7 | 27 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 28 | 8 | 28 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 29 | 9 | 29 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 30 | 0 | 30 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 31 | 1 | 31 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 32 | 2 | 32 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 33 | 3 | 33 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 34 | 4 | 34 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 35 | 5 | 35 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 36 | 6 | 36 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 37 | 7 | 37 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 38 | 8 | 38 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 39 | 9 | 39 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 40 | 0 | 40 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 41 | 1 | 41 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 42 | 2 | 42 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 43 | 3 | 43 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 44 | 4 | 44 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 45 | 5 | 45 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 46 | 6 | 46 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 47 | 7 | 47 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 48 | 8 | 48 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 49 | 9 | 49 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 50 | 0 | 50 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 51 | 1 | 51 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 52 | 2 | 52 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 53 | 3 | 53 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 54 | 4 | 54 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 55 | 5 | 55 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 56 | 6 | 56 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 57 | 7 | 57 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 58 | 8 | 58 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 59 | 9 | 59 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 60 | 0 | 60 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 61 | 1 | 61 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 62 | 2 | 62 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 63 | 3 | 63 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 64 | 4 | 64 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 65 | 5 | 65 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 66 | 6 | 66 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 67 | 7 | 67 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 68 | 8 | 68 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 69 | 9 | 69 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 70 | 0 | 70 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 71 | 1 | 71 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 72 | 2 | 72 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 73 | 3 | 73 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 74 | 4 | 74 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 75 | 5 | 75 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 76 | 6 | 76 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 77 | 7 | 77 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 78 | 8 | 78 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 79 | 9 | 79 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 80 | 0 | 80 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 81 | 1 | 81 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 82 | 2 | 82 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 83 | 3 | 83 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 84 | 4 | 84 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 85 | 5 | 85 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 86 | 6 | 86 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 87 | 7 | 87 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 88 | 8 | 88 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 89 | 9 | 89 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 90 | 0 | 90 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 91 | 1 | 91 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 92 | 2 | 92 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 93 | 3 | 93 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 94 | 4 | 94 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 95 | 5 | 95 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 96 | 6 | 96 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 97 | 7 | 97 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 98 | 8 | 98 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 99 | 9 | 99 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 100 | 0 | 100 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(110 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1]; -- ArrayRef - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = (([c1, c2, 3])[1]))) -(3 rows) - -SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1] ORDER BY c1; -- ArrayRef - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 1 | 1 | 1 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 2 | 2 | 2 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 3 | 3 | 3 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 4 | 4 | 4 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 5 | 5 | 5 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 6 | 6 | 6 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 7 | 7 | 7 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 8 | 8 | 8 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 9 | 9 | 9 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 10 | 0 | 10 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 11 | 1 | 11 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 12 | 2 | 12 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 13 | 3 | 13 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 14 | 4 | 14 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 15 | 5 | 15 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 16 | 6 | 16 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 17 | 7 | 17 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 18 | 8 | 18 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 19 | 9 | 19 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 20 | 0 | 20 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 21 | 1 | 21 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 22 | 2 | 22 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 23 | 3 | 23 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 24 | 4 | 24 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 25 | 5 | 25 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 26 | 6 | 26 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 27 | 7 | 27 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 28 | 8 | 28 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 29 | 9 | 29 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 30 | 0 | 30 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 31 | 1 | 31 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 32 | 2 | 32 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 33 | 3 | 33 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 34 | 4 | 34 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 35 | 5 | 35 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 36 | 6 | 36 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 37 | 7 | 37 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 38 | 8 | 38 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 39 | 9 | 39 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 40 | 0 | 40 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 41 | 1 | 41 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 42 | 2 | 42 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 43 | 3 | 43 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 44 | 4 | 44 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 45 | 5 | 45 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 46 | 6 | 46 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 47 | 7 | 47 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 48 | 8 | 48 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 49 | 9 | 49 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 50 | 0 | 50 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 51 | 1 | 51 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 52 | 2 | 52 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 53 | 3 | 53 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 54 | 4 | 54 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 55 | 5 | 55 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 56 | 6 | 56 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 57 | 7 | 57 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 58 | 8 | 58 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 59 | 9 | 59 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 60 | 0 | 60 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 61 | 1 | 61 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 62 | 2 | 62 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 63 | 3 | 63 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 64 | 4 | 64 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 65 | 5 | 65 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 66 | 6 | 66 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 67 | 7 | 67 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 68 | 8 | 68 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 69 | 9 | 69 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 70 | 0 | 70 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 71 | 1 | 71 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 72 | 2 | 72 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 73 | 3 | 73 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 74 | 4 | 74 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 75 | 5 | 75 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 76 | 6 | 76 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 77 | 7 | 77 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 78 | 8 | 78 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 79 | 9 | 79 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 80 | 0 | 80 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 81 | 1 | 81 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 82 | 2 | 82 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 83 | 3 | 83 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 84 | 4 | 84 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 85 | 5 | 85 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 86 | 6 | 86 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 87 | 7 | 87 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 88 | 8 | 88 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 89 | 9 | 89 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 90 | 0 | 90 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 91 | 1 | 91 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 92 | 2 | 92 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 93 | 3 | 93 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 94 | 4 | 94 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 95 | 5 | 95 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 96 | 6 | 96 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 97 | 7 | 97 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 98 | 8 | 98 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 99 | 9 | 99 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 100 | 0 | 100 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(110 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c6 = E'foo''s\\bar'; -- check special chars - QUERY PLAN --------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c6 = 'foo''s\\bar')) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c8 = 'foo'; -- can't be sent to remote - QUERY PLAN ------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c8 = 'foo')) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT (CASE WHEN c1 < 10 THEN 1 WHEN c1 < 50 THEN 2 ELSE 3 END) a, - sum(length(c2)) FROM ft2 GROUP BY a ORDER BY a; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: (CASE WHEN (c1 < 10) THEN 1 WHEN (c1 < 50) THEN 2 ELSE 3 END), (sum(length(c2))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END, sum(lengthUTF8(c2)) FROM binary_queries_test.t2 GROUP BY (CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END) ORDER BY CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END ASC NULLS LAST -(4 rows) - -SELECT (CASE WHEN c1 < 10 THEN 1 WHEN c1 < 50 THEN 2 ELSE 3 END) a, - sum(length(c2)) FROM ft2 GROUP BY a ORDER BY a; - a | sum ----+----- - 1 | 36 - 2 | 200 - 3 | 256 -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT SUM(c1) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN ------------------------------------------------------------------------------- - Foreign Scan - Output: (sum(c1) FILTER (WHERE (c1 < 20))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT sumIf(c1,(((c1 < 20)) > 0)) FROM binary_queries_test.t2 -(4 rows) - -SELECT SUM(c1) FILTER (WHERE c1 < 20) FROM ft2; - sum ------ - 190 -(1 row) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(DISTINCT c1) FROM ft2; - QUERY PLAN ---------------------------------------------------------------------- - Foreign Scan - Output: (count(DISTINCT c1)) - Relations: Aggregate on (ft2) - Remote SQL: SELECT count(DISTINCT c1) FROM binary_queries_test.t2 -(4 rows) - -SELECT COUNT(DISTINCT c1) FROM ft2; - count -------- - 100 -(1 row) - -/* DISTINCT with IF */ -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(DISTINCT c1) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN --------------------------------------------------------------------------------------- - Aggregate - Output: count(DISTINCT c1) FILTER (WHERE (c1 < 20)) - -> Foreign Scan on public.ft2 - Output: c1, c2 - Remote SQL: SELECT c1 FROM binary_queries_test.t2 ORDER BY c1 ASC NULLS LAST -(5 rows) - -SELECT COUNT(DISTINCT c1) FILTER (WHERE c1 < 20) FROM ft2; - count -------- - 19 -(1 row) - -/* https://github.com/ClickHouse/pg_clickhouse/issues/25 */ -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(*) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN ------------------------------------------------------------------------------ - Foreign Scan - Output: (count(*) FILTER (WHERE (c1 < 20))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT countIf((((c1 < 20)) > 0)) FROM binary_queries_test.t2 -(4 rows) - -SELECT COUNT(*) FILTER (WHERE c1 < 10) FROM ft2; - count -------- - 9 -(1 row) - -SELECT clickhouse_raw_query('DROP DATABASE binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -DROP USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback2; -DROP USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback; -DROP SERVER binary_queries_loopback2 CASCADE; -NOTICE: drop cascades to foreign table ft6 -DROP SERVER binary_queries_loopback CASCADE; -NOTICE: drop cascades to 4 other objects -DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft2 -drop cascades to foreign table ft4 -drop cascades to foreign table ft5 diff --git a/test/expected/import_schema.out b/test/expected/import_schema.out index c50c4dd1..1d57c9ab 100644 --- a/test/expected/import_schema.out +++ b/test/expected/import_schema.out @@ -21,6 +21,12 @@ SELECT clickhouse_raw_query('CREATE DATABASE import_test'); (1 row) +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test_2'); + clickhouse_raw_query +---------------------- + +(1 row) + SELECT clickhouse_raw_query('CREATE DATABASE import_test_2'); clickhouse_raw_query ---------------------- @@ -703,6 +709,7 @@ SELECT * FROM clickhouse_bin.ints WHERE c1 IN (127, -128) ORDER BY c1; ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; @@ -714,10 +721,10 @@ SELECT c1, c2, c3, c4, c5, c6, c7, NULL, c9, c10 FROM clickhouse_bin.ints WHERE c1 = 127 ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -------+--------+-------------+----------------------+-----+----+------------+----+---------------+------------------------- - -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754942e-38 | 2.2250738585072014e-308 - 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | -1 | 4294967295 | | 3.4028233e+38 | 1.7976931348623157e+308 + c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 +------+--------+-------------+----------------------+-----+-------+------------+----+---------------+------------------------- + -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754942e-38 | 2.2250738585072014e-308 + 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | 65535 | 4294967295 | | 3.4028233e+38 | 1.7976931348623157e+308 (2 rows) SELECT * FROM clickhouse.ints WHERE c1 = -128 diff --git a/test/expected/import_schema_1.out b/test/expected/import_schema_1.out index af315452..1e1a14b3 100644 --- a/test/expected/import_schema_1.out +++ b/test/expected/import_schema_1.out @@ -21,6 +21,12 @@ SELECT clickhouse_raw_query('CREATE DATABASE import_test'); (1 row) +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test_2'); + clickhouse_raw_query +---------------------- + +(1 row) + SELECT clickhouse_raw_query('CREATE DATABASE import_test_2'); clickhouse_raw_query ---------------------- @@ -603,6 +609,7 @@ SELECT * FROM clickhouse_bin.ints WHERE c1 IN (127, -128) ORDER BY c1; ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; @@ -614,10 +621,10 @@ SELECT c1, c2, c3, c4, c5, c6, c7, NULL, c9, c10 FROM clickhouse_bin.ints WHERE c1 = 127 ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -------+--------+-------------+----------------------+-----+----+------------+----+---------------+------------------------- - -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754942e-38 | 2.2250738585072014e-308 - 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | -1 | 4294967295 | | 3.4028233e+38 | 1.7976931348623157e+308 + c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 +------+--------+-------------+----------------------+-----+-------+------------+----+---------------+------------------------- + -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754942e-38 | 2.2250738585072014e-308 + 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | 65535 | 4294967295 | | 3.4028233e+38 | 1.7976931348623157e+308 (2 rows) SELECT * FROM clickhouse.ints WHERE c1 = -128 diff --git a/test/expected/import_schema_2.out b/test/expected/import_schema_2.out deleted file mode 100644 index ed19d783..00000000 --- a/test/expected/import_schema_2.out +++ /dev/null @@ -1,774 +0,0 @@ -SET datestyle = 'ISO'; -CREATE SERVER import_loopback FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'import_test', driver 'http'); -CREATE SERVER import_loopback_bin FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'import_test', driver 'binary'); -CREATE SCHEMA clickhouse; -CREATE SCHEMA clickhouse_bin; -CREATE SCHEMA clickhouse_limit; -CREATE SCHEMA clickhouse_except; -CREATE USER MAPPING FOR CURRENT_USER SERVER import_loopback; -CREATE USER MAPPING FOR CURRENT_USER SERVER import_loopback_bin; -SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE import_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE import_test_2'); - clickhouse_raw_query ----------------------- - -(1 row) - --- integer types -SELECT clickhouse_raw_query('CREATE TABLE import_test.ints ( - c1 Int8, c2 Int16, c3 Int32, c4 Int64, - c5 UInt8, c6 UInt16, c7 UInt32, c8 UInt64, - c9 Float32, c10 Nullable(Float64) -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.ints SELECT - number, number + 1, number + 2, number + 3, number + 4, number + 5, - number + 6, number + 7, number + 8.1, number + 9.2 FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.ints SELECT - number, number + 1, number + 2, number + 3, number + 4, number + 5, - number + 6, number + 7, number + 8.1, NULL FROM numbers(10, 2);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE import_test.types ( - c1 Date, c2 DateTime, c3 String, c4 FixedString(5), c5 UUID, - c6 Enum8(''one'' = 1, ''two'' = 2), - c7 Enum16(''one'' = 1, ''two'' = 2, ''three'' = 3), - c9 Nullable(FixedString(50)), - c8 LowCardinality(String) -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.types SELECT - addDays(toDate(''1990-01-01''), number), - addMinutes(addSeconds(addDays(toDateTime(''1990-01-01 10:00:00''), number), number), number), - format(''number {0}'', toString(number)), - format(''num {0}'', toString(number)), - format(''f4bf890f-f9dc-4332-ad5c-0c18e73f28e{0}'', toString(number)), - ''two'', - ''three'', - toString(number), - format(''cardinal {0}'', toString(number)) - FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE import_test.types2 ( - c1 LowCardinality(Nullable(String)) -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1) SETTINGS allow_nullable_key = 1; -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.types2 SELECT - format(''cardinal {0}'', toString(number + 1)) - FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE import_test.ip ( - c1 IPv4, - c2 IPv6 -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query($$ - INSERT INTO import_test.ip VALUES - ('116.106.34.242', '2001:44c8:129:2632:33:0:252:2'), - ('116.106.34.243', '2a02:e980:1e::1'), - ('116.106.34.244', '::1'); -$$); - clickhouse_raw_query ----------------------- - -(1 row) - --- array types -SELECT clickhouse_raw_query('CREATE TABLE import_test.arrays ( - c1 Array(Int), c2 Array(String) -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.arrays SELECT - [number, number + 1], - [format(''num{0}'', toString(number)), format(''num{0}'', toString(number + 1))] - FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - --- tuple -SELECT clickhouse_raw_query('CREATE TABLE import_test.tuples ( - c1 Int8, - c2 Tuple(Int, String, Float32), - c3 Nested(a Int, b Int), - c4 Int16 -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.tuples SELECT - number, - (number, toString(number), number + 1.0), - [toInt32(number),1,1], - [toInt32(number),2,2], - toInt16(number) - FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - --- datetime with timezones -SELECT clickhouse_raw_query('CREATE TABLE import_test.timezones ( - t1 DateTime64(6,''UTC''), - t2 DateTime64(6,''Europe/Berlin''), - t4 DateTime(''Europe/Berlin''), - t5 DateTime64(6)) - ENGINE = MergeTree ORDER BY (t1) SETTINGS index_granularity=8192;'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.timezones VALUES ( - ''2020-01-01 11:00:00'', - ''2020-01-01 11:00:00'', - ''2020-01-01 11:00:00'', - ''2020-01-01 11:00:00'')'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.timezones VALUES ( - ''2020-01-01 12:00:00'', - ''2020-01-01 12:00:00'', - ''2020-01-01 12:00:00'', - ''2020-01-01 12:00:00'')'); - clickhouse_raw_query ----------------------- - -(1 row) - --- Injection attempt should import nothing. -IMPORT FOREIGN SCHEMA "import_test' OR database = 'public" FROM SERVER import_loopback INTO clickhouse; -\det clickhouse.* - List of foreign tables - Schema | Table | Server ---------+-------+-------- -(0 rows) - --- Should succeed. -IMPORT FOREIGN SCHEMA import_test FROM SERVER import_loopback INTO clickhouse; -NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "c2", please create composite type and alter the column if needed -\d+ clickhouse.ints; - Foreign table "clickhouse.ints" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------------------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | smallint | | not null | | | plain | | - c3 | integer | | not null | | | plain | | - c4 | bigint | | not null | | | plain | | - c5 | smallint | | not null | | | plain | | - c6 | integer | | not null | | | plain | | - c7 | bigint | | not null | | | plain | | - c8 | bigint | | not null | | | plain | | - c9 | real | | not null | | | plain | | - c10 | double precision | | | | | plain | | -Not-null constraints: - "ints_c1_not_null" NOT NULL "c1" - "ints_c2_not_null" NOT NULL "c2" - "ints_c3_not_null" NOT NULL "c3" - "ints_c4_not_null" NOT NULL "c4" - "ints_c5_not_null" NOT NULL "c5" - "ints_c6_not_null" NOT NULL "c6" - "ints_c7_not_null" NOT NULL "c7" - "ints_c8_not_null" NOT NULL "c8" - "ints_c9_not_null" NOT NULL "c9" -Server: import_loopback -FDW options: (database 'import_test', table_name 'ints', engine 'MergeTree') - -\d+ clickhouse.types; - Foreign table "clickhouse.types" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | date | | not null | | | plain | | - c2 | timestamp without time zone | | not null | | | plain | | - c3 | text | | not null | | | extended | | - c4 | character varying(5) | | not null | | | extended | | - c5 | uuid | | not null | | | plain | | - c6 | text | | not null | | | extended | | - c7 | text | | not null | | | extended | | - c9 | character varying(50) | | | | | extended | | - c8 | text | | not null | | | extended | | -Not-null constraints: - "types_c1_not_null" NOT NULL "c1" - "types_c2_not_null" NOT NULL "c2" - "types_c3_not_null" NOT NULL "c3" - "types_c4_not_null" NOT NULL "c4" - "types_c5_not_null" NOT NULL "c5" - "types_c6_not_null" NOT NULL "c6" - "types_c7_not_null" NOT NULL "c7" - "types_c8_not_null" NOT NULL "c8" -Server: import_loopback -FDW options: (database 'import_test', table_name 'types', engine 'MergeTree') - -\d+ clickhouse.types2; - Foreign table "clickhouse.types2" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | text | | | | | extended | | -Server: import_loopback -FDW options: (database 'import_test', table_name 'types2', engine 'MergeTree') - -\d+ clickhouse.arrays; - Foreign table "clickhouse.arrays" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | integer[] | | not null | | | extended | | - c2 | text[] | | not null | | | extended | | -Not-null constraints: - "arrays_c1_not_null" NOT NULL "c1" - "arrays_c2_not_null" NOT NULL "c2" -Server: import_loopback -FDW options: (database 'import_test', table_name 'arrays', engine 'MergeTree') - -\d+ clickhouse.tuples; - Foreign table "clickhouse.tuples" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | text | | not null | | | extended | | - c3.a | integer[] | | not null | | | extended | | - c3.b | integer[] | | not null | | | extended | | - c4 | smallint | | not null | | | plain | | -Not-null constraints: - "tuples_c1_not_null" NOT NULL "c1" - "tuples_c2_not_null" NOT NULL "c2" - "tuples_c3.a_not_null" NOT NULL "c3.a" - "tuples_c3.b_not_null" NOT NULL "c3.b" - "tuples_c4_not_null" NOT NULL "c4" -Server: import_loopback -FDW options: (database 'import_test', table_name 'tuples', engine 'MergeTree') - -\d+ clickhouse.timezones; - Foreign table "clickhouse.timezones" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+---------+--------------+------------- - t1 | timestamp without time zone | | not null | | | plain | | - t2 | timestamp without time zone | | not null | | | plain | | - t4 | timestamp without time zone | | not null | | | plain | | - t5 | timestamp without time zone | | not null | | | plain | | -Not-null constraints: - "timezones_t1_not_null" NOT NULL "t1" - "timezones_t2_not_null" NOT NULL "t2" - "timezones_t4_not_null" NOT NULL "t4" - "timezones_t5_not_null" NOT NULL "t5" -Server: import_loopback -FDW options: (database 'import_test', table_name 'timezones', engine 'MergeTree') - -\d+ clickhouse.ip; - Foreign table "clickhouse.ip" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | inet | | not null | | | main | | - c2 | inet | | not null | | | main | | -Not-null constraints: - "ip_c1_not_null" NOT NULL "c1" - "ip_c2_not_null" NOT NULL "c2" -Server: import_loopback -FDW options: (database 'import_test', table_name 'ip', engine 'MergeTree') - -SELECT * FROM clickhouse.ints ORDER BY c1 DESC LIMIT 4; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -----+----+----+----+----+----+----+----+------+------ - 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19.1 | - 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18.1 | - 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17.1 | 18.2 - 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16.1 | 17.2 -(4 rows) - -SELECT * FROM clickhouse.types ORDER BY c1 LIMIT 2; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c9 | c8 -------------+---------------------+----------+-------+--------------------------------------+-----+-------+----+------------ - 1990-01-01 | 1990-01-01 10:00:00 | number 0 | num 0 | f4bf890f-f9dc-4332-ad5c-0c18e73f28e0 | two | three | 0 | cardinal 0 - 1990-01-02 | 1990-01-02 10:01:01 | number 1 | num 1 | f4bf890f-f9dc-4332-ad5c-0c18e73f28e1 | two | three | 1 | cardinal 1 -(2 rows) - -SELECT * FROM clickhouse.types2 ORDER BY c1 LIMIT 2; - c1 -------------- - cardinal 1 - cardinal 10 -(2 rows) - -SELECT * FROM clickhouse.arrays ORDER BY c1 LIMIT 2; - c1 | c2 --------+----------------- - {0,1} | {'num0','num1'} - {1,2} | {'num1','num2'} -(2 rows) - -SELECT * FROM clickhouse.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+-----------+---------+---------+---- - 0 | (0,'0',1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,'1',2) | {1,1,1} | {1,2,2} | 1 -(2 rows) - -SELECT * FROM clickhouse.timezones ORDER BY t1 LIMIT 2; - t1 | t2 | t4 | t5 -------------------------+------------------------+------------------------+------------------------ - 2020-01-01 03:00:00-08 | 2020-01-01 02:00:00-08 | 2020-01-01 02:00:00-08 | 2020-01-01 03:00:00-08 - 2020-01-01 04:00:00-08 | 2020-01-01 03:00:00-08 | 2020-01-01 03:00:00-08 | 2020-01-01 04:00:00-08 -(2 rows) - -SELECT * FROM clickhouse.ip ORDER BY c1; - c1 | c2 -----------------+------------------------------- - 116.106.34.242 | 2001:44c8:129:2632:33:0:252:2 - 116.106.34.243 | 2a02:e980:1e::1 - 116.106.34.244 | ::1 -(3 rows) - -IMPORT FOREIGN SCHEMA "import_test" FROM SERVER import_loopback_bin INTO clickhouse_bin; -NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "c2", please create composite type and alter the column if needed -\d+ clickhouse_bin.ints; - Foreign table "clickhouse_bin.ints" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------------------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | smallint | | not null | | | plain | | - c3 | integer | | not null | | | plain | | - c4 | bigint | | not null | | | plain | | - c5 | smallint | | not null | | | plain | | - c6 | integer | | not null | | | plain | | - c7 | bigint | | not null | | | plain | | - c8 | bigint | | not null | | | plain | | - c9 | real | | not null | | | plain | | - c10 | double precision | | | | | plain | | -Not-null constraints: - "ints_c1_not_null" NOT NULL "c1" - "ints_c2_not_null" NOT NULL "c2" - "ints_c3_not_null" NOT NULL "c3" - "ints_c4_not_null" NOT NULL "c4" - "ints_c5_not_null" NOT NULL "c5" - "ints_c6_not_null" NOT NULL "c6" - "ints_c7_not_null" NOT NULL "c7" - "ints_c8_not_null" NOT NULL "c8" - "ints_c9_not_null" NOT NULL "c9" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'ints', engine 'MergeTree') - -\d+ clickhouse_bin.types; - Foreign table "clickhouse_bin.types" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | date | | not null | | | plain | | - c2 | timestamp without time zone | | not null | | | plain | | - c3 | text | | not null | | | extended | | - c4 | character varying(5) | | not null | | | extended | | - c5 | uuid | | not null | | | plain | | - c6 | text | | not null | | | extended | | - c7 | text | | not null | | | extended | | - c9 | character varying(50) | | | | | extended | | - c8 | text | | not null | | | extended | | -Not-null constraints: - "types_c1_not_null" NOT NULL "c1" - "types_c2_not_null" NOT NULL "c2" - "types_c3_not_null" NOT NULL "c3" - "types_c4_not_null" NOT NULL "c4" - "types_c5_not_null" NOT NULL "c5" - "types_c6_not_null" NOT NULL "c6" - "types_c7_not_null" NOT NULL "c7" - "types_c8_not_null" NOT NULL "c8" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'types', engine 'MergeTree') - -\d+ clickhouse_bin.types2; - Foreign table "clickhouse_bin.types2" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | text | | | | | extended | | -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'types2', engine 'MergeTree') - -\d+ clickhouse_bin.arrays; - Foreign table "clickhouse_bin.arrays" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | integer[] | | not null | | | extended | | - c2 | text[] | | not null | | | extended | | -Not-null constraints: - "arrays_c1_not_null" NOT NULL "c1" - "arrays_c2_not_null" NOT NULL "c2" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'arrays', engine 'MergeTree') - -\d+ clickhouse_bin.tuples; - Foreign table "clickhouse_bin.tuples" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | text | | not null | | | extended | | - c3.a | integer[] | | not null | | | extended | | - c3.b | integer[] | | not null | | | extended | | - c4 | smallint | | not null | | | plain | | -Not-null constraints: - "tuples_c1_not_null" NOT NULL "c1" - "tuples_c2_not_null" NOT NULL "c2" - "tuples_c3.a_not_null" NOT NULL "c3.a" - "tuples_c3.b_not_null" NOT NULL "c3.b" - "tuples_c4_not_null" NOT NULL "c4" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'tuples', engine 'MergeTree') - -\d+ clickhouse_bin.timezones; - Foreign table "clickhouse_bin.timezones" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+---------+--------------+------------- - t1 | timestamp without time zone | | not null | | | plain | | - t2 | timestamp without time zone | | not null | | | plain | | - t4 | timestamp without time zone | | not null | | | plain | | - t5 | timestamp without time zone | | not null | | | plain | | -Not-null constraints: - "timezones_t1_not_null" NOT NULL "t1" - "timezones_t2_not_null" NOT NULL "t2" - "timezones_t4_not_null" NOT NULL "t4" - "timezones_t5_not_null" NOT NULL "t5" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'timezones', engine 'MergeTree') - -\d+ clickhouse_bin.ip; - Foreign table "clickhouse_bin.ip" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | inet | | not null | | | main | | - c2 | inet | | not null | | | main | | -Not-null constraints: - "ip_c1_not_null" NOT NULL "c1" - "ip_c2_not_null" NOT NULL "c2" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'ip', engine 'MergeTree') - -SELECT * FROM clickhouse_bin.ints ORDER BY c1 DESC LIMIT 4; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -----+----+----+----+----+----+----+----+------+------ - 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19.1 | - 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18.1 | - 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17.1 | 18.2 - 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16.1 | 17.2 -(4 rows) - -SELECT * FROM clickhouse_bin.types ORDER BY c1 LIMIT 2; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c9 | c8 -------------+---------------------+----------+-------+--------------------------------------+-----+-------+----+------------ - 1990-01-01 | 1990-01-01 10:00:00 | number 0 | num 0 | f4bf890f-f9dc-4332-ad5c-0c18e73f28e0 | two | three | 0 | cardinal 0 - 1990-01-02 | 1990-01-02 10:01:01 | number 1 | num 1 | f4bf890f-f9dc-4332-ad5c-0c18e73f28e1 | two | three | 1 | cardinal 1 -(2 rows) - -SELECT * FROM clickhouse_bin.types2 ORDER BY c1 LIMIT 2; - c1 -------------- - cardinal 1 - cardinal 10 -(2 rows) - -SELECT * FROM clickhouse_bin.arrays ORDER BY c1 LIMIT 2; - c1 | c2 --------+------------- - {0,1} | {num0,num1} - {1,2} | {num1,num2} -(2 rows) - -SELECT * FROM clickhouse_bin.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+---------+---------+---------+---- - 0 | (0,0,1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,1,2) | {1,1,1} | {1,2,2} | 1 -(2 rows) - -SELECT * FROM clickhouse_bin.timezones ORDER BY t1 LIMIT 2; - t1 | t2 | t4 | t5 ----------------------+---------------------+---------------------+--------------------- - 2020-01-01 11:00:00 | 2020-01-01 10:00:00 | 2020-01-01 10:00:00 | 2020-01-01 11:00:00 - 2020-01-01 12:00:00 | 2020-01-01 11:00:00 | 2020-01-01 11:00:00 | 2020-01-01 12:00:00 -(2 rows) - -SELECT * FROM clickhouse.ip ORDER BY c1; - c1 | c2 -----------------+------------------------------- - 116.106.34.242 | 2001:44c8:129:2632:33:0:252:2 - 116.106.34.243 | 2a02:e980:1e::1 - 116.106.34.244 | ::1 -(3 rows) - -IMPORT FOREIGN SCHEMA "import_test" LIMIT TO (ints, types) FROM SERVER import_loopback INTO clickhouse_limit; -\d+ clickhouse_limit.ints; - Foreign table "clickhouse_limit.ints" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------------------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | smallint | | not null | | | plain | | - c3 | integer | | not null | | | plain | | - c4 | bigint | | not null | | | plain | | - c5 | smallint | | not null | | | plain | | - c6 | integer | | not null | | | plain | | - c7 | bigint | | not null | | | plain | | - c8 | bigint | | not null | | | plain | | - c9 | real | | not null | | | plain | | - c10 | double precision | | | | | plain | | -Not-null constraints: - "ints_c1_not_null" NOT NULL "c1" - "ints_c2_not_null" NOT NULL "c2" - "ints_c3_not_null" NOT NULL "c3" - "ints_c4_not_null" NOT NULL "c4" - "ints_c5_not_null" NOT NULL "c5" - "ints_c6_not_null" NOT NULL "c6" - "ints_c7_not_null" NOT NULL "c7" - "ints_c8_not_null" NOT NULL "c8" - "ints_c9_not_null" NOT NULL "c9" -Server: import_loopback -FDW options: (database 'import_test', table_name 'ints', engine 'MergeTree') - -\d+ clickhouse_limit.types; - Foreign table "clickhouse_limit.types" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | date | | not null | | | plain | | - c2 | timestamp without time zone | | not null | | | plain | | - c3 | text | | not null | | | extended | | - c4 | character varying(5) | | not null | | | extended | | - c5 | uuid | | not null | | | plain | | - c6 | text | | not null | | | extended | | - c7 | text | | not null | | | extended | | - c9 | character varying(50) | | | | | extended | | - c8 | text | | not null | | | extended | | -Not-null constraints: - "types_c1_not_null" NOT NULL "c1" - "types_c2_not_null" NOT NULL "c2" - "types_c3_not_null" NOT NULL "c3" - "types_c4_not_null" NOT NULL "c4" - "types_c5_not_null" NOT NULL "c5" - "types_c6_not_null" NOT NULL "c6" - "types_c7_not_null" NOT NULL "c7" - "types_c8_not_null" NOT NULL "c8" -Server: import_loopback -FDW options: (database 'import_test', table_name 'types', engine 'MergeTree') - -\d+ clickhouse_limit.arrays; -\d+ clickhouse_limit.tuples; -IMPORT FOREIGN SCHEMA "import_test" EXCEPT (ints, types) FROM SERVER import_loopback INTO clickhouse_except; -NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "c2", please create composite type and alter the column if needed -\d+ clickhouse_except.ints; -\d+ clickhouse_except.types; -\d+ clickhouse_except.arrays; - Foreign table "clickhouse_except.arrays" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | integer[] | | not null | | | extended | | - c2 | text[] | | not null | | | extended | | -Not-null constraints: - "arrays_c1_not_null" NOT NULL "c1" - "arrays_c2_not_null" NOT NULL "c2" -Server: import_loopback -FDW options: (database 'import_test', table_name 'arrays', engine 'MergeTree') - -\d+ clickhouse_except.tuples; - Foreign table "clickhouse_except.tuples" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | text | | not null | | | extended | | - c3.a | integer[] | | not null | | | extended | | - c3.b | integer[] | | not null | | | extended | | - c4 | smallint | | not null | | | plain | | -Not-null constraints: - "tuples_c1_not_null" NOT NULL "c1" - "tuples_c2_not_null" NOT NULL "c2" - "tuples_c3.a_not_null" NOT NULL "c3.a" - "tuples_c3.b_not_null" NOT NULL "c3.b" - "tuples_c4_not_null" NOT NULL "c4" -Server: import_loopback -FDW options: (database 'import_test', table_name 'tuples', engine 'MergeTree') - --- check custom database -SELECT clickhouse_raw_query('CREATE TABLE import_test_2.custom_option (a Int64) ENGINE = MergeTree ORDER BY (a)'); - clickhouse_raw_query ----------------------- - -(1 row) - -IMPORT FOREIGN SCHEMA "import_test_2" FROM SERVER import_loopback INTO clickhouse; -EXPLAIN VERBOSE SELECT * FROM clickhouse.custom_option; - QUERY PLAN ---------------------------------------------------------------------------------- - Foreign Scan on clickhouse.custom_option (cost=10.00..20.00 rows=1000 width=8) - Output: a - Remote SQL: SELECT a FROM import_test_2.custom_option -(3 rows) - -ALTER FOREIGN TABLE clickhouse.custom_option OPTIONS (DROP database); -EXPLAIN VERBOSE SELECT * FROM clickhouse.custom_option; - QUERY PLAN ---------------------------------------------------------------------------------- - Foreign Scan on clickhouse.custom_option (cost=10.00..20.00 rows=1000 width=8) - Output: a - Remote SQL: SELECT a FROM import_test.custom_option -(3 rows) - --- check overflows. -SELECT clickhouse_raw_query($$ - INSERT INTO import_test.ints (c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) VALUES - ( - -- Min values - -128, -32768, -2147483648, -9223372036854775808, - 0, 0, 0, 0, - 1.175494351e-38, 2.2250738585072014e-308 - ), - ( - -- Max values - 127, 32767, 2147483647, 9223372036854775807, - 255, 65535, 4294967295, 18446744073709551615, - 3.402823466e+38, 1.7976931348623158e+308 - ) -$$); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query($$ - SELECT * FROM import_test.ints - WHERE c1 IN (127, -128) - ORDER BY c1; -$$); - clickhouse_raw_query --------------------------------------------------------------------------------------------------------------------------------------------------------- - -128 -32768 -2147483648 -9223372036854775808 0 0 0 0 1.1754944e-38 2.2250738585072014e-308 + - 127 32767 2147483647 9223372036854775807 255 65535 4294967295 18446744073709551615 3.4028235e38 1.7976931348623157e308+ - -(1 row) - --- Error on 18446744073709551615. -SELECT * FROM clickhouse_bin.ints -WHERE c1 IN (127, -128) -ORDER BY c1; -ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint -SELECT * FROM clickhouse.ints -WHERE c1 IN (127, -128) -ORDER BY c1; -ERROR: value "18446744073709551615" is out of range for type bigint --- Ignore 18446744073709551615 -SELECT * FROM clickhouse_bin.ints WHERE c1 = -128 -UNION -SELECT c1, c2, c3, c4, c5, c6, c7, NULL, c9, c10 -FROM clickhouse_bin.ints -WHERE c1 = 127 -ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -------+--------+-------------+----------------------+-----+----+------------+----+---------------+------------------------- - -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754944e-38 | 2.2250738585072014e-308 - 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | -1 | 4294967295 | | 3.4028235e+38 | 1.7976931348623157e+308 -(2 rows) - -SELECT * FROM clickhouse.ints WHERE c1 = -128 -UNION -SELECT c1, c2, c3, c4, c5, c6, c7, NULL, c9, c10 -FROM clickhouse.ints -WHERE c1 = 127 -ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -------+--------+-------------+----------------------+-----+-------+------------+----+---------------+------------------------- - -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754944e-38 | 2.2250738585072014e-308 - 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | 65535 | 4294967295 | | 3.4028235e+38 | 1.7976931348623157e+308 -(2 rows) - -DROP USER MAPPING FOR CURRENT_USER SERVER import_loopback; -DROP USER MAPPING FOR CURRENT_USER SERVER import_loopback_bin; -SELECT clickhouse_raw_query('DROP DATABASE import_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('DROP DATABASE import_test_2'); - clickhouse_raw_query ----------------------- - -(1 row) - -DROP SERVER import_loopback_bin CASCADE; -NOTICE: drop cascades to 7 other objects -DETAIL: drop cascades to foreign table clickhouse_bin.arrays -drop cascades to foreign table clickhouse_bin.ints -drop cascades to foreign table clickhouse_bin.ip -drop cascades to foreign table clickhouse_bin.timezones -drop cascades to foreign table clickhouse_bin.tuples -drop cascades to foreign table clickhouse_bin.types -drop cascades to foreign table clickhouse_bin.types2 -DROP SERVER import_loopback CASCADE; -NOTICE: drop cascades to 15 other objects -DETAIL: drop cascades to foreign table clickhouse.arrays -drop cascades to foreign table clickhouse.ints -drop cascades to foreign table clickhouse.ip -drop cascades to foreign table clickhouse.timezones -drop cascades to foreign table clickhouse.tuples -drop cascades to foreign table clickhouse.types -drop cascades to foreign table clickhouse.types2 -drop cascades to foreign table clickhouse_limit.ints -drop cascades to foreign table clickhouse_limit.types -drop cascades to foreign table clickhouse_except.arrays -drop cascades to foreign table clickhouse_except.ip -drop cascades to foreign table clickhouse_except.timezones -drop cascades to foreign table clickhouse_except.tuples -drop cascades to foreign table clickhouse_except.types2 -drop cascades to foreign table clickhouse.custom_option diff --git a/test/expected/json_3.out b/test/expected/json_3.out index b7a968f2..ee946eb3 100644 --- a/test/expected/json_3.out +++ b/test/expected/json_3.out @@ -58,7 +58,7 @@ INSERT INTO json_http.things VALUES (4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}') ; SELECT * FROM json_bin.things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.things ORDER BY id; id | data @@ -82,7 +82,7 @@ ERROR: pg_clickhouse: could not finish INSERT - DB::Exception: Invalid version INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; id | data @@ -123,7 +123,7 @@ SELECT data['name'] FROM json_bin.things; (3 rows) SELECT data['name'] FROM json_bin.things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- DISTINCT forces an ORDER BY or HashAgg; the subscript must stay local. EXPLAIN (VERBOSE, COSTS OFF) @@ -158,7 +158,7 @@ SELECT DISTINCT data['size'] FROM json_bin.things; (6 rows) SELECT DISTINCT data['size'] FROM json_bin.things; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT data FROM json_test.things -- GROUP BY with a JSON subscript expression. EXPLAIN (VERBOSE, COSTS OFF) @@ -193,7 +193,7 @@ SELECT data['size'], count(*) FROM json_bin.things GROUP BY data['size']; (6 rows) SELECT data['size'], count(*) FROM json_bin.things GROUP BY data['size']; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT data FROM json_test.things -- The jsonb ->> operator runs locally when used in SELECT target lists. EXPLAIN (VERBOSE, COSTS OFF) @@ -223,7 +223,7 @@ SELECT data ->> 'name' FROM json_bin.things; (3 rows) SELECT data ->> 'name' FROM json_bin.things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- The json ->> operator runs locally when used in SELECT target lists. EXPLAIN (VERBOSE, COSTS OFF) @@ -253,7 +253,7 @@ SELECT data ->> 'name' FROM json_bin.json_things; (3 rows) SELECT data ->> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb ->> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) @@ -454,12 +454,12 @@ SELECT * FROM json_http.things SELECT * FROM json_bin.things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST SELECT * FROM json_bin.things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_things @@ -483,12 +483,12 @@ SELECT * FROM json_http.json_things SELECT * FROM json_bin.json_things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST SELECT * FROM json_bin.json_things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST -- ORDER BY with jsonb ->> pushdown. EXPLAIN (VERBOSE, COSTS OFF) @@ -518,7 +518,7 @@ SELECT * FROM json_bin.things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.things ORDER BY data ->> 'name'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.things ORDER BY data ->> 'name' LIMIT 2; @@ -557,7 +557,7 @@ SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.json_things ORDER BY data ->> 'name' LIMIT 2; @@ -596,7 +596,7 @@ SELECT data -> 'name' FROM json_bin.things; (3 rows) SELECT data -> 'name' FROM json_bin.things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_http.json_things; @@ -625,7 +625,7 @@ SELECT data -> 'name' FROM json_bin.json_things; (3 rows) SELECT data -> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) @@ -711,7 +711,7 @@ SELECT * FROM json_bin.things WHERE data -> 'stocked' = 'true'::jsonb; (3 rows) SELECT * FROM json_bin.things WHERE data -> 'stocked' = 'true'::jsonb ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE ((toJSONString(data.stocked) = 'true')) ORDER BY id ASC NULLS LAST -- WHERE clause with json -> JSON boolean literal must push down. EXPLAIN (VERBOSE, COSTS OFF) @@ -739,7 +739,7 @@ SELECT * FROM json_bin.json_things WHERE (data -> 'stocked')::text = 'true'; (3 rows) SELECT * FROM json_bin.json_things WHERE (data -> 'stocked')::text = 'true' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE ((CAST(toJSONString(data.stocked) AS String) = 'true')) ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> wraps the dot notation in toJSONString() so the -- result is a proper JSON value (->, unlike ->>, returns jsonb). @@ -852,7 +852,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; @@ -879,7 +879,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) -- Key with mixed case. EXPLAIN (VERBOSE, COSTS OFF) @@ -907,7 +907,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; @@ -934,7 +934,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) -- Key that is a SQL reserved word. EXPLAIN (VERBOSE, COSTS OFF) @@ -962,7 +962,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; @@ -989,7 +989,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) -- Key containing embedded double quotes. EXPLAIN (VERBOSE, COSTS OFF) @@ -1396,7 +1396,7 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) @@ -1425,7 +1425,7 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: jsonb_extract_path (returns jsonb, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) @@ -1454,7 +1454,7 @@ SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level jsonb_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) @@ -1589,7 +1589,7 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) @@ -1618,7 +1618,7 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_event (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: json_extract_path (returns json, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) @@ -1647,7 +1647,7 @@ SELECT json_extract_path(props, 'address') FROM json_bin.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level json_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) diff --git a/test/expected/json_4.out b/test/expected/json_4.out index 1a2eca07..e9d22520 100644 --- a/test/expected/json_4.out +++ b/test/expected/json_4.out @@ -71,7 +71,8 @@ ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Objec INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; ERROR: invalid input syntax for type json DETAIL: Token "(" is invalid. @@ -179,7 +180,8 @@ SELECT data ->> 'name' FROM json_bin.json_things; (3 rows) SELECT data ->> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb ->> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data ->> 'name' = 'widget'; @@ -439,7 +441,8 @@ SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.json_things ORDER BY data ->> 'name' LIMIT 2; ERROR: pg_clickhouse: Code: 115. DB::Exception: Setting allow_suspicious_types_in_order_by is neither a builtin setting nor started with the prefix 'SQL_' registered for user-defined settings. (UNKNOWN_SETTING) @@ -491,7 +494,8 @@ SELECT data -> 'name' FROM json_bin.json_things; (3 rows) SELECT data -> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data -> 'name' = '"widget"'; @@ -677,7 +681,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; @@ -702,7 +706,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) -- Key with mixed case. EXPLAIN (VERBOSE, COSTS OFF) @@ -728,7 +732,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; @@ -753,7 +757,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) -- Key that is a SQL reserved word. EXPLAIN (VERBOSE, COSTS OFF) @@ -779,7 +783,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; @@ -804,7 +808,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) -- Key containing embedded double quotes. EXPLAIN (VERBOSE, COSTS OFF) @@ -1172,7 +1176,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; @@ -1197,7 +1202,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: jsonb_extract_path (returns jsonb, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; @@ -1222,7 +1228,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level jsonb_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.events WHERE jsonb_extract_path_text(props, 'customerId') = 'C100'; @@ -1353,7 +1360,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events; @@ -1378,7 +1386,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_event (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: json_extract_path (returns json, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_http.json_events; @@ -1403,7 +1412,8 @@ SELECT json_extract_path(props, 'address') FROM json_bin.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level json_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.json_events WHERE json_extract_path_text(props, 'customerId') = 'C100'; diff --git a/test/expected/json_5.out b/test/expected/json_5.out index 716a1df2..05b65248 100644 --- a/test/expected/json_5.out +++ b/test/expected/json_5.out @@ -71,7 +71,8 @@ ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Objec INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; ERROR: invalid input syntax for type json DETAIL: Token "(" is invalid. @@ -179,7 +180,8 @@ SELECT data ->> 'name' FROM json_bin.json_things; (3 rows) SELECT data ->> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb ->> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data ->> 'name' = 'widget'; @@ -439,7 +441,8 @@ SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.json_things ORDER BY data ->> 'name' LIMIT 2; ERROR: pg_clickhouse: Code: 115. DB::Exception: Setting allow_suspicious_types_in_order_by is neither a builtin setting nor started with the prefix 'SQL_' registered for user-defined settings. (UNKNOWN_SETTING) @@ -491,7 +494,8 @@ SELECT data -> 'name' FROM json_bin.json_things; (3 rows) SELECT data -> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data -> 'name' = '"widget"'; @@ -677,7 +681,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; @@ -702,7 +706,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) -- Key with mixed case. EXPLAIN (VERBOSE, COSTS OFF) @@ -728,7 +732,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; @@ -753,7 +757,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) -- Key that is a SQL reserved word. EXPLAIN (VERBOSE, COSTS OFF) @@ -779,7 +783,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; @@ -804,7 +808,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) -- Key containing embedded double quotes. EXPLAIN (VERBOSE, COSTS OFF) @@ -1172,7 +1176,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; @@ -1197,7 +1202,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: jsonb_extract_path (returns jsonb, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; @@ -1222,7 +1228,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level jsonb_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.events WHERE jsonb_extract_path_text(props, 'customerId') = 'C100'; @@ -1353,7 +1360,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events; @@ -1378,7 +1386,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_event (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: json_extract_path (returns json, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_http.json_events; @@ -1403,7 +1412,8 @@ SELECT json_extract_path(props, 'address') FROM json_bin.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level json_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.json_events WHERE json_extract_path_text(props, 'customerId') = 'C100'; diff --git a/test/expected/json_6.out b/test/expected/json_6.out index 3ba355f3..1aa614e4 100644 --- a/test/expected/json_6.out +++ b/test/expected/json_6.out @@ -71,7 +71,8 @@ ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Objec INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; ERROR: invalid input syntax for type json DETAIL: Token "(" is invalid. @@ -179,7 +180,8 @@ SELECT data ->> 'name' FROM json_bin.json_things; (3 rows) SELECT data ->> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb ->> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data ->> 'name' = 'widget'; @@ -439,7 +441,8 @@ SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.json_things ORDER BY data ->> 'name' LIMIT 2; ERROR: pg_clickhouse: Code: 115. DB::Exception: Unknown setting allow_suspicious_types_in_order_by. (UNKNOWN_SETTING) @@ -491,7 +494,8 @@ SELECT data -> 'name' FROM json_bin.json_things; (3 rows) SELECT data -> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data -> 'name' = '"widget"'; @@ -677,7 +681,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; @@ -702,7 +706,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) -- Key with mixed case. EXPLAIN (VERBOSE, COSTS OFF) @@ -728,7 +732,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; @@ -753,7 +757,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) -- Key that is a SQL reserved word. EXPLAIN (VERBOSE, COSTS OFF) @@ -779,7 +783,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; @@ -804,7 +808,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) -- Key containing embedded double quotes. EXPLAIN (VERBOSE, COSTS OFF) @@ -1172,7 +1176,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; @@ -1197,7 +1202,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: jsonb_extract_path (returns jsonb, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; @@ -1222,7 +1228,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level jsonb_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.events WHERE jsonb_extract_path_text(props, 'customerId') = 'C100'; @@ -1353,7 +1360,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events; @@ -1378,7 +1386,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_event (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: json_extract_path (returns json, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_http.json_events; @@ -1403,7 +1412,8 @@ SELECT json_extract_path(props, 'address') FROM json_bin.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level json_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.json_events WHERE json_extract_path_text(props, 'customerId') = 'C100'; diff --git a/test/expected/result_map.txt b/test/expected/result_map.txt index de31d92e..9bbe45f2 100644 --- a/test/expected/result_map.txt +++ b/test/expected/result_map.txt @@ -46,7 +46,7 @@ binary_inserts.sql ClickHouse | File ------------|-------------------- - 22-26 | binary_inserts.out + 23+ | binary_inserts.out binary_queries.sql ------------------ @@ -61,10 +61,8 @@ binary_queries.sql ClickHouse | File ------------|---------------------- 26 | binary_queries.out - 25 | binary_queries_4.out - 23-24 | binary_queries_5.out - 23 | binary_queries_6.out - 22 | binary_queries_7.out + 23.8-25 | binary_queries_4.out + 23.3 | binary_queries_5.out deparse_checks.sql ------------------ @@ -76,7 +74,7 @@ deparse_checks.sql ClickHouse | File ------------|-------------------- - 22+ | deparse_checks.out + 23+ | deparse_checks.out engines.sql ----------- @@ -88,7 +86,7 @@ engines.sql ClickHouse | File ------------|------------- - 22+ | engines.out + 23+ | engines.out functions.sql ------------- @@ -158,7 +156,6 @@ import_schema.sql ClickHouse | File ------------|--------------------- 23+ | import_schema.out - 22 | import_schema_2.out json.sql -------- @@ -223,7 +220,7 @@ subquery_pushdown.sql ClickHouse | File ------------|----------------------- - 22+ | subquery_pushdown.out + 23+ | subquery_pushdown.out timezone.sql ------------ @@ -247,7 +244,7 @@ where_sub.sql ClickHouse | File ------------|----------------- - 22+ | where_sub.out + 23+ | where_sub.out window_functions.sql -------------------- diff --git a/test/sql/import_schema.sql b/test/sql/import_schema.sql index 34e6c8e0..211c051f 100644 --- a/test/sql/import_schema.sql +++ b/test/sql/import_schema.sql @@ -12,6 +12,7 @@ CREATE USER MAPPING FOR CURRENT_USER SERVER import_loopback_bin; SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test'); SELECT clickhouse_raw_query('CREATE DATABASE import_test'); +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test_2'); SELECT clickhouse_raw_query('CREATE DATABASE import_test_2'); -- integer types diff --git a/vendor/clickhouse-c b/vendor/clickhouse-c new file mode 160000 index 00000000..5df8f6e8 --- /dev/null +++ b/vendor/clickhouse-c @@ -0,0 +1 @@ +Subproject commit 5df8f6e8ee2c978a808b96c5ccc1142c73fe6d49 diff --git a/vendor/clickhouse-cpp b/vendor/clickhouse-cpp deleted file mode 160000 index f067937e..00000000 --- a/vendor/clickhouse-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f067937e47ff75767709b7471821c6cefbea7666