diff --git a/.clang-format b/.clang-format deleted file mode 100644 index a2832f58..00000000 --- a/.clang-format +++ /dev/null @@ -1,295 +0,0 @@ ---- -# Language: Cpp -AccessModifierOffset: -2 -AlignAfterOpenBracket: Align -AlignArrayOfStructures: None -AlignConsecutiveAssignments: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: true -AlignConsecutiveBitFields: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignConsecutiveDeclarations: - Enabled: true - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: true - AlignFunctionPointers: true - PadOperators: false -AlignConsecutiveMacros: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignConsecutiveShortCaseStatements: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCaseArrows: false - AlignCaseColons: false -AlignConsecutiveTableGenBreakingDAGArgColons: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignConsecutiveTableGenCondOperatorColons: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignConsecutiveTableGenDefinitionColons: - Enabled: false - AcrossEmptyLines: false - AcrossComments: false - AlignCompound: false - AlignFunctionDeclarations: false - AlignFunctionPointers: false - PadOperators: false -AlignEscapedNewlines: Right -AlignOperands: Align -AlignTrailingComments: - Kind: Always - OverEmptyLines: 0 -AllowAllArgumentsOnNextLine: true -AllowAllParametersOfDeclarationOnNextLine: true -AllowBreakBeforeNoexceptSpecifier: Never -AllowShortBlocksOnASingleLine: Never -AllowShortCaseExpressionOnASingleLine: true -AllowShortCaseLabelsOnASingleLine: false -AllowShortCompoundRequirementOnASingleLine: true -AllowShortEnumsOnASingleLine: true -AllowShortFunctionsOnASingleLine: All -AllowShortIfStatementsOnASingleLine: Never -AllowShortLambdasOnASingleLine: All -AllowShortLoopsOnASingleLine: false -AllowShortNamespacesOnASingleLine: false -AlwaysBreakAfterDefinitionReturnType: All -AlwaysBreakBeforeMultilineStrings: false -AttributeMacros: - - __capability -BinPackArguments: true -BinPackLongBracedList: true -BinPackParameters: BinPack -BitFieldColonSpacing: Both -BracedInitializerIndentWidth: -1 -BraceWrapping: - AfterCaseLabel: true - AfterClass: true - AfterControlStatement: Always - AfterEnum: true - AfterExternBlock: true - AfterFunction: true - AfterNamespace: true - AfterObjCDeclaration: true - AfterStruct: true - AfterUnion: true - BeforeCatch: true - BeforeElse: true - BeforeLambdaBody: true - BeforeWhile: true - IndentBraces: false - SplitEmptyFunction: true - SplitEmptyRecord: true - SplitEmptyNamespace: true -BreakAdjacentStringLiterals: true -BreakAfterAttributes: Leave -BreakAfterJavaFieldAnnotations: false -BreakAfterReturnType: AllDefinitions -BreakArrays: true -BreakBeforeBinaryOperators: All -BreakBeforeConceptDeclarations: Always -BreakBeforeBraces: Custom -BreakBeforeInlineASMColon: OnlyMultiline -BreakBeforeTemplateCloser: false -BreakBeforeTernaryOperators: true -BreakBinaryOperations: Never -BreakConstructorInitializers: BeforeColon -BreakFunctionDefinitionParameters: false -BreakInheritanceList: BeforeColon -BreakStringLiterals: true -BreakTemplateDeclarations: MultiLine -ColumnLimit: 120 -CommentPragmas: "^ IWYU pragma:" -CompactNamespaces: false -ConstructorInitializerIndentWidth: 4 -ContinuationIndentWidth: 0 -Cpp11BracedListStyle: false -DerivePointerAlignment: false -DisableFormat: false -EmptyLineAfterAccessModifier: Never -EmptyLineBeforeAccessModifier: LogicalBlock -EnumTrailingComma: Leave -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: false -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH -IfMacros: - - KJ_IF_MAYBE -IncludeBlocks: Preserve -IncludeCategories: - - Regex: '^"(llvm|llvm-c|clang|clang-c)/' - Priority: 2 - SortPriority: 0 - CaseSensitive: false - - Regex: '^(<|"(gtest|gmock|isl|json)/)' - Priority: 3 - SortPriority: 0 - CaseSensitive: false - - Regex: ".*" - Priority: 1 - SortPriority: 0 - CaseSensitive: false -IncludeIsMainRegex: "(Test)?$" -IncludeIsMainSourceRegex: "" -IndentAccessModifiers: false -IndentCaseBlocks: false -IndentCaseLabels: true -IndentExportBlock: true -IndentExternBlock: AfterExternBlock -IndentGotoLabels: true -IndentPPDirectives: None -IndentRequiresClause: true -IndentWidth: 4 -IndentWrappedFunctionNames: false -InsertBraces: false -InsertNewlineAtEOF: false -InsertTrailingCommas: None -IntegerLiteralSeparator: - Binary: 0 - BinaryMinDigits: 0 - Decimal: 0 - DecimalMinDigits: 0 - Hex: 0 - HexMinDigits: 0 -JavaScriptQuotes: Leave -JavaScriptWrapImports: true -KeepEmptyLines: - AtEndOfFile: false - AtStartOfBlock: true - AtStartOfFile: true -KeepFormFeed: true -LambdaBodyIndentation: Signature -LineEnding: DeriveLF -MacroBlockBegin: "" -MacroBlockEnd: "" -MainIncludeChar: Quote -MaxEmptyLinesToKeep: 1 -NamespaceIndentation: None -ObjCBinPackProtocolList: Auto -ObjCBlockIndentWidth: 2 -ObjCBreakBeforeNestedBlockParam: true -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: true -OneLineFormatOffRegex: "" -PackConstructorInitializers: BinPack -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 19 -PenaltyBreakBeforeMemberAccess: 150 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakOpenParenthesis: 0 -PenaltyBreakScopeResolution: 500 -PenaltyBreakString: 1000 -PenaltyBreakTemplateDeclaration: 10 -PenaltyExcessCharacter: 1000000 -PenaltyIndentedWhitespace: 0 -PenaltyReturnTypeOnItsOwnLine: 60 -PointerAlignment: Right -PPIndentWidth: -1 -QualifierAlignment: Leave -ReferenceAlignment: Right -ReflowComments: Always -RemoveBracesLLVM: false -RemoveEmptyLinesInUnwrappedLines: false -RemoveParentheses: Leave -RemoveSemicolon: false -RequiresClausePosition: OwnLine -RequiresExpressionIndentation: OuterScope -SeparateDefinitionBlocks: Leave -ShortNamespaceLines: 1 -SkipMacroDefinitionBody: false -SortIncludes: - Enabled: true - IgnoreCase: false -SortJavaStaticImport: Before -SortUsingDeclarations: LexicographicNumeric -SpaceAfterCStyleCast: false -SpaceAfterLogicalNot: false -SpaceAfterOperatorKeyword: false -SpaceAfterTemplateKeyword: true -SpaceAroundPointerQualifiers: Default -SpaceBeforeAssignmentOperators: true -SpaceBeforeCaseColon: false -SpaceBeforeCpp11BracedList: false -SpaceBeforeCtorInitializerColon: true -SpaceBeforeInheritanceColon: true -SpaceBeforeJsonColon: false -SpaceBeforeParens: ControlStatements -SpaceBeforeParensOptions: - AfterControlStatements: false - AfterForeachMacros: false - AfterFunctionDefinitionName: false - AfterFunctionDeclarationName: false - AfterIfMacros: false - AfterNot: false - AfterOverloadedOperator: false - AfterPlacementOperator: true - AfterRequiresInClause: false - AfterRequiresInExpression: false - BeforeNonEmptyParentheses: false -SpaceBeforeRangeBasedForLoopColon: true -SpaceBeforeSquareBrackets: false -SpaceInEmptyBlock: false -SpacesBeforeTrailingComments: 1 -SpacesInAngles: Never -SpacesInContainerLiterals: true -SpacesInLineCommentPrefix: - Minimum: 1 - Maximum: -1 -SpacesInParens: Never -SpacesInParensOptions: - ExceptDoubleParentheses: false - InCStyleCasts: false - InConditionalStatements: false - InEmptyParentheses: false - Other: false -SpacesInSquareBrackets: false -Standard: Latest -StatementAttributeLikeMacros: - - Q_EMIT -StatementMacros: - - Q_UNUSED - - QT_REQUIRE_VERSION -TableGenBreakInsideDAGArg: DontBreak -TabWidth: 4 -UseTab: Always -VerilogBreakBetweenInstancePorts: true -WhitespaceSensitiveMacros: - - BOOST_PP_STRINGIZE - - CF_SWIFT_NAME - - NS_SWIFT_NAME - - PP_STRINGIZE - - STRINGIZE -WrapNamespaceBodyWithEmptyLines: Leave diff --git a/.github/workflows/bake.yml b/.github/workflows/bake.yml index 1e855771..8d216a24 100644 --- a/.github/workflows/bake.yml +++ b/.github/workflows/bake.yml @@ -29,11 +29,6 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Vendor Build - uses: actions/cache@v4 - with: - path: vendor/_build - key: ${{ runner.os }}-vendor-build - name: Set Bake Variables run: make bake-vars REGISTRY=ghcr.io/clickhouse PG_VERSIONS=${{ matrix.pgv }} >> $GITHUB_ENV - name: Build${{ startsWith(github.ref, 'refs/tags') && ' and Push' || '' }} diff --git a/.github/workflows/clickhouse.yml b/.github/workflows/clickhouse.yml index 441ab1ce..b7420157 100644 --- a/.github/workflows/clickhouse.yml +++ b/.github/workflows/clickhouse.yml @@ -27,7 +27,7 @@ jobs: container: pgxn/pgxn-tools steps: - name: Start Postgres - run: pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev libre2-dev + run: pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev libre2-dev - name: Install Extensions run: pgxn install re2 - name: Checkout the Repository @@ -36,19 +36,5 @@ jobs: - name: Start ClickHouse env: { CH_RELEASE: "${{ matrix.ch }}" } run: .github/ubuntu/clickhouse.sh - - name: Cache Dependencies - uses: actions/cache@v5 - with: - path: vendor/_build/* - key: vendor-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.git/modules/clickhouse-cpp/refs/heads/master') }} - # Required to prevent clickhouse-cpp from rebuilding because it depends - # on this file in the Makefile. https://github.com/actions/checkout/issues/968 - - name: Reset Vendor Timestamp - run: cd vendor/clickhouse-cpp && touch -d $(git log -1 --format="@%ct" CMakeLists.txt) CMakeLists.txt - - name: Test DSO + - name: Test run: pg-build-test - - name: Clean - run: make clean NO_VENDOR_CLEAN=1 - - name: Test Static - run: pg-build-test - env: { CH_BUILD: static } diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 76bc162d..43273af8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,9 +14,7 @@ jobs: env: { pg: 18 } steps: - name: Install Postgres - run: NO_CLUSTER=1 pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev clang-tidy - - name: Install clang-format # version in Ubuntu is too old - run: curl -sLo /usr/local/bin/clang-format https://github.com/cpp-linter/clang-tools-static-binaries/releases/latest/download/clang-format-22_linux-amd64 && chmod +x /usr/local/bin/clang-format + run: NO_CLUSTER=1 pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev clang-tidy - name: Checkout the Repository uses: actions/checkout@v6 - name: Install pre-commit diff --git a/.github/workflows/postgres.yml b/.github/workflows/postgres.yml index 6f44f561..f2552aaf 100644 --- a/.github/workflows/postgres.yml +++ b/.github/workflows/postgres.yml @@ -19,7 +19,7 @@ jobs: container: pgxn/pgxn-tools steps: - name: Start Postgres ${{ matrix.pg }} - run: pg-start ${{ matrix.pg }} libcurl4-openssl-dev uuid-dev libre2-dev + run: pg-start ${{ matrix.pg }} libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev libre2-dev - name: Install Extensions run: pgxn install re2 - name: Checkout the Repository @@ -27,19 +27,5 @@ jobs: with: { submodules: true } - name: Start ClickHouse run: .github/ubuntu/clickhouse.sh - - name: Cache Dependencies - uses: actions/cache@v5 - with: - path: vendor/_build/* - key: vendor-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.git/modules/clickhouse-cpp/refs/heads/master') }} - # Required to prevent clickhouse-cpp from rebuilding because it depends - # on this file in the Makefile. https://github.com/actions/checkout/issues/968 - - name: Reset Vendor Timestamp - run: cd vendor/clickhouse-cpp && touch -d $(git log -1 --format="@%ct" CMakeLists.txt) CMakeLists.txt - - name: Test DSO + - name: Test run: pg-build-test - - name: Clean - run: make clean NO_VENDOR_CLEAN=1 - - name: Test Static - run: pg-build-test - env: { CH_BUILD: static } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d3ebe1c5..24014b01 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v6 - name: Start Postgres - run: pg-start 18 libcurl4-openssl-dev uuid-dev + run: pg-start 18 libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev - name: Start ClickHouse env: { CH_RELEASE: "${{ matrix.ch }}" } run: .github/ubuntu/clickhouse.sh diff --git a/.gitmodules b/.gitmodules index 55b8de33..bd173a51 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "clickhouse-cpp"] - path = vendor/clickhouse-cpp - url = https://github.com/ClickHouse/clickhouse-cpp.git +[submodule "vendor/clickhouse-c"] + path = vendor/clickhouse-c + url = https://github.com/ClickHouse/clickhouse-c diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88ed6f01..514c14ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,8 +36,8 @@ repos: - id: clang-tidy name: clang-tidy static analysis language: system - entry: sh -c 'test -f compile_commands.json && clang-tidy -p . "$@" || true' -- - files: '\.(c|cpp)$' + entry: sh -c 'test -f compile_commands.json && clang-tidy -p . "$@"' -- + files: '\.c$' exclude: "^vendor/" - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.1.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7add1799..47f44542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ All notable changes to this project will be documented in this file. It uses the ### ⚡ Improvements +* Replaced the `clickhouse-cpp` binary client with [ClickHouse/clickhouse-c], + pulled in as a git submodule and included in the release package under + `vendor/clickhouse-c`. This change eliminates conflicts between the C++ + and Postgres memory & exception handling and streams query results by the + ClickHouse block for reduced memory consumption. It also greatly reduces + build time and the size of the library by over 75%. Thanks to Philip Dubé + for the the new library and the PR ([#254]). * Added multidimensional array support across SELECT and INSERT to both the binary and http drivers. Rectangular ClickHouse `Array(Array(...))` values now map to PostgreSQL multidimensional arrays, jagged arrays not supported, @@ -22,11 +29,21 @@ All notable changes to this project will be documented in this file. It uses the `re2regexpquotemeta`, and `re2splitbyregexp`. Thanks to Philip Dubé for the PR ([#232]). +### 🐞 Bug Fixes + +* Fixed incorrect casting of ClickHouse `UInt16` values to `int16` in the + Binary driver. They now correctly convert to `int32` (Postgres `INT4`). + Part of the omnibus binary c driver conversion contributed by Philip Dubé + ([#233]). + [v0.3.1]: https://github.com/ClickHouse/pg_clickhouse/compare/v0.3.0...v0.3.1 [#232]: https://github.com/ClickHouse/pg_clickhouse/pull/232 "ClickHouse/pg_clickhouse#232 pushdown for new functions in pg_re2 0.3" [#233]: https://github.com/ClickHouse/pg_clickhouse/pull/233 "ClickHouse/pg_clickhouse#233 Support multidimensional arrays" + [ClickHouse/clickhouse-c]: https://github.com/ClickHouse/clickhouse-c + [#254]: https://github.com/ClickHouse/pg_clickhouse/pull/254 + "ClickHouse/pg_clickhouse#254 Replace clickhouse-cpp with clickhouse-c" ## [v0.3.0] — 2026-05-11 diff --git a/Dockerfile b/Dockerfile index 4b8b4ee1..185ecb32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ make \ cmake \ libssl-dev \ + liblz4-dev \ + libzstd-dev \ libre2-dev \ pgxnclient \ unzip \ @@ -24,7 +26,7 @@ RUN cd /tmp && pgxn download re2 && unzip re2-*.zip && rm re2-*.zip && cd re2-* FROM postgres:$PG_MAJOR-trixie # Install dependencies -RUN apt-get update && apt-get install -y --no-install-recommends libcurl4t64 uuid libre2-11 ca-certificates \ +RUN apt-get update && apt-get install -y --no-install-recommends libcurl4t64 uuid libre2-11 liblz4-1 libzstd1 ca-certificates \ && apt-get clean \ && rm -rf /var/cache/apt/* /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index 2d348c8d..2ef4e07b 100644 --- a/Makefile +++ b/Makefile @@ -16,97 +16,42 @@ CURL_CONFIG ?= curl-config OS ?= $(shell uname -s | tr A-Z a-z) ARCH = $(shell uname -m) -# Collect all the C++ and C files to compile into MODULE_big. -OBJS = $(subst .cpp,.o, $(wildcard src/*.cpp src/*/*.cpp)) \ - $(subst .c,.o, $(wildcard src/*.c src/*/*.c)) - -# Build static on Darwin by default. -ifndef CH_BUILD -# ifeq ($(OS),darwin) - CH_BUILD = static -# else -# CH_BUILD = dynamic -# endif -endif +# Collect all the C files to compile into MODULE_big. +OBJS = $(subst .c,.o, $(wildcard src/*.c src/*/*.c)) -# clickhouse-cpp source and build directories. -CH_CPP_DIR = vendor/clickhouse-cpp -CH_CPP_BUILD_DIR = vendor/_build/$(OS)-$(ARCH)-$(CH_BUILD)-$(shell git submodule status $(CH_CPP_DIR) | awk '{print substr($$1, 0, 7)}') - -# List the clickhouse-cpp libraries we require. -CH_CPP_LIB = $(CH_CPP_BUILD_DIR)/clickhouse/libclickhouse-cpp-lib$(DLSUFFIX) -CH_CPP_FLAGS = -D CMAKE_BUILD_TYPE=Release -D WITH_OPENSSL=ON - -# Are we statically compiling clickhouse-cpp into the extension or no? -ifeq ($(CH_BUILD), static) -# We'll need all the clickhouse-cpp static libraries. - CH_CPP_LIB = $(CH_CPP_BUILD_DIR)/clickhouse/libclickhouse-cpp-lib.a - SHLIB_LINK = $(CH_CPP_LIB) \ - $(CH_CPP_BUILD_DIR)/contrib/cityhash/cityhash/libcityhash.a \ - $(CH_CPP_BUILD_DIR)/contrib/absl/absl/libabsl_int128.a \ - $(CH_CPP_BUILD_DIR)/contrib/lz4/lz4/liblz4.a \ - $(CH_CPP_BUILD_DIR)/contrib/zstd/zstd/libzstdstatic.a -else -# Build and install the shared library. - SHLIB_LINK = -L$(CH_CPP_BUILD_DIR)/clickhouse -lclickhouse-cpp-lib - CH_CPP_FLAGS += -D BUILD_SHARED_LIBS=ON -endif +# clickhouse-c is a header-only single-header library. Override +# CH_C_DIR to point elsewhere when developing against a local checkout. +CH_C_DIR ?= vendor/clickhouse-c # Add include directories. -PG_CPPFLAGS = -I./src/include -I$(CH_CPP_DIR) -I$(CH_CPP_DIR)/contrib/absl - -# Include other libraries compiled into clickhouse-cpp. -PG_LDFLAGS = -lstdc++ -lssl -lcrypto $(shell $(CURL_CONFIG) --libs) +PG_CPPFLAGS = -I./src/include -I$(CH_C_DIR) -# clickhouse-cpp requires C++ v17. -PG_CXXFLAGS = -std=c++17 +# Link OpenSSL (for TLS in the binary driver), curl (for the HTTP driver), +# libuuid (for http_streaming.c's query-id generator), and lz4 / zstd +# (for the binary driver's compressed-frame codecs). +PG_LDFLAGS = -lssl -lcrypto -llz4 -lzstd $(shell $(CURL_CONFIG) --libs) -# Suppress annoying pre-c99 warning and include curl flags. -PG_CFLAGS = -Wno-declaration-after-statement -Werror=type-limits $(shell $(CURL_CONFIG) --cflags) - -# We'll need libuuid except on darwin, where it's included in the OS. +# libuuid is provided by the OS on darwin; explicit link elsewhere. ifneq ($(OS),darwin) PG_LDFLAGS += -luuid endif -# Clean up the clickhouse-cpp build directory and generated files. +# Suppress annoying pre-c99 warning and include curl flags. +PG_CFLAGS = -Wno-declaration-after-statement -Werror=type-limits $(shell $(CURL_CONFIG) --cflags) + +# Clean up generated files. EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql src/include/version.h compile_commands.json test/schedule $(EXTENSION)-$(DISTVERSION).zip -ifndef NO_VENDOR_CLEAN - EXTRA_CLEAN += $(CH_CPP_BUILD_DIR) -endif # Import PGXS. PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) -# We'll need the clickhouse-cpp library and rpath so it can be found. -SHLIB_LINK += -Wl,-rpath,$(pkglibdir)/ - -# PostgreSQL 15 and earlier violate a C++ v17 storage specifier error. -ifeq ($(shell test $(MAJORVERSION) -lt 16; echo $$?),0) - PG_CXXFLAGS += -Wno-register -endif - -# Add the flags to the bitcode compiler variables. -COMPILE.cc.bc += $(PG_CPPFLAGS) -COMPILE.cxx.bc += $(PG_CXXFLAGS) - -# shlib is the final output product: clickhouse-cpp and all .o dependencies. -$(shlib): $(CH_CPP_LIB) $(OBJS) - -# Clone clickhouse-cpp submodule. -$(CH_CPP_DIR)/CMakeLists.txt: +# Clone clickhouse-c submodule. +$(CH_C_DIR)/clickhouse.h: .gitmodules git submodule update --init -# Require the vendored clickhouse-cpp and the version header. -$(OBJS): $(CH_CPP_LIB) src/include/version.h - -# Build clickhouse-cpp. -$(CH_CPP_LIB): export CXXFLAGS=-fPIC -$(CH_CPP_LIB): export CFLAGS=-fPIC -$(CH_CPP_LIB): $(CH_CPP_DIR)/CMakeLists.txt # Sync with "Reset Vendor Timestamp" steps in workflows. - cmake -B $(CH_CPP_BUILD_DIR) -S $(CH_CPP_DIR) $(CH_CPP_FLAGS) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - cmake --build $(CH_CPP_BUILD_DIR) --parallel $$(nproc) --target all +# Require clickhouse-c and the version header. +$(OBJS): $(CH_C_DIR)/clickhouse.h src/include/version.h # Require the versioned C source and SQL script. all: sql/$(EXTENSION)--$(EXTVERSION).sql @@ -119,17 +64,6 @@ sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql src/include/version.h: src/include/version.h.in sed -e 's,__VERSION__,$(DISTVERSION),g' $< > $@ -# Configure install/uninstall of the clickhouse-cpp library. -ifneq ($(CH_BUILD), static) -# Copy all dynamic files; use -a to preserve symlinks. -install-ch-cpp: $(CH_CPP_LIB) $(shlib) - cp -a $(CH_CPP_BUILD_DIR)/clickhouse/libclickhouse-cpp-lib*$(DLSUFFIX)* $(DESTDIR)$(pkglibdir)/ -uninstall-ch-cpp: - rm -f $(DESTDIR)$(pkglibdir)/libclickhouse-cpp-lib*$(DLSUFFIX)* -install: install-ch-cpp -uninstall: uninstall-ch-cpp -endif - # Build a PGXN distribution bundle. dist: $(EXTENSION)-$(DISTVERSION).zip @@ -187,7 +121,7 @@ bake-vars: @echo "revision=$(REVISION)" @echo "pg_versions=$(PG_VERSIONS)" -# Format the .c, .h, and .hh files according to the PostgreSQL indentation +# Format the .c and .h files according to the PostgreSQL indentation # standard. Requires `pg_bsd_indent` to be in the path. indent: dev/indent.sh @$< @@ -199,7 +133,7 @@ lint: .pre-commit-config.yaml .PHONY: clang-tidy # Run clang-tidy static analysis (requires compile_commands.json) clang-tidy: compile_commands.json - clang-tidy -p . $(wildcard src/*.c src/*.cpp) + clang-tidy -p . $(wildcard src/*.c src/*/*.c) ## .git/hooks/pre-commit: Install the pre-commit hook .git/hooks/pre-commit: @@ -216,8 +150,8 @@ lsp: compile_commands.json # Requires https://github.com/rizsotto/Bear. compile_commands.json: - $(MAKE) clean -j $$(nproc) NO_VENDOR_CLEAN=$(NO_VENDOR_CLEAN) - bear -- $(MAKE) all -j $$(nproc) NO_VENDOR_CLEAN=$(NO_VENDOR_CLEAN) + $(MAKE) clean -j $$(nproc) + bear -- $(MAKE) all -j $$(nproc) # ClickHouse Docker Containers start-containers: dev/Makefile dev/docker-compose.yml diff --git a/README.md b/README.md index 58ec4b93..31beb463 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,11 @@ sudo apt install \ postgresql-server-18 \ libcurl4-openssl-dev \ uuid-dev \ + liblz4-dev \ + libzstd-dev \ libssl-dev \ make \ - cmake \ - g++ + gcc ``` #### RedHat / CentOS / Yum @@ -93,9 +94,10 @@ sudo yum install \ postgresql-server \ libcurl-devel \ libuuid-devel \ + lz4-devel \ + libzstd-devel \ openssl-libs \ automake \ - cmake \ gcc ``` @@ -121,18 +123,6 @@ make sudo make install ``` - - If your host has several PostgreSQL installations, you might need to specify the appropriate version of `pg_config`: @@ -239,8 +229,8 @@ CREATE EXTENSION pg_clickhouse SCHEMA env; ## Dependencies The `pg_clickhouse` extension requires [PostgreSQL] 13 or higher, [libcurl], -[libuuid]. Building the extension requires a C and C++ compiler, [libSSL], [GNU -make], and [CMake]. +[libuuid], [liblz4], and [libzstd]. Building the extension requires a C +compiler, [libSSL], and [GNU make]. ## Road Map @@ -294,8 +284,9 @@ adding DML features. Our road map: [PostgreSQL]: https://www.postgresql.org "PostgreSQL: The World's Most Advanced Open Source Relational Database" [libcurl]: https://curl.se/libcurl/ "libcurl — your network transfer library" [libuuid]: https://linux.die.net/man/3/libuuid "libuuid - DCE compatible Universally Unique Identifier library" + [liblz4]: https://lz4.org "LZ4 - Extremely fast compression" + [libzstd]: https://facebook.github.io/zstd/ "Zstandard - Fast real-time compression algorithm" [GNU make]: https://www.gnu.org/software/make "GNU Make" - [CMake]: https://cmake.org/ "CMake: A Powerful Software Build System" [LibSSL]: https://openssl-library.org "OpenSSL Library" [TPC-H]: https://www.tpc.org/tpch/ [re2]: https://github.com/ClickHouse/pg_re2 "pg_re2: ClickHouse-compatible regex functions using RE2" diff --git a/dev/README.md b/dev/README.md index c855e81a..dc7d729b 100644 --- a/dev/README.md +++ b/dev/README.md @@ -14,7 +14,6 @@ make start docker logs -f ch_25_11 # Wait a few minutes for the services to start (log output will stop) docker exec -it ch_25_11 bash -make clean NO_VENDOR_CLEAN=1 # Option to prevent clickhouse-cpp rebuild pg-build-test # or make && make install && make installcheck ``` diff --git a/dev/indent.sh b/dev/indent.sh index 942e6fb7..a407b22c 100755 --- a/dev/indent.sh +++ b/dev/indent.sh @@ -2,10 +2,8 @@ set -e -for fn in src/**/*.h* src/**/*.h.in src/*.c; do +for fn in src/**/*.h* src/**/*.h.in src/*.c src/**/*.c; do printf "%s\n" "$fn" pg_bsd_indent -bad -bap -bbb -bc -bl -cli1 -cp33 -cdb -nce -d0 -di12 -nfc1 -i4 -l79 -lp -lpl -nip -npro -sac -tpg -ts4 "$fn" done rm -f ./*.BAK - -clang-format --style=file:.clang-format -i src/binary.cpp diff --git a/src/binary.cpp b/src/binary.cpp deleted file mode 100644 index 61567c00..00000000 --- a/src/binary.cpp +++ /dev/null @@ -1,1353 +0,0 @@ -#include -#include -#include -#include - -#include "clickhouse/columns/date.h" -#include "clickhouse/columns/factory.h" -#include "clickhouse/columns/ip4.h" -#include "clickhouse/columns/lowcardinality.h" -#include "clickhouse/columns/nullable.h" -#include -#include -#include - -#if __cplusplus > 199711L -#define register /* Deprecated in C++11. */ -#endif /* #if __cplusplus > 199711L */ - -extern "C" -{ - -#include "postgres.h" - -#include "fmgr.h" -#include "funcapi.h" -#include "internal.h" -#include "pgtime.h" - -#include "access/htup_details.h" -#include "access/tupdesc.h" -#include "catalog/pg_type_d.h" -#include "utils/array.h" -#include "utils/builtins.h" -#include "utils/date.h" -#include "utils/elog.h" -#include "utils/inet.h" -#include "utils/lsyscache.h" -#include "utils/memdebug.h" -#include "utils/palloc.h" -#include "utils/timestamp.h" -#include "utils/uuid.h" - -#include "binary.hh" - - using namespace clickhouse; - -#if defined(__APPLE__) /* Byte ordering on macOS */ -#include -#include -#define HOST_TO_BIG_ENDIAN_64(x) OSSwapHostToBigInt64(x) -#define BIG_ENDIAN_64_TO_HOST(x) OSSwapBigToHostInt64(x) -#else -#include -#define HOST_TO_BIG_ENDIAN_64(x) htobe64(x) -#define BIG_ENDIAN_64_TO_HOST(x) be64toh(x) -#endif - -#define THROW_UNEXPECTED_COLUMN(exp_type, col) \ - throw std::runtime_error("unexpected column type for " + std::string(exp_type) + ": " + col->Type()->GetName()) - - /* palloc which will throw exceptions */ - static void * - exc_palloc(Size size) - { - /* duplicates MemoryContextAlloc to avoid increased overhead */ - void *ret; - MemoryContext context = CurrentMemoryContext; - - Assert(MemoryContextIsValid(context)); - - if (!AllocSizeIsValid(size)) - throw std::bad_alloc(); - - context->isReset = false; - -#if PG_VERSION_NUM >= 170000 - ret = context->methods->alloc(context, size, 0); -#else - ret = context->methods->alloc(context, size); -#endif - if (unlikely(ret == NULL)) - throw std::bad_alloc(); - - VALGRIND_MEMPOOL_ALLOC(context, ret, size); - - return ret; - } - - void * - exc_palloc0(Size size) - { - /* duplicates MemoryContextAllocZero to avoid increased overhead */ - void *ret; - MemoryContext context = CurrentMemoryContext; - - Assert(MemoryContextIsValid(context)); - - if (!AllocSizeIsValid(size)) - throw std::bad_alloc(); - - context->isReset = false; - -#if PG_VERSION_NUM >= 170000 - ret = context->methods->alloc(context, size, 0); -#else - ret = context->methods->alloc(context, size); -#endif - if (unlikely(ret == NULL)) - throw std::bad_alloc(); - - VALGRIND_MEMPOOL_ALLOC(context, ret, size); - - MemSetAligned(ret, 0, size); - - return ret; - } - -#define CLICKHOUSE_SECURE_PORT 9440 - - ch_binary_connection_t * - ch_binary_connect(ch_connection_details *details, char **error) - { - ClientOptions *options = NULL; - ch_binary_connection_t *conn = NULL; - - try - { - options = new ClientOptions(); - options->SetPingBeforeQuery(true); - - if (details->host) - { - options->SetHost(std::string(details->host)); - if (!details->port && ch_is_cloud_host(details->host)) - options->SetPort(CLICKHOUSE_SECURE_PORT); - } - if (details->port) - options->SetPort(details->port); - if (details->dbname) - options->SetDefaultDatabase(std::string(details->dbname)); - if (details->username) - options->SetUser(std::string(details->username)); - if (details->password) - options->SetPassword(std::string(details->password)); - if (options->port == CLICKHOUSE_SECURE_PORT) - options->SetSSLOptions(ClientOptions::SSLOptions()); - - /* options->SetRethrowException(false); */ - conn = new ch_binary_connection_t(); - - Client *client = new Client(*options); - conn->client = client; - conn->options = options; - } - catch (const std::exception &e) - { - if (error) - *error = strdup(e.what()); - - if (conn != NULL) - delete conn; - - if (options != NULL) - delete options; - - conn = NULL; - } - return conn; - } - - static void - set_resp_error(ch_binary_response_t *resp, const char *str) - { - if (resp->error) - return; - - resp->error = strdup(str); - } - - /* - * Converts query->settings to QuerySettings. - */ - static QuerySettings - ch_binary_settings(const Client *client, const ch_query *query) - { - kv_iter iter; - auto res = QuerySettings{}; - - for (iter = new_kv_iter(query->settings); !kv_iter_done(&iter); kv_iter_next(&iter)) - { - res.insert_or_assign(iter.name, QuerySettingsField{ iter.value, 1 }); - } - auto info = client->GetServerInfo(); - if (info.version_major >= 25 || (info.version_major == 24 && info.version_minor >= 10)) - res.insert_or_assign("output_format_native_write_json_as_string", QuerySettingsField{ "1", 1 }); - - return res; - } - - /* - * Converts query->param_values to QueryParams. - */ - static QueryParams - ch_binary_params(const ch_query *query) - { - int i; - auto res = QueryParams{}; - - for (i = 0; i < query->num_params; i++) - res.insert_or_assign(psprintf("p%d", i + 1), QueryParamValue(query->param_values[i])); - - return res; - } - - static void - set_state_error(ch_binary_read_state_t *state, const char *str) - { - assert(state->error == NULL); - state->error = strdup(str); - } - - ch_binary_response_t * - ch_binary_simple_query(ch_binary_connection_t *conn, const ch_query *query, bool (*check_cancel)(void)) - { - Client *client = (Client *)conn->client; - ch_binary_response_t *resp; - std::vector> *values; - - try - { - resp = new ch_binary_response_t(); - values = new std::vector>(); - client->Select(clickhouse::Query(query->sql) - .SetQuerySettings(ch_binary_settings(client, query)) - .SetParams(ch_binary_params(query)) - .OnProgress( - [&check_cancel](const Progress &) - { - if (check_cancel && check_cancel()) - throw std::runtime_error("query was canceled"); - }) - .OnDataCancelable( - [&resp, &values, &check_cancel](const Block &block) - { - if (check_cancel && check_cancel()) - { - set_resp_error(resp, "query was canceled"); - return false; - } - - /* some empty block */ - if (block.GetColumnCount() == 0) - return true; - - auto vec = std::vector(); - - if (resp->columns_count && block.GetColumnCount() != resp->columns_count) - { - set_resp_error(resp, "columns mismatch in blocks"); - return false; - } - - resp->columns_count = block.GetColumnCount(); - resp->blocks_count++; - - for (size_t i = 0; i < resp->columns_count; ++i) - vec.push_back(block[i]); - - values->push_back(std::move(vec)); - return true; - })); - - resp->values = (void *)values; - } - catch (const std::exception &e) - { - client->ResetConnection(); - - values->clear(); - set_resp_error(resp, e.what()); - delete values; - values = NULL; - } - - resp->success = (resp->error == NULL); - return resp; - } - - /* - * Given TypeRef for the value returned by a clickhouse-cpp query and the - * Postgres data type from the corresponding column in the foreign table, - * return the Oid of the type that should be created for the TypeRef. - */ - static Oid - get_corr_postgres_type(const TypeRef &type, Oid pg_type) - { - switch (type->GetCode()) - { - case Type::Code::Int8: - case Type::Code::Int16: - case Type::Code::UInt8: - return INT2OID; - case Type::Code::Int32: - case Type::Code::UInt16: - return INT4OID; - case Type::Code::Int64: - case Type::Code::UInt64: - case Type::Code::UInt32: - return INT8OID; - case Type::Code::Float32: - return FLOAT4OID; - case Type::Code::Float64: - return FLOAT8OID; - case Type::Code::Decimal128: - case Type::Code::Decimal64: - case Type::Code::Decimal32: - case Type::Code::Decimal: - return NUMERICOID; - case Type::Code::FixedString: - case Type::Code::Enum8: - case Type::Code::Enum16: - case Type::Code::String: - return TEXTOID; - case Type::Code::LowCardinality: - return get_corr_postgres_type(type->As()->GetNestedType(), pg_type); - case Type::Code::Date: - case Type::Code::Date32: - return DATEOID; - case Type::Code::DateTime: - return TIMESTAMPTZOID; - case Type::Code::DateTime64: - return TIMESTAMPTZOID; - case Type::Code::UUID: - return UUIDOID; - case Type::Code::Array: - { - /* postgres uses one array type for any number of dimensions, so - * walk past nested Array layers to the leaf element type. */ - auto leaf = type->As()->GetItemType(); - while (leaf->GetCode() == Type::Code::Array) - leaf = leaf->As()->GetItemType(); - - Oid array_type = get_array_type(get_corr_postgres_type(leaf, get_element_type(pg_type))); - if (array_type == InvalidOid) - throw std::runtime_error("pg_clickhouse: could not find array " - " type for column type " - + type->GetName()); - - return array_type; - } - case Type::Code::Tuple: - return RECORDOID; - case Type::Code::Nullable: - return get_corr_postgres_type(type->As()->GetNestedType(), pg_type); - case Type::Code::IPv4: - case Type::Code::IPv6: - return INETOID; - case Type::Code::JSON: - /* We support json and default to jsonb. */ - return pg_type == JSONOID ? JSONOID : JSONBOID; - default: - throw std::runtime_error("pg_clickhouse: unsupported column type " + type->GetName()); - } - } - - void - ch_binary_insert_state_free(void *c) - { - auto *state = (ch_binary_insert_state *)c; - if (state->insert_block) - { - /* Finish the insert to set the proper ClickHouse state */ - delete (Block *)state->insert_block; - Client *client = (Client *)state->conn->client; - try - { - client->EndInsert(); - } - catch (const std::exception &e) - { - client->ResetConnection(); - elog(ERROR, "pg_clickhouse: could not finish INSERT - %s", e.what()); - } - } - } - - void - ch_binary_prepare_insert(void *conn, const ch_query *query, ch_binary_insert_state *state) - { - /* Start the INSERT. */ - Block *block; - Client *client = (Client *)((ch_binary_connection_t *)conn)->client; - try - { - block = new Block(client->BeginInsert(std::string(query->sql) + " VALUES")); - /* XXX https://github.com/ClickHouse/clickhouse-cpp/pull/453/ - block = new Block(client->BeginInsert( - clickhouse::Query(std::string(query->sql)+ " VALUES").SetQuerySettings( - ch_binary_settings(client, query) - ).SetParams( - ch_binary_params(query) - ) - )); - */ - } - catch (const std::exception &e) - { - client->ResetConnection(); - elog(ERROR, "pg_clickhouse: could not prepare insert - %s", e.what()); - } - - /* Setup the column config (or return if no columns). */ - state->len = block->GetColumnCount(); - if (state->len == 0) - { - delete block; - return; - } - state->outdesc = CreateTemplateTupleDesc(state->len); - - /* Iterate over the list of columns returned by ClickHouse. */ - AttrNumber i = 0; - for (Block::Iterator bi(*block); bi.IsValid(); bi.Next()) - { - /* Start with the data type from the foreign table. */ - Oid pg_type = query->tupdesc - ? TupleDescAttr(query->tupdesc, lfirst_int(list_nth_cell(query->attr_nums, i)) - 1)->atttypid - : InvalidOid; - const char *colname = bi.Name().c_str(); - try - { - /* - * Determine the Postgres type to which to convert values so - * that column_append() knows how to append it to a ClickHouse - * column. - */ - pg_type = get_corr_postgres_type(bi.Type(), pg_type); - colname = bi.Name().c_str(); - } - catch (const std::exception &e) - { - elog(ERROR, "pg_clickhouse: could not prepare insert - %s", e.what()); - } - PG_TRY(); - { - TupleDescInitEntry(state->outdesc, ++i, colname, pg_type, -1, 0); - } - PG_CATCH(); - { - /* Clean up and re-throw. */ - client->ResetConnection(); - delete block; - PG_RE_THROW(); - } - PG_END_TRY(); - } - - state->insert_block = (ch_insert_block_h *)block; - } - - /* - * Append val to col. If isnull is true and val is nullable, append a - * null. Otherwise, determine how to convert val, of type valtype, to a - * value appropriate to col, and append that value. Raises an exception if - * valtype is not compatible with col's type. - */ - static void column_append(clickhouse::ColumnRef col, Datum val, Oid valtype, bool isnull); - - /* - * Build a single ColumnArray row's worth of data from a (possibly nested) - * ch_binary_array_t. items_type is the per-row element type of the parent - * ColumnArray: scalar T for Array(T), Array(T) for Array(Array(T)), etc. - * - * For nested types, recurses to build child ColumnArrays and stitches them - * via AppendAsColumn so the outer column's offsets describe the row shape. - */ - static clickhouse::ColumnRef - build_array_row_column(ch_binary_array_t *arr, clickhouse::TypeRef items_type) - { - using namespace clickhouse; - - auto col = CreateColumnByType(items_type->GetName()); - - /* Empty postgres array fits any nesting depth: nothing to walk, so - * skip the dim check and return an empty column at this level. */ - if (arr->len == 0) - return col; - - if (items_type->GetCode() == Type::Code::Array) - { - if (arr->ndim < 2) - throw std::runtime_error("pg_clickhouse: insert array has fewer dimensions than column type"); - - auto inner_arr = col->AsStrict(); - auto inner_items_t = items_type->As()->GetItemType(); - - for (size_t i = 0; i < arr->len; i++) - { - auto child = (ch_binary_array_t *)DatumGetPointer(arr->datums[i]); - auto sub = build_array_row_column(child, inner_items_t); - - inner_arr->AppendAsColumn(sub); - } - return col; - } - - if (arr->ndim != 1) - throw std::runtime_error("pg_clickhouse: insert array has more dimensions than column type"); - - for (size_t i = 0; i < arr->len; i++) - column_append(col, arr->datums[i], arr->item_type, arr->nulls[i]); - return col; - } - - static void - column_append(clickhouse::ColumnRef col, Datum val, Oid valtype, bool isnull) - { - bool nullable = false; - - if (col->Type()->GetCode() == Type::Code::Nullable) - nullable = true; - - /* - * Prevent insertion if the column isn't Nullable. This differs from the - * http engine, which isn't aware of Nullable but just sends values off, - * in which case ClickHouse inserts default values (e.g., 0 for numbers, - * "" for strings, etc.). - * - * XXX Do we want to consider rejecting NULL inserts when the Postgres - * `NOT NULL` constraint exits? And when it doesn't exist and the column - * is not nullable, do we want to send the default value instead? Dropping - * this block should do that. - */ - if (isnull && !nullable) - throw std::runtime_error("cannot append NULL to NOT NULL " + col->Type()->GetName() + " column"); - - if (nullable) - { - auto nullable = col->AsStrict(); - nullable->Append(isnull); - col = nullable->Nested(); - } - - switch (valtype) - { - case INT2OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::UInt8: - col->AsStrict()->Append((uint8_t)val); - break; - case Type::Code::Int8: - col->AsStrict()->Append((int8_t)val); - break; - case Type::Code::Int16: - col->AsStrict()->Append((int16_t)val); - break; - default: - THROW_UNEXPECTED_COLUMN("INT2", col); - } - break; - } - case INT4OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::Int32: - col->AsStrict()->Append((int32_t)val); - break; - case Type::Code::UInt16: - col->AsStrict()->Append((uint16_t)val); - break; - default: - THROW_UNEXPECTED_COLUMN("INT4", col); - } - break; - } - case INT8OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::Int64: - col->AsStrict()->Append((int64_t)val); - break; - case Type::Code::UInt32: - col->AsStrict()->Append((uint32_t)val); - break; - case Type::Code::UInt64: - col->AsStrict()->Append((uint64_t)val); - break; - default: - THROW_UNEXPECTED_COLUMN("INT8", col); - } - break; - } - case FLOAT4OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::Float32: - col->AsStrict()->Append(DatumGetFloat4(val)); - break; - default: - THROW_UNEXPECTED_COLUMN("FLOAT4", col); - } - break; - } - case FLOAT8OID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::Float64: - col->AsStrict()->Append(DatumGetFloat8(val)); - break; - default: - THROW_UNEXPECTED_COLUMN("FLOAT8", col); - } - break; - } - case NUMERICOID: - { - /* Convert numeric to string and let ColumnDecimal parse it. */ - switch (col->Type()->GetCode()) - { - case Type::Code::Decimal128: - case Type::Code::Decimal64: - case Type::Code::Decimal32: - case Type::Code::Decimal: - if (isnull) - col->AsStrict()->Append(Int128{}); - else - { - char *s = DatumGetCString(DirectFunctionCall1(numeric_out, val)); - col->AsStrict()->Append(std::string(s)); - pfree(s); - } - break; - default: - THROW_UNEXPECTED_COLUMN("NUMERIC", col); - } - break; - } - case TEXTOID: - { - /* - * ClickHouse allows nuls in strings, and val can be bytea, so - * don't rely on nul termination but copy the full length of the - * data. - */ - std::string s; - if (!isnull) - { - text *string = PG_DETOAST_DATUM(val); - s.assign(VARDATA(string), VARSIZE_ANY_EXHDR(string)); - } - - switch (col->Type()->GetCode()) - { - case Type::Code::FixedString: - col->AsStrict()->Append(s); - break; - case Type::Code::String: - col->AsStrict()->Append(s); - break; - case Type::Code::Enum8: - if (isnull) - col->AsStrict()->Append(0, false); - else - col->AsStrict()->Append(s); - break; - case Type::Code::Enum16: - if (isnull) - col->AsStrict()->Append(0, false); - else - col->AsStrict()->Append(s); - break; - case Type::Code::LowCardinality: - /* XXX Handle LowCardinality(Nullable(x)) */ - if (col->AsStrict()->GetNestedType()->GetCode() == Type::Nullable) - throw std::runtime_error("nested Nullable is not supported"); - col->AsStrict>()->Append(s); - break; - default: - THROW_UNEXPECTED_COLUMN("TEXT", col); - } - - break; - } - case DATEOID: - { - Timestamp t = date2timestamp_no_overflow(DatumGetDateADT(val)); - pg_time_t d = timestamptz_to_time_t(t); - - switch (col->Type()->GetCode()) - { - case Type::Code::Date: - col->AsStrict()->Append(d); - break; - case Type::Code::Date32: - col->AsStrict()->Append(d); - break; - default: - THROW_UNEXPECTED_COLUMN("DATE", col); - } - break; - } - case TIMESTAMPOID: - case TIMESTAMPTZOID: - { - switch (col->Type()->GetCode()) - { - case Type::Code::DateTime: - { - pg_time_t d = timestamptz_to_time_t(DatumGetTimestamp(val)); - col->AsStrict()->Append(d); - break; - } - case Type::Code::DateTime64: - { - auto dt64_col = col->AsStrict(); - Timestamp t = DatumGetTimestamp(val); - Int64 dt64 - = ((1.0 * t) / USECS_PER_SEC + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)) - * pow(10.0, dt64_col->GetPrecision()); - - dt64_col->Append(dt64); - break; - } - default: - THROW_UNEXPECTED_COLUMN("TIMESTAMP", col); - } - break; - } - case ANYARRAYOID: - { - switch (col->Type()->GetCode()) - { - case Type::Array: - { - auto arrcol = col->AsStrict(); - auto items_type = arrcol->GetType().As()->GetItemType(); - auto arr = (ch_binary_array_t *)DatumGetPointer(val); - auto one_row = build_array_row_column(arr, items_type); - - arrcol->AppendAsColumn(one_row); - break; - } - default: - THROW_UNEXPECTED_COLUMN("array", col); - } - break; - } - case UUIDOID: - { - auto uuid_val = DatumGetUUIDP(val)->data; - auto ch_uuid = UUID{}; - if (!isnull) - { - memcpy(&ch_uuid.first, uuid_val, 8); - memcpy(&ch_uuid.second, uuid_val + 8, 8); - ch_uuid.first = BIG_ENDIAN_64_TO_HOST(ch_uuid.first); - ch_uuid.second = BIG_ENDIAN_64_TO_HOST(ch_uuid.second); - } - col->AsStrict()->Append(ch_uuid); - } - break; - case INETOID: - { - if (isnull) - { - switch (col->Type()->GetCode()) - { - case Type::Code::IPv4: - col->AsStrict()->Append(0); - break; - case Type::Code::IPv6: - col->AsStrict()->Append("::"); - break; - default: - THROW_UNEXPECTED_COLUMN("INET", col); - } - } - else - { - char *s = DatumGetCString(DirectFunctionCall1(inet_out, val)); - switch (col->Type()->GetCode()) - { - case Type::Code::IPv4: - col->AsStrict()->Append(s); - break; - case Type::Code::IPv6: - col->AsStrict()->Append(s); - break; - default: - THROW_UNEXPECTED_COLUMN("INET", col); - } - } - } - break; - case JSONBOID: - { - char *s = DatumGetCString(DirectFunctionCall1(jsonb_out, val)); - col->AsStrict()->Append(s); - } - break; - case JSONOID: - { - char *s = DatumGetCString(DirectFunctionCall1(json_out, val)); - col->AsStrict()->Append(s); - } - break; - default: - { - THROW_UNEXPECTED_COLUMN(format_type_extended(valtype, -1, FORMAT_TYPE_ALLOW_INVALID), col); - } - } - } - - /* - * Append state->values[colidx] to state->insert_block[colidx] or raise an - * exception. - */ - void - ch_binary_column_append_data(ch_binary_insert_state *state, size_t colidx) - { - try - { - auto block = (Block *)state->insert_block; - auto col = (*block)[colidx]; - - Datum val = state->values[colidx]; - - Oid valtype = TupleDescAttr(state->outdesc, colidx)->atttypid; - bool isnull = state->nulls[colidx]; - - column_append(col, val, valtype, isnull); - } - catch (const std::exception &e) - { - elog(ERROR, "pg_clickhouse: could not append data to column - %s", e.what()); - } - } - - void - ch_binary_insert_columns(ch_binary_insert_state *state) - { - Client *client = (Client *)state->conn->client; - auto block = (Block *)state->insert_block; - try - { - block->RefreshRowCount(); - client->SendInsertBlock(*block); - block->Clear(); - } - catch (const std::exception &e) - { - client->ResetConnection(); - delete block; - elog(ERROR, "pg_clickhouse: could not insert columns - %s", e.what()); - } - } - - void - ch_binary_close(ch_binary_connection_t *conn) - { - delete (Client *)conn->client; - delete (ClientOptions *)conn->options; - } - - void - ch_binary_response_free(ch_binary_response_t *resp) - { - if (resp->values) - { - auto values = (std::vector> *)resp->values; - values->clear(); - delete values; - } - - if (resp->error) - free(resp->error); - - delete resp; - } - - void - ch_binary_read_state_init(ch_binary_read_state_t *state, ch_binary_response_t *resp, const ch_query *query) - { - state->resp = resp; - state->block = 0; - state->row = 0; - state->done = false; - state->error = NULL; - state->coltypes = NULL; - state->values = NULL; - state->nulls = NULL; - - /* it response was errored just set error in state too */ - if (resp->error) - { - state->done = true; - set_state_error(state, resp->error); - return; - } - - try - { - assert(resp->values); - auto &values = *((std::vector> *)resp->values); - - if (resp->columns_count && values.size() > 0) - { - state->coltypes = new Oid[resp->columns_count]; - state->values = new Datum[resp->columns_count]; - state->nulls = new bool[resp->columns_count]; - } - } - catch (const std::exception &e) - { - set_state_error(state, e.what()); - return; - } - - /* Initialize coltypes to SELECT types, when provided. */ - if (query->tupdesc) - for (size_t i = 0; i < (size_t)list_length(query->attr_nums); i++) - state->coltypes[i] - = TupleDescAttr(query->tupdesc, lfirst_int(list_nth_cell(query->attr_nums, i)) - 1)->atttypid; - else - for (size_t i = 0; i < resp->columns_count; i++) - state->coltypes[i] = InvalidOid; - } - - /* - * This function prepares values for `convert_datum` which is called in - * upper code. - * - * This function calls postgres functions, which can call `palloc` so we can end up - * with elog(ERROR) and longjmp to upper postgres code with leaking c++ memory. - * - * There is not an adequate (without huge overheads) solution, we just consider - * this state unfixable. - * - * Expects `valtype` to already be set to the type of the foreign table - * column for the value, but it may replace it with another type that will - * later be cast to valtype. - * - * Always sets `is_null`. - */ - static Datum - make_datum(clickhouse::ColumnRef col, size_t row, Oid *valtype, bool *is_null) - { - Datum ret = (Datum)0; - - nested_col: - auto type_code = col->Type()->GetCode(); - - *is_null = false; - - switch (type_code) - { - case Type::Code::UInt8: - { - int16 val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT2OID; - } - break; - case Type::Code::UInt16: - { - int16 val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT4OID; - } - break; - case Type::Code::UInt32: - { - int64 val = col->AsStrict()->At(row); - ret = Int64GetDatum(val); - *valtype = INT8OID; - } - break; - case Type::Code::UInt64: - { - uint64 val = col->AsStrict()->At(row); - /* XXX Consider using, e.g., https://pgxn.org/dist/uint128. */ - if (val > LONG_MAX) - throw std::overflow_error("value " + std::to_string(val) + " is out of range of bigint"); - - ret = Int64GetDatum((int64)val); - *valtype = INT8OID; - } - break; - case Type::Code::Int8: - { - int16 val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT2OID; - } - break; - case Type::Code::Int16: - { - int16 val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT2OID; - } - break; - case Type::Code::Int32: - { - int val = col->AsStrict()->At(row); - ret = (Datum)val; - *valtype = INT4OID; - } - break; - case Type::Code::Int64: - { - int64 val = col->AsStrict()->At(row); - ret = Int64GetDatum(val); - *valtype = INT8OID; - } - break; - case Type::Code::Float32: - { - float val = col->AsStrict()->At(row); - ret = Float4GetDatum(val); - *valtype = FLOAT4OID; - } - break; - case Type::Code::Float64: - { - double val = col->AsStrict()->At(row); - ret = Float8GetDatum(val); - *valtype = FLOAT8OID; - } - break; - case Type::Code::Decimal128: - case Type::Code::Decimal64: - case Type::Code::Decimal32: - case Type::Code::Decimal: - { - auto decCol = col->AsStrict(); - auto val = decCol->At(row); - - /* Convert the Int128 to a string. */ - std::stringstream ss; - ss << val; - std::string str = ss.str(); - - /* Start a destination string. */ - std::stringstream res; - auto scale = decCol->GetScale(); - - /* Output a dash for negative values */ - if (val < 0) - { - res << '-'; - str.erase(0, 1); - } - - if (scale == 0) - { - /* No decimal point, just output the entire value. */ - res << str; - } - else if (str.length() <= scale) - { - /* Append the entire value prepended with zeros after the decimal. */ - res << "0." << std::string(scale - str.length(), '0') << str; - } - else - { - /* There are digits before the decimal. */ - auto decAt = str.length() - scale; - res << str.substr(0, decAt); - - /* Append any digits after the decimal. */ - if (decAt < str.length()) - { - res << '.' << str.substr(decAt); - } - } - - ret = DirectFunctionCall3(numeric_in, CStringGetDatum(res.str().c_str()), ObjectIdGetDatum(0), - Int32GetDatum(-1)); - *valtype = NUMERICOID; - } - break; - case Type::Code::FixedString: - { - auto s = std::string(col->AsStrict()->At(row)); - /* ClickHouse allows nuls in strings, so copy by full length. */ - ret = PointerGetDatum(cstring_to_text_with_len(s.data(), s.size())); - *valtype = TEXTOID; - } - break; - case Type::Code::String: - { - auto s = std::string(col->AsStrict()->At(row)); - /* ClickHouse allows nuls in strings, so copy by full length. */ - ret = PointerGetDatum(cstring_to_text_with_len(s.data(), s.size())); - *valtype = TEXTOID; - } - break; - case Type::Code::Enum8: - { - auto s = std::string(col->AsStrict()->NameAt(row)); - ret = CStringGetTextDatum(s.c_str()); - *valtype = TEXTOID; - } - break; - case Type::Code::Enum16: - { - auto s = std::string(col->AsStrict()->NameAt(row)); - ret = CStringGetTextDatum(s.c_str()); - *valtype = TEXTOID; - } - break; - case Type::Code::Date: - { - auto val = static_cast(col->AsStrict()->At(row)); - *valtype = DATEOID; - ret = DirectFunctionCall1(timestamp_date, time_t_to_timestamptz(val)); - } - break; - case Type::Code::Date32: - { - auto val = static_cast(col->AsStrict()->At(row)); - *valtype = DATEOID; - ret = DirectFunctionCall1(timestamp_date, time_t_to_timestamptz(val)); - } - break; - case Type::Code::DateTime: - { - auto val = static_cast(col->AsStrict()->At(row)); - *valtype = TIMESTAMPTZOID; - ret = TimestampTzGetDatum(time_t_to_timestamptz(val)); - } - break; - case Type::Code::DateTime64: - { - auto dt_col = col->AsStrict(); - auto val = dt_col->At(row); - int64 power = pow(10, dt_col->GetPrecision()); - *valtype = TIMESTAMPTZOID; - ret = TimestampTzGetDatum(time_t_to_timestamptz(val / power)) + (val % power) * (USECS_PER_SEC / power); - } - break; - case Type::Code::UUID: - { - /* we form char[16] from two uint64 numbers, and they should - * be big endian */ - auto val = col->AsStrict()->At(row); - pg_uuid_t *uuid_val = (pg_uuid_t *)exc_palloc(sizeof(pg_uuid_t)); - - val.first = HOST_TO_BIG_ENDIAN_64(val.first); - val.second = HOST_TO_BIG_ENDIAN_64(val.second); - memcpy(uuid_val->data, &val.first, 8); - memcpy(uuid_val->data + 8, &val.second, 8); - - ret = UUIDPGetDatum(uuid_val); - *valtype = UUIDOID; - } - break; - case Type::Code::Nullable: - { - auto nullable = col->AsStrict(); - if (nullable->IsNull(row)) - { - *is_null = true; - } - else - { - col = nullable->Nested(); - goto nested_col; - } - } - break; - case Type::Code::Array: - { - auto arr = col->AsStrict()->GetAsColumn(row); - size_t len = arr->Size(); - auto slot = (ch_binary_array_t *)exc_palloc(sizeof(ch_binary_array_t)); - - /* find leaf scalar type & nesting depth, since postgres has one - * array type per element type regardless of ndim */ - int ndim = 1; - auto leaf_type = arr->Type(); - while (leaf_type->GetCode() == Type::Code::Array) - { - leaf_type = leaf_type->As()->GetItemType(); - ndim++; - } - - Oid item_type = get_corr_postgres_type(leaf_type, get_element_type(*valtype)); - Oid array_type = get_array_type(item_type); - - if (array_type == InvalidOid) - throw std::runtime_error(std::string("pg_clickhouse: could not") + " find array type for " - + std::to_string(item_type)); - - slot->len = len; - slot->ndim = ndim; - slot->array_type = array_type; - slot->item_type = item_type; - - if (len > 0) - { - slot->datums = (Datum *)exc_palloc0(sizeof(Datum) * len); - slot->nulls = (bool *)exc_palloc0(sizeof(bool) * len); - - /* For ndim==1 inner make_datum returns leaf scalars; for - * ndim>1 it recurses into the Array branch and produces - * nested ch_binary_array_t* values. Use a scratch valtype - * to avoid clobbering slot->item_type. */ - Oid scratch; - for (size_t i = 0; i < len; ++i) - slot->datums[i] = make_datum(arr, i, &scratch, &slot->nulls[i]); - } - - /* this one will need additional work, since we just return raw slot */ - ret = PointerGetDatum(slot); - *valtype = ANYARRAYOID; - } - break; - case Type::Code::Tuple: - { - auto tuple = col->AsStrict(); - auto len = tuple->TupleSize(); - - if (len == 0) - throw std::runtime_error("pg_clickhouse: returned tuple is empty"); - - auto slot = (ch_binary_tuple_t *)exc_palloc(sizeof(ch_binary_tuple_t)); - - slot->datums = (Datum *)exc_palloc(sizeof(Datum) * len); - slot->nulls = (bool *)exc_palloc0(sizeof(bool) * len); - slot->types = (Oid *)exc_palloc0(sizeof(Oid) * len); - slot->len = len; - - for (size_t i = 0; i < len; ++i) - slot->datums[i] = make_datum((*tuple)[i], row, &slot->types[i], &slot->nulls[i]); - - /* this one will need additional work, since we just return raw slot */ - ret = PointerGetDatum(slot); - *valtype = RECORDOID; - } - break; - case Type::Code::LowCardinality: - { - auto item = col->AsStrict()->GetItem(row); - auto data = item.AsBinaryData(); - ret = PointerGetDatum(cstring_to_text_with_len(data.data(), data.size())); - *valtype = TEXTOID; - } - break; - case Type::Code::IPv4: - { - auto item = col->AsStrict()->AsString(row); - ret = DirectFunctionCall1(inet_in, CStringGetDatum(item.c_str())); - *valtype = INETOID; - } - break; - case Type::Code::IPv6: - { - auto item = col->AsStrict()->AsString(row); - ret = DirectFunctionCall1(inet_in, CStringGetDatum(item.c_str())); - *valtype = INETOID; - } - break; - case Type::Code::JSON: - { - if (!*valtype) - *valtype = JSONBOID; - /* Sadly must copy to a null-terminated string as Postgres expects. */ - auto item = std::string(col->AsStrict()->At(row)); - ret = DirectFunctionCall1(*valtype == JSONBOID ? jsonb_in : json_in, CStringGetDatum(item.c_str())); - } - break; - default: - throw std::runtime_error("unsupported type " + std::string(Type::TypeName(type_code)) - + " in binary protocol"); - } - - return ret; - } - - /* - * Read a row from state->block and fill state->values with the - * corresponding values. - */ - bool - ch_binary_read_row(ch_binary_read_state_t *state, TupleDesc tupdesc, List *attrs) - { - /* coltypes is NULL means there are no blocks */ - bool res = false; - - if (state->done || state->coltypes == NULL || state->error) - return false; - - assert(state->resp->values); - auto &values = *((std::vector> *)state->resp->values); - try - { - again: - assert(state->block < state->resp->blocks_count); - auto &block = values[state->block]; - size_t row_count = block[0]->Size(); - - if (row_count == 0) - goto next_row; - - for (size_t i = 0; i < state->resp->columns_count; i++) - { - /* fill value and null arrays */ - state->values[i] = make_datum(block[i], state->row, &state->coltypes[i], &state->nulls[i]); - } - res = true; - - next_row: - state->row++; - if (state->row >= row_count) - { - state->row = 0; - state->block++; - if (state->block >= state->resp->blocks_count) - state->done = true; - else if (row_count == 0) - goto again; - } - } - catch (const std::exception &e) - { - set_state_error(state, e.what()); - } - - return res; - } - - void - ch_binary_read_state_free(ch_binary_read_state_t *state) - { - if (state->coltypes) - { - delete[] state->coltypes; - delete[] state->values; - delete[] state->nulls; - } - - if (state->error) - free(state->error); - } -} diff --git a/src/binary/binary.c b/src/binary/binary.c new file mode 100644 index 00000000..a540e6c4 --- /dev/null +++ b/src/binary/binary.c @@ -0,0 +1,94 @@ +/* + * binary.c + * + * Core glue for the binary driver. Owns the chc_alloc thunks (palloc on + * CurrentMemoryContext, MCXT_ALLOC_HUGE so block buffers can escape the 1GB + * cap), error mapping into ereport, and a couple of small type helpers shared + * across the subdir. + * + * This also holds the CHC_IMPLEMENTATION define for the clickhouse-c + * header-only library; the other .c files in src/binary include the + * same headers without that define and link against the bodies emitted + * here. + * + * API lives in src/include/binary.h. Driver internals shared between + * connection.c / select.c / insert.c live in binary_internal.h + * alongside this file. + */ + +#include "postgres.h" + +#include "utils/palloc.h" + +#define CHC_IMPLEMENTATION +#include "clickhouse.h" +#include "clickhouse-compression.h" +#include "clickhouse-posix-io.h" +#include "clickhouse-openssl.h" +#include "clickhouse-client.h" + +#include "binary_internal.h" + +static void * +pg_chc_palloc(void *ud pg_attribute_unused(), size_t n) +{ + return MemoryContextAllocExtended(CurrentMemoryContext, n, MCXT_ALLOC_HUGE); +} + +static void * +pg_chc_repalloc(void *ud pg_attribute_unused(), void *p, + size_t old_bytes pg_attribute_unused(), size_t new_bytes) +{ + if (!p) + return MemoryContextAllocExtended(CurrentMemoryContext, new_bytes, MCXT_ALLOC_HUGE); + return repalloc_huge(p, new_bytes); +} + +static void +pg_chc_pfree(void *ud pg_attribute_unused(), void *p, size_t bytes pg_attribute_unused()) +{ + if (p) + pfree(p); +} + +const chc_alloc pg_chc_alloc = { + .ud = NULL, + .alloc = pg_chc_palloc, + .realloc = pg_chc_repalloc, + .free = pg_chc_pfree, +}; + +/* power-of-10 lookup; CH bounds DateTime64 / Decimal scale to [0, 9] */ +const int64_t pow10i[10] = { + 1, 10, 100, 1000, 10000, 100000, 1000000, + 10000000, 100000000, 1000000000 +}; + +/* + * output_format_native_write_json_as_string exists on the server from + * 24.10 onwards. Sending it as `important` against an older server would + * fail the query, so gate. + */ +bool +server_supports_json_as_string(const chc_client * c) +{ + const chc_server_info *info = chc_client_server_info(c); + + if (!info) + return false; + if (info->version_major > 24) + return true; + if (info->version_major == 24 && info->version_minor >= 10) + return true; + return false; +} + +void +raise_chc(const chc_err * err, int sqlstate, const char *prefix) +{ + const char *m = (err && err->msg[0]) ? err->msg : "unknown error"; + + ereport(ERROR, + (errcode(sqlstate), + errmsg("pg_clickhouse: %s%s", prefix ? prefix : "", m))); +} diff --git a/src/binary/binary_internal.h b/src/binary/binary_internal.h new file mode 100644 index 00000000..42e7cfba --- /dev/null +++ b/src/binary/binary_internal.h @@ -0,0 +1,210 @@ +/* + * binary_internal.h + * + * Cross-file shared state for the binary driver subdir. Not exposed via + * src/include, anything outside src/binary should use binary.h. + */ + +#ifndef PG_CLICKHOUSE_BINARY_INTERNAL_H +#define PG_CLICKHOUSE_BINARY_INTERNAL_H + +#include "postgres.h" + +#include + +#include "clickhouse.h" +#include "clickhouse-client.h" +#include "clickhouse-posix-io.h" +#include "clickhouse-openssl.h" + +#include "binary.h" +#include "internal.h" + +#if PG_VERSION_NUM < 180000 +#define pg_noreturn pg_attribute_noreturn() +#endif + +typedef struct +{ + Datum *datums; + bool *nulls; + size_t len; + Oid *types; + const char *ch_type_name; +} ch_binary_tuple_t; + +/* + * Holds an array decoded from ClickHouse or built for INSERT. For nested + * arrays (Array(Array(...))) ndim > 1 and datums[i] points to a child + * ch_binary_array_t with ndim-1. item_type is leaf scalar PG type, + * array_type is postgres array type (same across nesting depths since + * postgres uses one array type per element type regardless of ndim). + */ +typedef struct +{ + Datum *datums; + bool *nulls; + size_t len; + int ndim; /* nesting depth, >=1 */ + Oid item_type; /* leaf scalar PG type */ + Oid array_type; /* PG array type (same at every level) */ +} ch_binary_array_t; + +/* Column metadata returned from ch_binary_begin_insert. */ +typedef struct ch_binary_column_info +{ + const char *name; + const chc_type *type; /* type unwrapped of Nullable + LowCardinality */ + bool is_nullable; +} ch_binary_column_info; + +/* + * Pump next non-empty Data block off wire. Returned pointer is borrowed, + * valid until next fetch_next_block or ch_binary_response_free. NULL when + * stream ends (eos, error, canceled), ch_binary_response_error reports + * cause if any. + */ +extern const chc_block *ch_binary_response_fetch_next_block(ch_binary_response_t * resp); + +extern ch_binary_insert_handle * ch_binary_begin_insert(ch_binary_connection_t * conn, + const ch_query * query, + ch_binary_column_info * *out_cols, + size_t * out_n); + +/* + * Tear down handle. Never raises and never talks to server, safe to call + * from a MemoryContext reset callback during transaction abort. Flags + * connection broken if finalize did not run. + */ +extern void ch_binary_release_insert(ch_binary_insert_handle * h); + +/* + * Per-connection state smuggled through ch_binary_connection_t.client. + * + * Lives in own context `cxt` (child of CacheMemoryContext) so connection + * survives transaction boundaries. Result block buffers don't live here: + * pg_chc_alloc routes through CurrentMemoryContext, so blocks land in + * whichever per-query context the caller has switched to. + */ +struct ch_binary_state +{ + /* + * Connection-lifetime context; holds this struct, the chc_client, and + * chc_client's initial read buffer. Deleted by ch_binary_close. + */ + MemoryContext cxt; + + /* + * Registered on cxt; closes fd / SSL on reset so half-built connections + * release OS resources when PG_CATCH deletes cxt. + */ + MemoryContextCallback reset_cb; + + chc_client *client; + /* Transport vtable; backed by posix_state or openssl_state below. */ + chc_io io; + + int fd; /* -1 once closed by reset_cb */ + SSL_CTX *ssl_ctx; + SSL *ssl; + bool tls; + chc_posix_io posix_state; + chc_openssl_io openssl_state; + + /* Per-query cancel callback (no userdata). */ + bool (*check_cancel_fn) (void); + + /* + * Set when an unrecoverable protocol/IO error happened (server raised an + * exception mid-INSERT and closed the socket, write hit EPIPE, etc). + * Cache layer checks via ch_binary_is_broken & drops the entry. + */ + bool broken; +}; + +static inline struct ch_binary_state * +conn_state(ch_binary_connection_t * conn) +{ + return (struct ch_binary_state *) conn->client; +} + +/* chc allocator wired through palloc; defined in binary.c. */ +extern const chc_alloc pg_chc_alloc; + +/* power-of-10 lookup; CH bounds DateTime64 / Decimal scale to [0, 9] */ +extern const int64_t pow10i[10]; + +/* ereport ERROR carrying chc_err->msg with sqlstate / prefix. */ +pg_noreturn extern void raise_chc(const chc_err * err, int sqlstate, + const char *prefix); + +/* True if server advertises output_format_native_write_json_as_string. */ +extern bool server_supports_json_as_string(const chc_client * c); + +/* + * Per-row, per-column append. Set isnull for NULL values (column must be + * Nullable). ereports on type mismatch / NULL-into-NOT-NULL. + */ +extern void ch_binary_append_int(ch_binary_insert_handle * h, size_t col, + int64_t val, bool isnull); +extern void ch_binary_append_uint(ch_binary_insert_handle * h, size_t col, + uint64_t val, bool isnull); +extern void ch_binary_append_bool(ch_binary_insert_handle * h, size_t col, + bool val, bool isnull); +extern void ch_binary_append_double(ch_binary_insert_handle * h, size_t col, + double val, bool isnull); +extern void ch_binary_append_float(ch_binary_insert_handle * h, size_t col, + float val, bool isnull); +extern void ch_binary_append_bytes(ch_binary_insert_handle * h, size_t col, + const void *p, size_t n, bool isnull); +extern void ch_binary_append_decimal(ch_binary_insert_handle * h, size_t col, + const char *digits, bool isnull); +extern void ch_binary_append_uuid(ch_binary_insert_handle * h, size_t col, + const uint8_t bytes[16], bool isnull); + +/* + * IPv4: addr_be is 4 BE bytes (matches PG inet ip_addr layout). + * IPv6: addr_be is 16 BE bytes. Pass NULL with isnull=true. + */ +extern void ch_binary_append_inet(ch_binary_insert_handle * h, size_t col, + const uint8_t * addr_be, size_t addrlen, + bool isnull); + +/* + * Per-row Date/DateTime/DateTime64 sent as seconds-since-epoch (int64). + * For DateTime64 value is wire-level integer at column's scale; + * encode.c does scaling. + */ +extern void ch_binary_append_date_seconds(ch_binary_insert_handle * h, size_t col, + int64_t seconds, bool isnull); +extern void ch_binary_append_datetime_seconds(ch_binary_insert_handle * h, size_t col, + int64_t seconds, bool isnull); +extern void ch_binary_append_datetime64_raw(ch_binary_insert_handle * h, size_t col, + int64_t raw, bool isnull); + +/* + * Array element append. Open with array_begin. All subsequent + * ch_binary_append_* calls target inner items column regardless of + * `col` until ch_binary_array_end. + */ +extern void ch_binary_array_begin(ch_binary_insert_handle * h, size_t col); +extern void ch_binary_array_end(ch_binary_insert_handle * h); + +/* True when handle has an active array context (for assertions). */ +extern bool ch_binary_array_active(const ch_binary_insert_handle * h); + +/* + * Inspect underlying CH column kind. Used by encode.c to + * dispatch on (Oid pg, chc_kind ch). When an array context is open + * (between ch_binary_array_begin/_end) returned kind is element kind, + * not CHC_ARRAY. + */ +extern chc_kind ch_binary_column_kind(const ch_binary_insert_handle * h, + size_t col); +extern uint32_t ch_binary_column_datetime64_precision(const ch_binary_insert_handle * h, + size_t col); + +/* Send buffered rows and clear; ready for next batch. */ +extern void ch_binary_flush_block(ch_binary_insert_handle * h); + +#endif /* PG_CLICKHOUSE_BINARY_INTERNAL_H */ diff --git a/src/binary/connection.c b/src/binary/connection.c new file mode 100644 index 00000000..566049f4 --- /dev/null +++ b/src/binary/connection.c @@ -0,0 +1,261 @@ +/* + * connection.c + * + * Open / probe / close for the binary driver. TCP + optional TLS, then + * chc_client_init handshakes the protocol. Cleanup runs through a + * MemoryContext reset callback so half-built state still releases the + * fd / SSL pair on transaction abort. + */ + +#include "postgres.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/memutils.h" +#include "utils/palloc.h" + +#include +#include + +#include "binary_internal.h" +#include "engine.h" + +#define CLICKHOUSE_SECURE_PORT 9440 + +static bool +cancel_adapter(void *ud) +{ + struct ch_binary_state *s = ud; + + return s->check_cancel_fn ? s->check_cancel_fn() : false; +} + +/* + * Releases OS resources owned by the connection. The chc_client and its + * read buffer live in s->cxt and are freed by the surrounding + * MemoryContextDelete; only fd / SSL need an explicit close. + */ +static void +binary_state_reset_cb(void *arg) +{ + struct ch_binary_state *s = arg; + + if (s->client) + { + chc_client_close(s->client); + s->client = NULL; + } + if (s->ssl) + { + SSL_shutdown(s->ssl); + SSL_free(s->ssl); + s->ssl = NULL; + } + if (s->ssl_ctx) + { + SSL_CTX_free(s->ssl_ctx); + s->ssl_ctx = NULL; + } + if (s->fd >= 0) + { + close(s->fd); + s->fd = -1; + } +} + +static int +tcp_connect(const char *host, int port) +{ + struct addrinfo hints = {}; + struct addrinfo *res = NULL; + char port_s[16]; + + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + snprintf(port_s, sizeof(port_s), "%d", port); + int rc = getaddrinfo(host, port_s, &hints, &res); + + if (rc != 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: getaddrinfo(%s:%d): %s", + host, port, gai_strerror(rc)))); + + int fd = -1; + int save_errno = ECONNREFUSED; + + for (struct addrinfo *ai = res; ai; ai = ai->ai_next) + { + fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) + { + save_errno = errno; + continue; + } + if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) + break; + save_errno = errno; + close(fd); + fd = -1; + } + freeaddrinfo(res); + if (fd < 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: connect(%s:%d): %s", + host, port, strerror(save_errno)))); + + int one = 1; + + (void) setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + (void) setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(one)); + return fd; +} + +static void +tls_connect(struct ch_binary_state *s, const char *host) +{ + static int openssl_init_done = 0; + + if (!openssl_init_done) + { + SSL_library_init(); + SSL_load_error_strings(); + OpenSSL_add_all_algorithms(); + openssl_init_done = 1; + } + + s->ssl_ctx = SSL_CTX_new(TLS_client_method()); + if (!s->ssl_ctx) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize opensssl"), + errdetail("SSL_CTX_new failed"))); + SSL_CTX_set_verify(s->ssl_ctx, SSL_VERIFY_NONE, NULL); + + s->ssl = SSL_new(s->ssl_ctx); + if (!s->ssl) + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize opensssl"), + errdetail("SSL_new failed"))); + SSL_set_tlsext_host_name(s->ssl, host); + SSL_set_fd(s->ssl, s->fd); + if (SSL_connect(s->ssl) != 1) + { + unsigned long e = ERR_peek_last_error(); + char ebuf[160]; + + ERR_error_string_n(e, ebuf, sizeof(ebuf)); + ereport(ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: openssl failed to connect"), + errdetail("%s", ebuf))); + } +} + +ch_binary_connection_t * +ch_binary_connect(ch_connection_details * details) +{ + const char *host = details->host ? details->host : "127.0.0.1"; + int port = details->port; + bool tls = false; + + if (port == 0) + { + if (ch_is_cloud_host(host)) + { + port = CLICKHOUSE_SECURE_PORT; + tls = true; + } + else + port = 9000; + } + else if (port == CLICKHOUSE_SECURE_PORT) + tls = true; + + MemoryContext cxt = AllocSetContextCreate(CacheMemoryContext, + "pg_clickhouse binary connection", + ALLOCSET_SMALL_SIZES); + MemoryContext old = MemoryContextSwitchTo(cxt); + struct ch_binary_state *s; + ch_binary_connection_t *conn; + + PG_TRY(); + { + s = palloc0(sizeof(*s)); + conn = palloc0(sizeof(*conn)); + s->cxt = cxt; + s->fd = -1; + s->tls = tls; + s->reset_cb.func = binary_state_reset_cb; + s->reset_cb.arg = s; + MemoryContextRegisterResetCallback(cxt, &s->reset_cb); + conn->client = s; + + s->fd = tcp_connect(host, port); + if (tls) + { + tls_connect(s, host); + chc_openssl_io_init(&s->openssl_state, &s->io, s->ssl, + cancel_adapter, s); + } + else + { + chc_posix_io_init(&s->posix_state, &s->io, s->fd, + cancel_adapter, s); + } + + chc_client_opts opts = { + .database = details->dbname ? details->dbname : "default", + .user = details->username ? details->username : "default", + .password = details->password ? details->password : "", + }; + chc_err err = {}; + int rc = chc_client_init(&s->client, &opts, &pg_chc_alloc, &s->io, &err); + + if (rc != CHC_OK) + raise_chc(&err, ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION, + "connection error: "); + } + PG_CATCH(); + { + MemoryContextSwitchTo(old); + MemoryContextDelete(cxt); + PG_RE_THROW(); + } + PG_END_TRY(); + + MemoryContextSwitchTo(old); + return conn; +} + +bool +ch_binary_is_broken(const ch_binary_connection_t * conn) +{ + if (!conn) + return false; + const struct ch_binary_state *s = (const struct ch_binary_state *) conn->client; + + return s ? s->broken : false; +} + +void +ch_binary_close(ch_binary_connection_t * conn) +{ + if (!conn) + return; + struct ch_binary_state *s = conn_state(conn); + + if (s) + MemoryContextDelete(s->cxt); + /* conn itself was palloc'd inside s->cxt; freed by the delete */ +} diff --git a/src/convert.c b/src/binary/convert.c similarity index 94% rename from src/convert.c rename to src/binary/convert.c index c056447b..e277c5e3 100644 --- a/src/convert.c +++ b/src/binary/convert.c @@ -1,6 +1,4 @@ -/* - * Conversion functions for the binary driver. -*/ +/* Conversion functions for the binary driver. */ #include "postgres.h" @@ -19,7 +17,7 @@ #include "executor/tuptable.h" #include "fdw.h" -#include "binary.hh" +#include "binary_internal.h" #include typedef struct ch_convert_state ch_convert_state; @@ -232,8 +230,8 @@ convert_array(ch_convert_state * state, Datum val) } else { - int dims[MAXDIM]; - int lbs[MAXDIM]; + int dims[MAXDIM] = {}; + int lbs[MAXDIM] = {}; size_t total = 1; size_t idx = 0; Datum *flat; @@ -319,12 +317,6 @@ convert_bool(ch_convert_state * state, Datum val) return BoolGetDatum(dat); } -inline static Datum -convert_bool_to_int16(ch_convert_output_state * state, Datum val) -{ - return Int16GetDatum(DatumGetBool(val) ? 1 : 0); -} - Datum ch_binary_convert_datum(void *state, Datum val) { @@ -416,7 +408,8 @@ ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype) if (typentry->tupDesc == NULL) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("type %s is not composite", + errmsg("pg_clickhouse: cannot return %s as %s", + slot->ch_type_name ? slot->ch_type_name : "?", format_type_be(outtype)))); tupdesc = typentry->tupDesc; @@ -514,21 +507,6 @@ init_output_convert_state(ch_convert_output_state * state) if (state->intype == BYTEAOID && state->outtype == TEXTOID) return; - /* column_append() handles both JSON and JSONB, so no need to convert. */ - if ( - (state->intype == JSONBOID && state->outtype == JSONOID) - || (state->intype == JSONOID && state->outtype == JSONBOID) - ) - return; - - /* Postgres has no cast from bool to INT16, so provide our own. */ - if (state->outtype == INT2OID && state->intype == BOOLOID) - { - state->func = convert_bool_to_int16; - state->ctype = COERCION_PATH_FUNC; - return; - } - state->func = convert_out_generic; state->ctype = find_coercion_pathway(state->outtype, state->intype, @@ -672,10 +650,10 @@ build_nested_binary_array(int level, int ndim, int *dims, Oid item_type, } /* - * For each value to be output, convert it, if necessary, from the Postgres - * Datum type defined for the foreign table to a Datum that column_append() in - * binary.cpp knows how to convert to a ClickHouse type. No conversion for - * binary-compatible types; other types require a CAST. + * For each output value, convert it, if necessary, from the Postgres Datum + * type defined for the foreign table to a Datum that the binary INSERT + * path in encode.c knows how to convert to a ClickHouse type. No conversion + * for binary-compatible types; other types require a CAST. * ch_binary_make_tuple_map() makes this determination for each type, stored * in insert_state->conversion_states) */ diff --git a/src/binary/decode.c b/src/binary/decode.c new file mode 100644 index 00000000..6f24c0c2 --- /dev/null +++ b/src/binary/decode.c @@ -0,0 +1,888 @@ +/* + * decode.c + * + * Build PG Datums from a chc_column produced by clickhouse-c. Walks the + * wire-shaped column accessors directly; per-row transforms (decimal to + * text, IPv4/IPv6 text, UUID byteswap, enum name lookup, LowCardinality + * key->dict deref, Nullable strip) happen inline at read time. + */ + +#include "postgres.h" + +#include +#include /* AF_INET, expanded by PG inet macros */ + +#include "catalog/pg_type_d.h" +#include "fmgr.h" +#include "port/pg_bswap.h" +#include "utils/builtins.h" +#include "utils/date.h" +#include "utils/inet.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/numeric.h" +#include "utils/timestamp.h" +#include "utils/uuid.h" + +#include "binary_internal.h" + +/* CH Date / Date32 epoch is unix; offset to PG epoch (2000-01-01) */ +#define CH_TO_PG_DATE_OFFSET (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) + +/* Little-endian fixed-width reads at row offset. */ +static inline int8_t +rd_i8(const uint8_t * p, uint64_t row) +{ + return (int8_t) p[row]; +} + +static inline uint8_t +rd_u8(const uint8_t * p, uint64_t row) +{ + return p[row]; +} + +static inline bool +rd_bool(const bool *p, uint64_t row) +{ + return (bool) p[row]; +} + +static inline int16_t +rd_i16(const uint8_t * p, uint64_t row) +{ + int16_t v; + + memcpy(&v, p + row * sizeof(int16_t), sizeof(int16_t)); + return v; +} + +static inline uint16_t +rd_u16(const uint8_t * p, uint64_t row) +{ + uint16_t v; + + memcpy(&v, p + row * sizeof(uint16_t), sizeof(uint16_t)); + return v; +} + +static inline int32_t +rd_i32(const uint8_t * p, uint64_t row) +{ + int32_t v; + + memcpy(&v, p + row * sizeof(int32_t), sizeof(int32_t)); + return v; +} + +static inline uint32_t +rd_u32(const uint8_t * p, uint64_t row) +{ + uint32_t v; + + memcpy(&v, p + row * sizeof(uint32_t), sizeof(uint32_t)); + return v; +} + +static inline int64_t +rd_i64(const uint8_t * p, uint64_t row) +{ + int64_t v; + + memcpy(&v, p + row * sizeof(int64_t), sizeof(int64_t)); + return v; +} + +static inline uint64_t +rd_u64(const uint8_t * p, uint64_t row) +{ + uint64_t v; + + memcpy(&v, p + row * sizeof(uint64_t), sizeof(uint64_t)); + return v; +} + +static inline float +rd_f32(const uint8_t * p, uint64_t row) +{ + float v; + + memcpy(&v, p + row * sizeof(float), sizeof(float)); + return v; +} + +static inline double +rd_f64(const uint8_t * p, uint64_t row) +{ + double v; + + memcpy(&v, p + row * sizeof(double), sizeof(double)); + return v; +} + +static inline void +slice_str(const chc_column * col, uint64_t row, + const char **out_ptr, size_t * out_len) +{ + const uint64_t *offs = chc_column_string_offsets(col); + const uint8_t *data = chc_column_string_data(col); + uint64_t start = row == 0 ? 0 : offs[row - 1]; + uint64_t end = offs[row]; + + *out_ptr = (const char *) data + start; + *out_len = (size_t) (end - start); +} + +static Oid +ch_kind_to_pg_oid(const chc_type * type) +{ + switch (chc_type_kind(type)) + { + case CHC_VOID: + case CHC_NOTHING: + return InvalidOid; + case CHC_NULLABLE: + case CHC_LOW_CARDINALITY: + return ch_kind_to_pg_oid(chc_type_child(type, 0)); + case CHC_INT8: + case CHC_INT16: + case CHC_UINT8: + return INT2OID; + case CHC_BOOL: + return BOOLOID; + case CHC_INT32: + case CHC_UINT16: + return INT4OID; + case CHC_INT64: + case CHC_UINT32: + case CHC_UINT64: + return INT8OID; + case CHC_FLOAT32: + return FLOAT4OID; + case CHC_FLOAT64: + return FLOAT8OID; + case CHC_DECIMAL32: + case CHC_DECIMAL64: + case CHC_DECIMAL128: + case CHC_DECIMAL256: + return NUMERICOID; + case CHC_STRING: + case CHC_FIXED_STRING: + case CHC_ENUM8: + case CHC_ENUM16: + return TEXTOID; + case CHC_JSON: + case CHC_OBJECT: + return JSONBOID; + case CHC_DATE: + case CHC_DATE32: + return DATEOID; + case CHC_DATETIME: + case CHC_DATETIME64: + return TIMESTAMPTZOID; + case CHC_UUID: + return UUIDOID; + case CHC_IPV4: + case CHC_IPV6: + return INETOID; + case CHC_ARRAY: + return ANYARRAYOID; + case CHC_TUPLE: + return RECORDOID; + default: + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported column type"))); + } + /* unreachable */ + return InvalidOid; +} + +static Datum read_value(const chc_column * col, const chc_type * type, + uint64_t row, Oid * valtype, bool *is_null); + +/* + * Format a ClickHouse Decimal (two's-complement signed integer in LE bytes of + * width 4/8/16/32 for Decimal32/64/128/256, with `scale` fractional digits + * carried on the column type) into `out`. Returns bytes written, -1 on overflow. + */ +static int +format_decimal_text(const uint8_t * bytes, size_t width, uint32_t scale, + char *out, size_t out_cap) +{ + uint32_t mag[8]; + size_t nwords = width / 4; + bool neg = false; + + if (width == 0 || width > 32 || width % 4 != 0) + return -1; + memcpy(mag, bytes, width); + /* top bit of MSW is sign; negate two's-complement to get magnitude */ + if (mag[nwords - 1] & 0x80000000u) + { + neg = true; + for (size_t i = 0; i < nwords; i++) + mag[i] = ~mag[i]; + uint64_t carry = 1; + + for (size_t i = 0; i < nwords && carry; i++) + { + uint64_t v = (uint64_t) mag[i] + carry; + + mag[i] = (uint32_t) v; + carry = v >> 32; + } + } + + char buf[80]; + int n = 0; + bool nonzero; + + /* base-10 division of mag yields digits LSB-first */ + do + { + uint64_t rem = 0; + + nonzero = false; + for (ssize_t i = (ssize_t) nwords - 1; i >= 0; i--) + { + uint64_t v = (rem << 32) | mag[i]; + + mag[i] = (uint32_t) (v / 10); + rem = v % 10; + if (mag[i]) + nonzero = true; + } + buf[n++] = (char) ('0' + (uint32_t) rem); + } while (nonzero && n < (int) sizeof(buf)); + + /* pad leading zeros so digit count covers fractional portion */ + while (n <= (int) scale) + buf[n++] = '0'; + + size_t need = (size_t) neg + (size_t) n + (scale ? 1 : 0) + 1; + + if (need > out_cap) + return -1; + char *p = out; + + if (neg) + *p++ = '-'; + + /* emit MSD-first, inserting '.' before `scale` trailing digits */ + for (int i = n - 1; i >= 0; i--) + { + if (i + 1 == scale) + *p++ = '.'; + *p++ = buf[i]; + } + *p = '\0'; + return (int) (p - out); +} + +static Datum +read_decimal(const chc_column * col, const chc_type * type, uint64_t row) +{ + size_t es; + const uint8_t *p = (const uint8_t *) chc_column_fixed_data(col, &es); + uint32_t scale = (uint32_t) chc_type_decimal_scale(type); + char buf[80]; + int rc; + +#if PG_VERSION_NUM >= 140000 + /* Decimal32/64 fit in int64; skip the byte-array text path. */ + if (es == 4) + return NumericGetDatum(int64_div_fast_to_numeric(rd_i32(p, row), (int) scale)); + if (es == 8) + return NumericGetDatum(int64_div_fast_to_numeric(rd_i64(p, row), (int) scale)); +#endif + + rc = format_decimal_text(p + row * es, es, scale, buf, sizeof(buf)); + if (rc < 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: decimal too wide"))); + return DirectFunctionCall3(numeric_in, CStringGetDatum(buf), + ObjectIdGetDatum(0), Int32GetDatum(-1)); +} + +static Datum +read_string_as_text(const chc_column * col, uint64_t row) +{ + const char *p; + size_t len; + + slice_str(col, row, &p, &len); + return PointerGetDatum(cstring_to_text_with_len(p, len)); +} + +static Datum +read_fixedstring_as_text(const chc_column * col, uint64_t row) +{ + size_t width; + const uint8_t *base = chc_column_fixed_data(col, &width); + + return PointerGetDatum(cstring_to_text_with_len((const char *) base + row * width, + width)); +} + +static Datum +read_uuid(const chc_column * col, uint64_t row) +{ + pg_uuid_t *u = (pg_uuid_t *) palloc(sizeof(pg_uuid_t)); + const uint8_t *p = (uint8_t *) chc_column_fixed_data(col, NULL) + row * 16; + uint64_t a, + b; + + memcpy(&a, p, 8); + memcpy(&b, p + 8, 8); + a = pg_hton64(a); + b = pg_hton64(b); + memcpy(u->data, &a, 8); + memcpy(u->data + 8, &b, 8); + return UUIDPGetDatum(u); +} + +/* + * IPv4 wire format: native uint32 (LE on supported hosts). PG inet wants + * BE bytes of dotted-quad, so pg_hton32 the value into ip_addr directly. + */ +static Datum +read_ipv4(const chc_column * col, uint64_t row) +{ + inet *res = (inet *) palloc0(sizeof(inet)); + uint32_t addr; + + memcpy(&addr, (const uint8_t *) chc_column_fixed_data(col, NULL) + row * 4, 4); + addr = pg_hton32(addr); + ip_family(res) = PGSQL_AF_INET; + ip_bits(res) = 32; + memcpy(ip_addr(res), &addr, 4); + SET_INET_VARSIZE(res); + return InetPGetDatum(res); +} + +/* IPv6 wire is already network order; same layout as PG inet ip_addr. */ +static Datum +read_ipv6(const chc_column * col, uint64_t row) +{ + inet *res = (inet *) palloc0(sizeof(inet)); + + ip_family(res) = PGSQL_AF_INET6; + ip_bits(res) = 128; + memcpy(ip_addr(res), (const uint8_t *) chc_column_fixed_data(col, NULL) + row * 16, 16); + SET_INET_VARSIZE(res); + return InetPGetDatum(res); +} + +static Datum +read_enum_as_text(const chc_column * col, const chc_type * type, uint64_t row) +{ + size_t es; + const uint8_t *p = (const uint8_t *) chc_column_fixed_data(col, &es); + int64_t v = 0; + + if (es == 1) + v = (int8_t) p[row]; + else + { + int16_t t; + + memcpy(&t, p + row * 2, 2); + v = t; + } + + size_t n = chc_type_enum_count(type); + + for (size_t i = 0; i < n; i++) + { + const char *en; + size_t el; + int64_t ev; + + chc_type_enum_at(type, i, &en, &el, &ev); + if (ev == v) + return PointerGetDatum(cstring_to_text_with_len(en ? en : "", el)); + } + return PointerGetDatum(cstring_to_text_with_len("", 0)); +} + +/* + * JSON body bytes are STRING-serialized JSON document text. Run them + * through json_in / jsonb_in depending on the caller's target valtype + * (set by ch_kind_to_pg_oid to JSONBOID, overridable when the foreign + * table declares the column as `json`). + */ +static Datum +read_json(const chc_column * col, uint64_t row, Oid valtype) +{ + const char *p; + size_t len; + char *cstr; + Datum ret; + + slice_str(col, row, &p, &len); + cstr = palloc(len + 1); + memcpy(cstr, p, len); + cstr[len] = '\0'; + ret = DirectFunctionCall1(valtype == JSONOID ? json_in : jsonb_in, + CStringGetDatum(cstr)); + pfree(cstr); + return ret; +} + +/* + * LowCardinality(String) or LowCardinality(Nullable(String)). Key for + * row indexes into the dict column; the dict's first slot is the null + * sentinel for the Nullable variant. + */ +static Datum +read_lc_string(const chc_column * col, const chc_type * type, uint64_t row, + Oid * valtype, bool *is_null) +{ + int ks = chc_column_lc_key_size(col); + const uint8_t *kp = (const uint8_t *) chc_column_lc_keys(col) + (size_t) row * ks; + uint64_t k = 0; + + switch (ks) + { + case 1: + k = kp[0]; + break; + case 2: + { + uint16_t v; + + memcpy(&v, kp, 2); + k = v; + break; + } + case 4: + { + uint32_t v; + + memcpy(&v, kp, 4); + k = v; + break; + } + case 8: + memcpy(&k, kp, 8); + break; + default: + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: unexpected LowCardinality key size %d", ks))); + } + + const chc_column *dict = chc_column_lc_dict(col); + const chc_type *inner_t = chc_type_child(type, 0); + + if (chc_type_kind(inner_t) == CHC_NULLABLE + && chc_column_layout(dict) == CHC_COL_NULLABLE) + { + const uint8_t *dnm = chc_column_null_map(dict); + + if (dnm && dnm[k]) + { + *valtype = TEXTOID; + *is_null = true; + return (Datum) 0; + } + dict = chc_column_nullable_inner(dict); + } + + *valtype = TEXTOID; + *is_null = false; + if (chc_column_layout(dict) != CHC_COL_STRING) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported LowCardinality inner type"))); + return read_string_as_text(dict, k); +} + +static Datum +read_array(const chc_column * col, const chc_type * type, uint64_t row, + Oid * valtype, bool *is_null) +{ + const uint64_t *offs = chc_column_array_offsets(col); + uint64_t start = row == 0 ? 0 : offs[row - 1]; + uint64_t end = offs[row]; + uint64_t len = end - start; + const chc_type *inner_t = chc_type_child(type, 0); + const chc_column *inner = chc_column_array_values(col); + ch_binary_array_t *slot = (ch_binary_array_t *) palloc(sizeof(ch_binary_array_t)); + const chc_type *leaf = type; + int ndim = 0; + + /* + * postgres uses one array type per element type regardless of nesting, so + * walk past nested Array layers to the leaf scalar type. + */ + while (chc_type_kind(leaf) == CHC_ARRAY) + { + ndim++; + leaf = chc_type_child(leaf, 0); + } + + slot->len = len; + slot->ndim = ndim; + slot->item_type = ch_kind_to_pg_oid(leaf); + slot->array_type = get_array_type(slot->item_type); + if (slot->array_type == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: could not find array type for column type \"%s\"", + chc_type_name(leaf, NULL)))); + + if (len > 0) + { + Oid scratch = slot->item_type; + + slot->datums = (Datum *) palloc0(sizeof(Datum) * len); + slot->nulls = (bool *) palloc0(sizeof(bool) * len); + + /* + * For ndim==1 read_value returns leaf scalars; for ndim>1 inner_t is + * itself CHC_ARRAY so recursion produces nested ch_binary_array_t*. + * Use a scratch valtype to avoid clobbering slot->item_type. + */ + for (uint64_t i = 0; i < len; ++i) + slot->datums[i] = read_value(inner, inner_t, start + i, + &scratch, &slot->nulls[i]); + } + else + { + slot->datums = NULL; + slot->nulls = NULL; + } + + *valtype = ANYARRAYOID; + *is_null = false; + return PointerGetDatum(slot); +} + +static Datum +read_tuple(const chc_column * col, const chc_type * type, uint64_t row, + Oid * valtype, bool *is_null) +{ + size_t n = chc_type_n_children(type); + ch_binary_tuple_t *slot; + + if (n == 0) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: returned tuple is empty"))); + + slot = (ch_binary_tuple_t *) palloc(sizeof(ch_binary_tuple_t)); + slot->datums = (Datum *) palloc(sizeof(Datum) * n); + slot->nulls = (bool *) palloc0(sizeof(bool) * n); + slot->types = (Oid *) palloc0(sizeof(Oid) * n); + slot->len = n; + slot->ch_type_name = chc_type_name(type, NULL); + + for (size_t i = 0; i < n; ++i) + { + const chc_type *ft = chc_type_child(type, i); + const chc_column *fc = chc_column_tuple_child(col, i); + + slot->datums[i] = read_value(fc, ft, row, &slot->types[i], &slot->nulls[i]); + } + + *valtype = RECORDOID; + *is_null = false; + return PointerGetDatum(slot); +} + +static Datum +read_value(const chc_column * col, const chc_type * type, uint64_t row, + Oid * valtype, bool *is_null) +{ + /* Unwrap outer Nullable, handling nulls here. */ + if (chc_type_kind(type) == CHC_NULLABLE) + { + const chc_type *inner_t = chc_type_child(type, 0); + + if (chc_column_layout(col) == CHC_COL_NULLABLE) + { + const uint8_t *nm = chc_column_null_map(col); + + if (nm && nm[row]) + { + *valtype = ch_kind_to_pg_oid(inner_t); + *is_null = true; + return (Datum) 0; + } + col = chc_column_nullable_inner(col); + } + type = inner_t; + } + *is_null = false; + + switch (chc_type_kind(type)) + { + case CHC_VOID: + case CHC_NOTHING: + *valtype = InvalidOid; + *is_null = true; + return (Datum) 0; + case CHC_UINT8: + *valtype = INT2OID; + return (Datum) rd_u8((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_BOOL: + *valtype = BOOLOID; + return (Datum) rd_bool((const bool *) chc_column_fixed_data(col, NULL), row); + case CHC_INT8: + *valtype = INT2OID; + return (Datum) rd_i8((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_INT16: + *valtype = INT2OID; + return (Datum) rd_i16((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_UINT16: + *valtype = INT4OID; + return (Datum) rd_u16((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_INT32: + *valtype = INT4OID; + return (Datum) rd_i32((const uint8_t *) chc_column_fixed_data(col, NULL), row); + case CHC_UINT32: + *valtype = INT8OID; + return Int64GetDatum((int64) rd_u32((const uint8_t *) chc_column_fixed_data(col, NULL), row)); + case CHC_INT64: + *valtype = INT8OID; + return Int64GetDatum(rd_i64((const uint8_t *) chc_column_fixed_data(col, NULL), row)); + case CHC_UINT64: + { + uint64_t v = rd_u64((const uint8_t *) chc_column_fixed_data(col, NULL), row); + + if (v > (uint64_t) PG_INT64_MAX) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("value " UINT64_FORMAT " is out of range of bigint", + v))); + *valtype = INT8OID; + return Int64GetDatum((int64) v); + } + case CHC_FLOAT32: + *valtype = FLOAT4OID; + return Float4GetDatum(rd_f32((const uint8_t *) chc_column_fixed_data(col, NULL), row)); + case CHC_FLOAT64: + *valtype = FLOAT8OID; + return Float8GetDatum(rd_f64((const uint8_t *) chc_column_fixed_data(col, NULL), row)); + case CHC_DECIMAL32: + case CHC_DECIMAL64: + case CHC_DECIMAL128: + case CHC_DECIMAL256: + *valtype = NUMERICOID; + return read_decimal(col, type, row); + case CHC_STRING: + *valtype = TEXTOID; + return read_string_as_text(col, row); + case CHC_ENUM8: + case CHC_ENUM16: + *valtype = TEXTOID; + return read_enum_as_text(col, type, row); + case CHC_JSON: + case CHC_OBJECT: + { + /* + * *valtype arrives set to JSONBOID by default (from + * ch_kind_to_pg_oid) but the caller's foreign-table + * declaration may have requested JSONOID; honor it. + */ + Oid target = (*valtype == JSONOID) ? JSONOID : JSONBOID; + + *valtype = target; + return read_json(col, row, target); + } + case CHC_FIXED_STRING: + *valtype = TEXTOID; + return read_fixedstring_as_text(col, row); + case CHC_DATE: + *valtype = DATEOID; + return DateADTGetDatum((DateADT) rd_u16((const uint8_t *) chc_column_fixed_data(col, NULL), row) + - CH_TO_PG_DATE_OFFSET); + case CHC_DATE32: + *valtype = DATEOID; + return DateADTGetDatum((DateADT) rd_i32((const uint8_t *) chc_column_fixed_data(col, NULL), row) + - CH_TO_PG_DATE_OFFSET); + case CHC_DATETIME: + { + uint32_t secs = rd_u32((const uint8_t *) chc_column_fixed_data(col, NULL), row); + + *valtype = TIMESTAMPTZOID; + return TimestampTzGetDatum(time_t_to_timestamptz((pg_time_t) secs)); + } + case CHC_DATETIME64: + { + int64 raw = rd_i64((const uint8_t *) chc_column_fixed_data(col, NULL), row); + uint32_t scale = chc_type_datetime64_scale(type); + int64 power = pow10i[scale]; + + *valtype = TIMESTAMPTZOID; + return TimestampTzGetDatum(time_t_to_timestamptz(raw / power) + + (raw % power) * (USECS_PER_SEC / power)); + } + case CHC_UUID: + *valtype = UUIDOID; + return read_uuid(col, row); + case CHC_IPV4: + *valtype = INETOID; + return read_ipv4(col, row); + case CHC_IPV6: + *valtype = INETOID; + return read_ipv6(col, row); + case CHC_LOW_CARDINALITY: + return read_lc_string(col, type, row, valtype, is_null); + case CHC_ARRAY: + return read_array(col, type, row, valtype, is_null); + case CHC_TUPLE: + return read_tuple(col, type, row, valtype, is_null); + default: + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported type in binary protocol"))); + } + /* unreachable */ + return (Datum) 0; +} + +/* ---- read state ----------------------------------------------------- */ + +static bool +load_block(ch_binary_read_state_t * state) +{ + state->cur = ch_binary_response_fetch_next_block(state->resp); + if (state->cur == NULL) + { + const char *resp_err = ch_binary_response_error(state->resp); + + if (resp_err) + state->error = pstrdup(resp_err); + state->done = true; + return false; + } + return true; +} + +void +ch_binary_read_state_init(ch_binary_read_state_t * state, ch_binary_response_t * resp) +{ + const char *resp_err; + size_t ncols; + + state->resp = resp; + state->block = 0; + state->row = 0; + state->done = false; + state->error = NULL; + state->coltypes = NULL; + state->values = NULL; + state->nulls = NULL; + state->cur = NULL; + + resp_err = ch_binary_response_error(resp); + if (resp_err) + { + state->done = true; + state->error = pstrdup(resp_err); + return; + } + + ncols = ch_binary_response_columns(resp); + if (ncols == 0) + { + state->done = true; + return; + } + + state->coltypes = palloc0(sizeof(Oid) * ncols); + state->values = palloc0(sizeof(Datum) * ncols); + state->nulls = palloc0(sizeof(bool) * ncols); + + if (!load_block(state)) + return; + + for (size_t i = 0; i < ncols; i++) + state->coltypes[i] = ch_kind_to_pg_oid(chc_block_column_type(state->cur, i)); +} + +bool +ch_binary_read_row(ch_binary_read_state_t * state) +{ + size_t ncols; + + if (state->done || state->coltypes == NULL || state->error) + return false; + + ncols = ch_binary_response_columns(state->resp); + +again: + if (state->cur == NULL) + { + if (!load_block(state)) + return false; + } + + if (state->row >= chc_block_n_rows(state->cur)) + { + state->row = 0; + state->cur = NULL; + goto again; + } + + PG_TRY(); + { + for (size_t i = 0; i < ncols; i++) + { + /* + * Currently, read_value overwrites *valtype for most types + * canonical PG type for that CH kind. For CHC_JSON we honor the + * incoming value so callers (binary_fetch_row) can pin the Datum + * type to JSONOID for `data json` foreign columns and avoid the + * jsonb_in -> jsonb_out round-trip that would strip CH's verbatim + * STRING- serialized formatting. + */ + Oid t = state->coltypes[i]; + const chc_column *col = chc_block_column(state->cur, i); + const chc_type *ct = chc_block_column_type(state->cur, i); + + state->values[i] = read_value(col, ct, state->row, &t, &state->nulls[i]); + } + } + PG_CATCH(); + { + MemoryContext mcxt = GetMemoryChunkContext(state); + MemoryContext oldcxt; + ErrorData *edata; + const char *msg; + static const char prefix[] = "pg_clickhouse: "; + + oldcxt = MemoryContextSwitchTo(mcxt); + edata = CopyErrorData(); + msg = edata->message ? edata->message : "unknown error"; + + /* + * binary_fetch_row re-prefixes with "pg_clickhouse: error while + * reading row:"; drop a leading "pg_clickhouse: " here so the final + * message doesn't carry it twice. + */ + if (strncmp(msg, prefix, sizeof(prefix) - 1) == 0) + msg += sizeof(prefix) - 1; + state->error = pstrdup(msg); + FlushErrorState(); + FreeErrorData(edata); + MemoryContextSwitchTo(oldcxt); + state->done = true; + return false; + } + PG_END_TRY(); + + state->row++; + return true; +} + +void +ch_binary_read_state_free(ch_binary_read_state_t * state) +{ + /* state->error is palloc'd; freed with surrounding memory context. */ + state->error = NULL; +} diff --git a/src/binary/encode.c b/src/binary/encode.c new file mode 100644 index 00000000..c21d50b6 --- /dev/null +++ b/src/binary/encode.c @@ -0,0 +1,428 @@ +/* + * encode.c + * + * PG-side INSERT path. Reads PG Datums and dispatches into the typed + * ch_binary_append_* shims exposed by insert.c. + */ + +#include "postgres.h" + +#include +#include /* AF_INET, expanded by PG inet macros */ + +#include "access/tupdesc.h" +#include "catalog/pg_type_d.h" +#include "nodes/pg_list.h" +#include "fmgr.h" +#include "utils/builtins.h" +#include "utils/date.h" +#include "utils/inet.h" +#include "utils/lsyscache.h" +#include "utils/palloc.h" +#include "utils/timestamp.h" +#include "utils/uuid.h" + +#include "binary_internal.h" + +/* CH Date epoch is unix; offset to PG epoch (2000-01-01) */ +#define CH_TO_PG_DATE_OFFSET (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) + +/* + * Map a CH column kind to the PG type our import path uses. Used when + * constructing the TupleDesc for INSERT...VALUES. + */ +static Oid +ch_kind_to_pg_oid_for_insert(const chc_type * type, const char *colname) +{ + switch (chc_type_kind(type)) + { + case CHC_INT8: + case CHC_INT16: + case CHC_UINT8: + return INT2OID; + case CHC_BOOL: + return BOOLOID; + case CHC_INT32: + case CHC_UINT16: + return INT4OID; + case CHC_INT64: + case CHC_UINT32: + case CHC_UINT64: + return INT8OID; + case CHC_FLOAT32: + return FLOAT4OID; + case CHC_FLOAT64: + return FLOAT8OID; + case CHC_DECIMAL32: + case CHC_DECIMAL64: + case CHC_DECIMAL128: + case CHC_DECIMAL256: + return NUMERICOID; + case CHC_STRING: + case CHC_FIXED_STRING: + case CHC_ENUM8: + case CHC_ENUM16: + return TEXTOID; + case CHC_JSON: + case CHC_OBJECT: + return JSONBOID; + case CHC_DATE: + case CHC_DATE32: + return DATEOID; + case CHC_DATETIME: + case CHC_DATETIME64: + return TIMESTAMPTZOID; + case CHC_UUID: + return UUIDOID; + case CHC_IPV4: + case CHC_IPV6: + return INETOID; + case CHC_ARRAY: + { + /* + * postgres uses one array type per element type regardless of + * nesting, so unwrap nested Array layers before looking up. + */ + const chc_type *leaf = type; + Oid item_type; + Oid array_type; + + while (chc_type_kind(leaf) == CHC_ARRAY) + leaf = chc_type_child(leaf, 0); + item_type = ch_kind_to_pg_oid_for_insert(leaf, colname); + array_type = get_array_type(item_type); + + if (array_type == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: could not find array type for column \"%s\"", + colname ? colname : "?"))); + return array_type; + } + case CHC_TUPLE: + return RECORDOID; + case CHC_NULLABLE: + case CHC_LOW_CARDINALITY: + return ch_kind_to_pg_oid_for_insert(chc_type_child(type, 0), colname); + default: + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported column type \"%s\" for \"%s\"", + chc_type_name(type, NULL), + colname ? colname : "?"))); + } + return InvalidOid; +} + +void +ch_binary_prepare_insert(void *conn, const ch_query * query, + ch_binary_insert_state * state) +{ + ch_binary_column_info *cols = NULL; + size_t n = 0; + ch_binary_insert_handle *h; + + h = ch_binary_begin_insert((ch_binary_connection_t *) conn, query, &cols, &n); + + state->len = n; + state->insert_block = h; + + if (n == 0) + return; + + state->outdesc = CreateTemplateTupleDesc(n); + for (size_t i = 0; i < n; i++) + { + Oid pg_type = ch_kind_to_pg_oid_for_insert(cols[i].type, cols[i].name); + + /* + * CHC_JSON defaults to JSONBOID; honor foreign-table's JSONOID + * declaration when present so outdesc matches the source slot and no + * json↔jsonb conversion is needed. + */ + if (pg_type == JSONBOID && query->tupdesc && query->attr_nums) + { + AttrNumber attnum = list_nth_int((List *) query->attr_nums, i); + + if (attnum >= 1 && attnum <= query->tupdesc->natts && + TupleDescAttr(query->tupdesc, attnum - 1)->atttypid == JSONOID) + pg_type = JSONOID; + } + + TupleDescInitEntry(state->outdesc, (AttrNumber) (i + 1), + cols[i].name ? cols[i].name : "", + pg_type, -1, 0); + } +} + +/* + * Append a single value (already extracted from a Datum + isnull) into the + * current column, dispatching on (PG Oid, CH kind). ereports on mismatch. + */ +static void +append_one(ch_binary_insert_handle * h, size_t colidx, + chc_kind kind, Datum val, Oid valtype, bool isnull) +{ + switch (valtype) + { + case INT2OID: + case INT4OID: + case INT8OID: + { + int64_t v = 0; + + /* Support mixing integer types. */ + if (!(kind == CHC_BOOL + || (kind >= CHC_INT8 && kind <= CHC_INT64) + || (kind >= CHC_UINT8 && kind <= CHC_UINT64))) + goto type_mismatch; + if (!isnull) + { + if (valtype == INT2OID) + v = (int64_t) DatumGetInt16(val); + else if (valtype == INT4OID) + v = (int64_t) DatumGetInt32(val); + else + v = DatumGetInt64(val); + } + ch_binary_append_int(h, colidx, v, isnull); + return; + } + case BOOLOID: + if (kind != CHC_BOOL && kind != CHC_UINT8) + goto type_mismatch; + ch_binary_append_bool(h, colidx, (bool) val, isnull); + return; + case FLOAT4OID: + if (kind != CHC_FLOAT32) + goto type_mismatch; + ch_binary_append_float(h, colidx, DatumGetFloat4(val), isnull); + return; + case FLOAT8OID: + if (kind != CHC_FLOAT64) + goto type_mismatch; + ch_binary_append_double(h, colidx, DatumGetFloat8(val), isnull); + return; + case NUMERICOID: + { + char *s = NULL; + + if (kind != CHC_DECIMAL32 && kind != CHC_DECIMAL64 + && kind != CHC_DECIMAL128 && kind != CHC_DECIMAL256) + goto type_mismatch; + if (!isnull) + s = DatumGetCString(DirectFunctionCall1(numeric_out, val)); + ch_binary_append_decimal(h, colidx, s, isnull); + if (s) + pfree(s); + return; + } + case TEXTOID: + { + const char *p = NULL; + size_t len = 0; + text *string = NULL; + + if (!isnull) + { + string = PG_DETOAST_DATUM(val); + p = VARDATA(string); + len = VARSIZE_ANY_EXHDR(string); + } + switch (kind) + { + case CHC_FIXED_STRING: + case CHC_STRING: + case CHC_ENUM8: + case CHC_ENUM16: + ch_binary_append_bytes(h, colidx, p, len, isnull); + return; + default: + goto type_mismatch; + } + } + case DATEOID: + { + int64_t seconds = 0; + + if (kind != CHC_DATE && kind != CHC_DATE32) + goto type_mismatch; + if (!isnull) + seconds = ((int64_t) DatumGetDateADT(val) + + CH_TO_PG_DATE_OFFSET) * SECS_PER_DAY; + ch_binary_append_date_seconds(h, colidx, seconds, isnull); + return; + } + case TIMESTAMPOID: + case TIMESTAMPTZOID: + { + switch (kind) + { + case CHC_DATETIME: + { + int64_t seconds = isnull + ? 0 + : (int64_t) timestamptz_to_time_t(DatumGetTimestamp(val)); + + ch_binary_append_datetime_seconds(h, colidx, seconds, isnull); + } break; + case CHC_DATETIME64: + { + int64_t raw = 0; + + if (!isnull) + { + uint32_t prec = ch_binary_column_datetime64_precision(h, colidx); + Timestamp t = DatumGetTimestamp(val); + int64 power = pow10i[prec]; + int64 secs = t / USECS_PER_SEC; + int64 us_rem = t % USECS_PER_SEC; + + /* + * floor-divide; C trunc-to-zero leaves + * negative remainder + */ + if (us_rem < 0) + { + secs -= 1; + us_rem += USECS_PER_SEC; + } + secs += CH_TO_PG_DATE_OFFSET * SECS_PER_DAY; + raw = secs * power + us_rem * power / USECS_PER_SEC; + } + ch_binary_append_datetime64_raw(h, colidx, raw, isnull); + } break; + default: + goto type_mismatch; + } + return; + } + case ANYARRAYOID: + { + ch_binary_array_t *arr; + chc_kind item_kind; + Oid child_valtype; + + if (kind != CHC_ARRAY) + goto type_mismatch; + + arr = (ch_binary_array_t *) DatumGetPointer(val); + ch_binary_array_begin(h, colidx); + + /* + * While array_begin is active ch_binary_column_kind targets + * the inner element kind. For nested arrays the children are + * themselves ch_binary_array_t* so recurse with ANYARRAYOID; + * at the leaf level use the scalar item_type. + */ + item_kind = ch_binary_column_kind(h, colidx); + child_valtype = (arr->ndim > 1) ? ANYARRAYOID : arr->item_type; + for (size_t i = 0; i < arr->len; i++) + append_one(h, 0, item_kind, arr->datums[i], child_valtype, arr->nulls[i]); + + ch_binary_array_end(h); + return; + } + case UUIDOID: + { + uint8_t bytes[16]; + + if (kind != CHC_UUID) + goto type_mismatch; + if (!isnull) + memcpy(bytes, DatumGetUUIDP(val)->data, 16); + else + memset(bytes, 0, 16); + ch_binary_append_uuid(h, colidx, bytes, isnull); + return; + } + case INETOID: + { + const uint8_t *addr = NULL; + size_t addrlen = 0; + + if (kind != CHC_IPV4 && kind != CHC_IPV6) + goto type_mismatch; + if (!isnull) + { + inet *ipa = DatumGetInetPP(val); + int fam = ip_family(ipa); + int expected = (kind == CHC_IPV4) ? PGSQL_AF_INET : PGSQL_AF_INET6; + + if (fam != expected) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: inet family mismatch for column %zu", + colidx))); + addr = ip_addr(ipa); + addrlen = ip_addrsize(ipa); + } + else + addrlen = (kind == CHC_IPV4) ? 4 : 16; + ch_binary_append_inet(h, colidx, addr, addrlen, isnull); + return; + } + case JSONOID: + case JSONBOID: + { + char *s = NULL; + size_t len = 0; + + if (kind != CHC_JSON && kind != CHC_OBJECT) + goto type_mismatch; + if (!isnull) + { + s = DatumGetCString(DirectFunctionCall1(valtype == JSONBOID + ? jsonb_out + : json_out, + val)); + len = strlen(s); + } + ch_binary_append_bytes(h, colidx, s, len, isnull); + if (s) + pfree(s); + return; + } + default: + goto type_mismatch; + } + +type_mismatch: + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: unexpected PG/CH type pair for column %zu", colidx))); +} + +void +ch_binary_column_append_data(ch_binary_insert_state * state, size_t colidx) +{ + Datum val = state->values[colidx]; + Oid valtype = TupleDescAttr(state->outdesc, colidx)->atttypid; + bool isnull = state->nulls[colidx]; + chc_kind kind = ch_binary_column_kind(state->insert_block, colidx); + + append_one(state->insert_block, colidx, kind, val, valtype, isnull); +} + +void +ch_binary_insert_columns(ch_binary_insert_state * state) +{ + ch_binary_flush_block(state->insert_block); +} + +void +ch_binary_insert_state_free(void *c) +{ + ch_binary_insert_state *state = (ch_binary_insert_state *) c; + + if (state->insert_block == NULL) + return; + + /* + * Reset-callback context: cannot ereport. Finalize runs from the FDW + * happy path before we get here; if it did not (mid-query abort), the + * release call flags the connection broken so the next query rebuilds. + */ + ch_binary_release_insert(state->insert_block); + state->insert_block = NULL; +} diff --git a/src/binary/insert.c b/src/binary/insert.c new file mode 100644 index 00000000..f202920e --- /dev/null +++ b/src/binary/insert.c @@ -0,0 +1,1475 @@ +/* + * insert.c + * + * INSERT path for the binary driver. begin_insert sends the query, takes + * the server's empty Data block as schema, classifies each column into + * one of a handful of accumulator layouts (fixed / string / LC / array + * variants), and exposes typed ch_binary_append_* shims. flush_block + * materialises the accumulators into a chc_block_builder and ships one + * Data packet; finalize_insert sends the closing empty Data and drains. + * release_insert is reset-callback safe: it never talks to the wire. + */ + +#include "postgres.h" + +#include + +#include "common/hashfn.h" +#include "port/pg_bswap.h" +#include "utils/memutils.h" +#include "utils/palloc.h" + +#include "binary_internal.h" + +/* dynamic buffer for raw bytes; palloc-backed, freed by context delete */ +typedef struct dynbuf +{ + uint8_t *data; + size_t len; + size_t cap; +} dynbuf; + +static void +dynbuf_reserve(dynbuf * b, size_t need) +{ + if (need <= b->cap) + return; + size_t ncap = b->cap ? b->cap : 64; + + while (ncap < need) + ncap *= 2; + b->data = b->data + ? repalloc_huge(b->data, ncap) + : MemoryContextAllocExtended(CurrentMemoryContext, ncap, MCXT_ALLOC_HUGE); + b->cap = ncap; +} + +static void +dynbuf_append(dynbuf * b, const void *src, size_t n) +{ + dynbuf_reserve(b, b->len + n); + if (src && n) + memcpy(b->data + b->len, src, n); + b->len += n; +} + +static void +dynbuf_append_zero(dynbuf * b, size_t n) +{ + dynbuf_reserve(b, b->len + n); + memset(b->data + b->len, 0, n); + b->len += n; +} + +static void +dynbuf_reset(dynbuf * b) +{ + b->len = 0; +} + +/* dynamic array of u64 */ +typedef struct u64buf +{ + uint64_t *data; + size_t len; + size_t cap; +} u64buf; + +static void +u64buf_push(u64buf * b, uint64_t v) +{ + if (b->len + 1 > b->cap) + { + size_t ncap = b->cap ? b->cap * 2 : 16; + size_t bytes = ncap * sizeof(uint64_t); + + b->data = b->data + ? repalloc_huge(b->data, bytes) + : MemoryContextAllocExtended(CurrentMemoryContext, bytes, MCXT_ALLOC_HUGE); + b->cap = ncap; + } + b->data[b->len++] = v; +} + +static void +u64buf_reset(u64buf * b) +{ + b->len = 0; +} + +/* + * Insert column layout decided at begin_insert from the column's chc_type. + * Each typed append_* appends into one of three storage groups: + * - body / body_offs: fixed-width or string values for the top-level + * column or, when an array context is open, the inner items. + * - arr_offs: cumulative outer ends for Array(*) columns. + * - nulls: one byte per top-level row for Nullable(*). + */ +typedef enum +{ + IC_FIXED, + IC_STRING, + IC_LC_STRING, + IC_ARRAY_FIXED, + IC_ARRAY_STRING, + IC_ARRAY_NESTED_FIXED, + IC_ARRAY_NESTED_STRING, + IC_JSON_STRING, +} ic_layout; + +static inline bool +ic_layout_array_fixed(ic_layout l) +{ + return l == IC_ARRAY_FIXED || l == IC_ARRAY_NESTED_FIXED; +} + +static inline bool +ic_layout_array_string(ic_layout l) +{ + return l == IC_ARRAY_STRING || l == IC_ARRAY_NESTED_STRING; +} + +typedef struct ic_col +{ + const chc_type *t; /* full column type (incl. Nullable wrapper) */ + const chc_type *inner_t; /* possibly unwrapped Nullable */ + const chc_type *elem_t; /* Array element type, with Nullable + * unwrapped */ + ic_layout layout; + bool is_nullable; + int array_depth; /* current open begin/end nesting */ + int ndim; /* >=1 for array layouts; 1 for top-level + * Array */ + bool array_inner_is_string; + size_t elem_size; /* fixed elem size (top-level FIXED or array + * element FIXED) */ + uint32_t dt64_precision; + dynbuf body; /* row-aligned for FIXED, byte-flat for + * STRING/LC */ + u64buf body_offs; /* offsets per row for STRING/LC inner */ + dynbuf nulls; /* per top-level row */ + u64buf arr_offs; /* cumulative ends for outermost array layer + * (or only layer when ndim==1) */ + u64buf *arr_offs_inner; /* for ndim>1: ndim-1 inner-layer offsets, + * [0]=just inside outer, [ndim-2]=adjacent to + * leaves */ + size_t n_rows; /* committed top-level rows */ + + /* + * Cached column info exposed to callers. info.type borrows from the + * initial_block's chc_type tree, unwrapping Nullable + outer LC. + */ + ch_binary_column_info info; +} ic_col; + +struct ch_binary_insert_handle +{ + MemoryContext cxt; + chc_client *client; + struct ch_binary_state *state; /* parent connection; used to flag broken + * state on error */ + chc_block *initial_block; /* schema source (server's empty Data) */ + size_t ncols; + ic_col *cols; + bool array_active; + size_t array_col_idx; + bool started; + bool finalized; /* finalize_insert has run (success or raised) */ +}; + +static void +classify_column(ic_col * ic, const chc_type * t) +{ + ic->t = t; + if (chc_type_kind(t) == CHC_NULLABLE) + { + ic->is_nullable = true; + t = chc_type_child(t, 0); + } + ic->inner_t = t; + + chc_kind k = chc_type_kind(t); + + if (k == CHC_LOW_CARDINALITY) + { + const chc_type *inner = chc_type_child(t, 0); + bool inner_nullable = chc_type_kind(inner) == CHC_NULLABLE; + const chc_type *base = inner_nullable ? chc_type_child(inner, 0) : inner; + + if (chc_type_kind(base) != CHC_STRING) + { + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported LowCardinality variant: %s", + chc_type_name(base, NULL)))); + } + ic->layout = IC_LC_STRING; + ic->elem_t = base; + return; + } + if (k == CHC_ARRAY) + { + /* + * Walk through nested Array layers to the leaf element type so + * Array(Array(...)) maps to a single ic_col with ndim levels. + */ + const chc_type *base = t; + int ndim = 0; + + while (chc_type_kind(base) == CHC_ARRAY) + { + ndim++; + base = chc_type_child(base, 0); + } + bool elem_nullable = chc_type_kind(base) == CHC_NULLABLE; + + /* Array(Nullable(T)) not supported yet. */ + if (elem_nullable) + { + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: %s not currently supported", + chc_type_name(base, NULL)))); + } + chc_kind ek = chc_type_kind(base); + + ic->elem_t = base; + ic->ndim = ndim; + if (ek == CHC_STRING || ek == CHC_FIXED_STRING) + { + ic->layout = ndim == 1 ? IC_ARRAY_STRING : IC_ARRAY_NESTED_STRING; + ic->array_inner_is_string = true; + } + else if (chc_type_elem_size(base) > 0) + { + ic->layout = ndim == 1 ? IC_ARRAY_FIXED : IC_ARRAY_NESTED_FIXED; + ic->elem_size = chc_type_elem_size(base); + } + else + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported Array element type: %s", chc_type_name(base, NULL)))); + if (ndim > 1) + ic->arr_offs_inner = palloc0((size_t) (ndim - 1) * sizeof(u64buf)); + return; + } + if (k == CHC_STRING) + { + ic->layout = IC_STRING; + return; + } + if (k == CHC_JSON) + { + ic->layout = IC_JSON_STRING; + return; + } + size_t es = chc_type_elem_size(t); + + if (es > 0) + { + ic->layout = IC_FIXED; + ic->elem_size = es; + if (k == CHC_DATETIME64) + ic->dt64_precision = (uint32_t) chc_type_datetime64_scale(t); + return; + } + if (k == CHC_FIXED_STRING) + { + ic->layout = IC_FIXED; + ic->elem_size = (size_t) chc_type_fixed_size(t); + return; + } + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: could not prepare insert - unsupported column type: %s", + chc_type_name(t, NULL)))); +} + +static void +recv_initial_block(struct ch_binary_state *s, ch_binary_insert_handle * h) +{ + for (;;) + { + chc_packet pkt = {}; + chc_err err = {}; + int rc = chc_client_recv_packet(s->client, &pkt, &err); + + if (rc != CHC_OK) + { + s->broken = true; + raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); + } + if (pkt.kind == CHC_PKT_EXCEPTION) + { + const char *msg = "server exception"; + + if (pkt.exception && pkt.exception->display_text) + msg = pkt.exception->display_text; + else if (pkt.exception && pkt.exception->name) + msg = pkt.exception->name; + char *msg_copy = pstrdup(msg); + + s->broken = true; + chc_packet_clear(s->client, &pkt); + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: could not prepare insert - %s", msg_copy))); + } + if (pkt.kind == CHC_PKT_DATA && pkt.block && + chc_block_n_columns(pkt.block) > 0) + { + h->initial_block = pkt.block; + pkt.block = NULL; + chc_packet_clear(s->client, &pkt); + return; + } + chc_packet_clear(s->client, &pkt); + } +} + +/* + * After classify_column ereports the server is mid-INSERT awaiting our + * Data; send empty Data + drain so the connection stays usable. + */ +static void +drain_aborted_insert(struct ch_binary_state *s) +{ + chc_err ce = {}; + + (void) chc_client_send_data(s->client, NULL, &ce); + for (;;) + { + chc_packet drain = {}; + + ce = (chc_err) + { + 0 + }; + int drc = chc_client_recv_packet(s->client, &drain, &ce); + bool eos = (drc == CHC_OK && + (drain.kind == CHC_PKT_END_OF_STREAM || + drain.kind == CHC_PKT_EXCEPTION)); + + chc_packet_clear(s->client, &drain); + if (drc != CHC_OK || eos) + break; + } +} + +ch_binary_insert_handle * +ch_binary_begin_insert(ch_binary_connection_t * conn, const ch_query * query, + ch_binary_column_info * *out_cols, size_t * out_n) +{ + struct ch_binary_state *s = conn_state(conn); + + /* + * Parent h's cxt to the connection's cxt, not CurrentMemoryContext. The + * caller registers a reset callback on a sibling context that drains the + * insert via end_insert(h); if h lived under that sibling, MemoryContext + * tree teardown would free h before the callback fired. + */ + MemoryContext cxt = AllocSetContextCreate(s->cxt, + "pg_clickhouse binary insert", + ALLOCSET_DEFAULT_SIZES); + MemoryContext old = MemoryContextSwitchTo(cxt); + ch_binary_insert_handle *h; + volatile bool need_drain = false; + + PG_TRY(); + { + h = palloc0(sizeof(*h)); + h->cxt = cxt; + h->client = s->client; + h->state = s; + + /* Append " VALUES" so server enters insert mode. */ + size_t sql_len = strlen(query->sql); + char *sql = palloc(sql_len + 8); + + memcpy(sql, query->sql, sql_len); + memcpy(sql + sql_len, " VALUES", 7); + sql[sql_len + 7] = '\0'; + + /* + * On servers that support it (24.10+), tell server to serialize any + * JSON columns using STRING wire format. INSERT path doesn't need + * this, server reads the per-column version prefix the builder + * writes, but we set it on the same packet for symmetry with the + * SELECT path and so any RETURNING-style projection on top still + * decodes. + */ + chc_query_setting json_setting = { + .name = "output_format_native_write_json_as_string", + .value = "1", + .important = true, + }; + chc_query_opts insert_opts = {}; + const chc_query_opts *opts_ptr = NULL; + + if (server_supports_json_as_string(s->client)) + { + insert_opts.settings = &json_setting; + insert_opts.n_settings = 1; + opts_ptr = &insert_opts; + } + + chc_err err = {}; + int rc = chc_client_send_query_ex(s->client, sql, sql_len + 7, opts_ptr, &err); + + if (rc != CHC_OK) + { + s->broken = true; + raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); + } + + recv_initial_block(s, h); + + /* + * Server is now waiting our Data; failures past this point need an + * empty-Data + drain so the connection stays usable. + */ + need_drain = true; + h->started = true; + + size_t nc = chc_block_n_columns(h->initial_block); + + h->ncols = nc; + h->cols = palloc0(nc * sizeof(ic_col)); + for (size_t i = 0; i < nc; i++) + { + ic_col *c = &h->cols[i]; + const chc_type *ct = chc_block_column_type(h->initial_block, i); + size_t nlen; + const char *nm; + + classify_column(c, ct); + nm = chc_block_column_name(h->initial_block, i, &nlen); + c->info.name = pnstrdup(nm ? nm : "", nlen); + c->info.is_nullable = c->is_nullable; + + /* + * inner_t already unwrapped Nullable; unwrap LowCardinality and + * perhaps its inner Nullable to expose innermost type. + */ + const chc_type *vt = c->inner_t; + + if (chc_type_kind(vt) == CHC_LOW_CARDINALITY) + { + vt = chc_type_child(vt, 0); + if (chc_type_kind(vt) == CHC_NULLABLE) + vt = chc_type_child(vt, 0); + } + c->info.type = vt; + } + + *out_cols = NULL; + if (nc) + { + ch_binary_column_info *arr = palloc(nc * sizeof(*arr)); + + for (size_t i = 0; i < nc; i++) + arr[i] = h->cols[i].info; + *out_cols = arr; + } + *out_n = nc; + } + PG_CATCH(); + { + if (need_drain) + drain_aborted_insert(s); + MemoryContextSwitchTo(old); + MemoryContextDelete(cxt); + PG_RE_THROW(); + } + PG_END_TRY(); + + MemoryContextSwitchTo(old); + return h; +} + +/* + * Resolve the storage to receive a value for column `col_idx`, accounting + * for an active array context. Returns the ic_col* whose body buffer is + * the target. For Nullable wrappers, records the null bit. ereports on + * NULL-into-NOT-NULL. + */ +static ic_col * +resolve_col(ch_binary_insert_handle * h, size_t col_idx, bool isnull) +{ + ic_col *c = h->array_active ? &h->cols[h->array_col_idx] : &h->cols[col_idx]; + + if (isnull && !h->array_active && !c->is_nullable) + { + size_t tnlen; + const char *tname = chc_type_name(c->t, &tnlen); + + ereport(ERROR, + (errcode(ERRCODE_NOT_NULL_VIOLATION), + errmsg("pg_clickhouse: cannot append NULL to NOT NULL %.*s column", + (int) tnlen, tname ? tname : "?"))); + } + if (!h->array_active && c->is_nullable) + { + uint8_t b = isnull ? 1 : 0; + + dynbuf_append(&c->nulls, &b, 1); + } + return c; +} + +static void +append_fixed_bytes(ic_col * c, const void *p, size_t n) +{ + dynbuf_append(&c->body, p, n); +} + +static void +append_fixed_zero(ic_col * c, size_t n) +{ + dynbuf_append_zero(&c->body, n); +} + +static void +append_string_row(ic_col * c, const void *p, size_t n) +{ + if (n) + dynbuf_append(&c->body, p, n); + u64buf_push(&c->body_offs, c->body.len); +} + +/* + * Convert decimal text "[-]digits[.frac]" into a ClickHouse Decimal wire value: + * two's-complement signed integer in `width` LE bytes (4/8/16/32 for + * Decimal32/64/128/256), with `scale` fractional digits folded into the value. + */ +static void +decimal_text_to_bytes(const char *s, uint32_t scale, size_t width, uint8_t * out) +{ + bool neg = false; + + if (!s) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("pg_clickhouse: decimal parse failure"))); + if (*s == '-') + { + neg = true; + s++; + } + else if (*s == '+') + s++; + + /* find offsets, going to iterate digits, skipping non-digits */ + const char *dot = strchr(s, '.'); + size_t ilen = dot ? (size_t) (dot - s) : strlen(s); + const char *frac = dot ? dot + 1 : ""; + size_t flen = strlen(frac); + size_t ndig = ilen + scale; + + uint32_t mag[8] = {}; + size_t nwords = width / 4; + + /* accumulate digits (padded/truncated to scale) into mag */ + for (size_t i = 0; i < ndig; i++) + { + char c = i < ilen ? s[i] + : i - ilen < flen ? frac[i - ilen] + : '0'; + uint64_t carry = (uint64_t) (c - '0'); + + for (size_t b = 0; b < nwords; b++) + { + uint64_t v = (uint64_t) mag[b] * 10 + carry; + + mag[b] = (uint32_t) v; + carry = v >> 32; + } + } + /* two's-complement negation */ + if (neg) + { + for (size_t b = 0; b < nwords; b++) + mag[b] = ~mag[b]; + uint64_t carry = 1; + + for (size_t b = 0; b < nwords && carry; b++) + { + uint64_t v = (uint64_t) mag[b] + carry; + + mag[b] = (uint32_t) v; + carry = v >> 32; + } + } + + memcpy(out, mag, width); +} + +static void +append_int_kind(ic_col * c, int64_t val) +{ + chc_kind k = ic_layout_array_fixed(c->layout) + ? chc_type_kind(c->elem_t) + : chc_type_kind(c->inner_t); + + switch (k) + { + case CHC_INT8: + case CHC_UINT8: + case CHC_BOOL: + { + int8_t v = (int8_t) val; + + append_fixed_bytes(c, &v, 1); + return; + } + case CHC_INT16: + case CHC_UINT16: + { + int16_t v = (int16_t) val; + + append_fixed_bytes(c, &v, 2); + return; + } + case CHC_INT32: + case CHC_UINT32: + { + int32_t v = (int32_t) val; + + append_fixed_bytes(c, &v, 4); + return; + } + case CHC_INT64: + case CHC_UINT64: + { + append_fixed_bytes(c, &val, 8); + return; + } + default: + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: int value into non-integer column"))); + } +} + +void +ch_binary_append_int(ch_binary_insert_handle * h, size_t col, int64_t val, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + + if (isnull) + append_fixed_zero(c, c->elem_size); + else + append_int_kind(c, val); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_uint(ch_binary_insert_handle * h, size_t col, uint64_t val, bool isnull) +{ + ch_binary_append_int(h, col, (int64_t) val, isnull); +} + +void +ch_binary_append_bool(ch_binary_insert_handle * h, size_t col, bool val, bool isnull) +{ + ch_binary_append_int(h, col, val, isnull); +} + +void +ch_binary_append_double(ch_binary_insert_handle * h, size_t col, double val, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + + if (isnull) + append_fixed_zero(c, 8); + else + append_fixed_bytes(c, &val, 8); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_float(ch_binary_insert_handle * h, size_t col, float val, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + + if (isnull) + append_fixed_zero(c, 4); + else + append_fixed_bytes(c, &val, 4); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_bytes(ch_binary_insert_handle * h, size_t col, const void *p, + size_t n, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + chc_kind k = ic_layout_array_string(c->layout) + ? chc_type_kind(c->elem_t) + : chc_type_kind(c->inner_t); + + if (c->layout == IC_LC_STRING) + { + /* Accumulate via body+body_offs; dict materialized at flush. */ + if (isnull) + append_string_row(c, NULL, 0); + else + append_string_row(c, p, n); + MemoryContextSwitchTo(old); + return; + } + + if (k == CHC_FIXED_STRING) + { + size_t w = (size_t) chc_type_fixed_size(ic_layout_array_string(c->layout) ? c->elem_t : c->inner_t); + + if (isnull) + append_fixed_zero(c, w); + else + { + size_t take = n < w ? n : w; + + if (take) + dynbuf_append(&c->body, p, take); + if (take < w) + dynbuf_append_zero(&c->body, w - take); + } + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_ENUM8 || k == CHC_ENUM16) + { + size_t ew = (k == CHC_ENUM8) ? 1 : 2; + + if (isnull) + { + append_fixed_zero(c, ew); + MemoryContextSwitchTo(old); + return; + } + const chc_type *et = ic_layout_array_string(c->layout) ? c->elem_t : c->inner_t; + size_t nenum = chc_type_enum_count(et); + int64_t val = 0; + bool found = false; + + for (size_t i = 0; i < nenum; i++) + { + const char *en; + size_t el; + int64_t ev; + + chc_type_enum_at(et, i, &en, &el, &ev); + if (el == n && memcmp(en, p, n) == 0) + { + val = ev; + found = true; + break; + } + } + if (!found) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("pg_clickhouse: enum value '%.*s' not found", + (int) n, (const char *) p))); + if (k == CHC_ENUM8) + { + int8_t v = (int8_t) val; + + append_fixed_bytes(c, &v, 1); + } + else + { + int16_t v = (int16_t) val; + + append_fixed_bytes(c, &v, 2); + } + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_STRING || k == CHC_JSON || k == CHC_OBJECT) + { + if (isnull) + append_string_row(c, NULL, 0); + else + append_string_row(c, p, n); + MemoryContextSwitchTo(old); + return; + } + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: bytes into non-text column"))); +} + +void +ch_binary_append_decimal(ch_binary_insert_handle * h, size_t col, + const char *digits, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + size_t w; + + switch (k) + { + case CHC_DECIMAL32: + w = 4; + break; + case CHC_DECIMAL64: + w = 8; + break; + case CHC_DECIMAL128: + w = 16; + break; + case CHC_DECIMAL256: + w = 32; + break; + default: + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: decimal into non-decimal column"))); + } + uint8_t raw[32] = {}; + + if (!isnull && digits) + { + uint32_t scale = (uint32_t) chc_type_decimal_scale(t); + + decimal_text_to_bytes(digits, scale, w, raw); + } + append_fixed_bytes(c, raw, w); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_uuid(ch_binary_insert_handle * h, size_t col, + const uint8_t bytes[16], bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + uint8_t wire[16] = {}; + + if (!isnull) + { + uint64_t a, + b; + + memcpy(&a, bytes, 8); + memcpy(&b, bytes + 8, 8); + a = pg_ntoh64(a); + b = pg_ntoh64(b); + memcpy(wire, &a, 8); + memcpy(wire + 8, &b, 8); + } + append_fixed_bytes(c, wire, 16); + MemoryContextSwitchTo(old); +} + +/* + * addr_be is BE bytes (PG inet ip_addr layout). For IPv4 CH wire is a + * host-order uint32; pg_ntoh32 turns BE bytes into the right host value. + * For IPv6 CH wire matches PG byte order. + */ +void +ch_binary_append_inet(ch_binary_insert_handle * h, size_t col, + const uint8_t * addr_be, size_t addrlen, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + + if (k == CHC_IPV4 && addrlen == 4) + { + uint32_t addr = 0; + + if (!isnull && addr_be) + { + uint32_t be; + + memcpy(&be, addr_be, 4); + addr = pg_ntoh32(be); + } + append_fixed_bytes(c, &addr, 4); + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_IPV6 && addrlen == 16) + { + uint8_t raw[16] = {}; + + if (!isnull && addr_be) + memcpy(raw, addr_be, 16); + append_fixed_bytes(c, raw, 16); + MemoryContextSwitchTo(old); + return; + } + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: cannot insert inet into non-inet column"))); +} + +void +ch_binary_append_date_seconds(ch_binary_insert_handle * h, size_t col, + int64_t seconds, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + + if (k == CHC_DATE) + { + uint16_t days = isnull ? 0 : (uint16_t) (seconds / 86400); + + append_fixed_bytes(c, &days, 2); + } + else if (k == CHC_DATE32) + { + int32_t days = isnull ? 0 : (int32_t) (seconds / 86400); + + append_fixed_bytes(c, &days, 4); + } + else + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: date into non-date column"))); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_datetime_seconds(ch_binary_insert_handle * h, size_t col, + int64_t seconds, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + uint32_t v = isnull ? 0 : (uint32_t) seconds; + + append_fixed_bytes(c, &v, 4); + MemoryContextSwitchTo(old); +} + +void +ch_binary_append_datetime64_raw(ch_binary_insert_handle * h, size_t col, + int64_t raw, bool isnull) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col *c = resolve_col(h, col, isnull); + int64_t v = isnull ? 0 : raw; + + append_fixed_bytes(c, &v, 8); + MemoryContextSwitchTo(old); +} + +void +ch_binary_array_begin(ch_binary_insert_handle * h, size_t col) +{ + /* + * Nested arrays recurse via append_one with col=0, so once an array is + * open the caller's col is meaningless, resolve from array_col_idx. + */ + size_t idx = h->array_active ? h->array_col_idx : col; + + if (idx >= h->ncols) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: col out of range"))); + ic_col *c = &h->cols[idx]; + + if (c->layout != IC_ARRAY_FIXED && c->layout != IC_ARRAY_STRING + && c->layout != IC_ARRAY_NESTED_FIXED && c->layout != IC_ARRAY_NESTED_STRING) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: column is not Array"))); + if (c->array_depth >= c->ndim) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: nesting exceeds column ndim"))); + if (c->array_depth == 0) + { + if (c->is_nullable) + { + uint8_t b = 0; + MemoryContext old = MemoryContextSwitchTo(h->cxt); + + dynbuf_append(&c->nulls, &b, 1); + MemoryContextSwitchTo(old); + } + h->array_active = true; + h->array_col_idx = idx; + } + c->array_depth++; +} + +void +ch_binary_array_end(ch_binary_insert_handle * h) +{ + if (!h->array_active) + return; + ic_col *c = &h->cols[h->array_col_idx]; + + if (c->array_depth == 0) + return; + + uint64_t end; + + /* + * At innermost depth, count is leaf-element count; at outer levels the + * count is total children written so far at the level below. + */ + if (c->array_depth == c->ndim) + { + if (c->layout == IC_ARRAY_FIXED || c->layout == IC_ARRAY_NESTED_FIXED) + end = c->elem_size ? (uint64_t) (c->body.len / c->elem_size) : 0; + else + end = c->body_offs.len; + } + else + end = c->arr_offs_inner[c->array_depth - 1].len; + + MemoryContext old = MemoryContextSwitchTo(h->cxt); + + if (c->array_depth == 1) + { + u64buf_push(&c->arr_offs, end); + c->n_rows++; + h->array_active = false; + } + else + u64buf_push(&c->arr_offs_inner[c->array_depth - 2], end); + MemoryContextSwitchTo(old); + c->array_depth--; +} + +bool +ch_binary_array_active(const ch_binary_insert_handle * h) +{ + return h && h->array_active; +} + +chc_kind +ch_binary_column_kind(const ch_binary_insert_handle * h, size_t col) +{ + if (col >= h->ncols) + return CHC_VOID; + const ic_col *c = &h->cols[col]; + + if (h->array_active) + { + c = &h->cols[h->array_col_idx]; + + /* + * While nested, surface CHC_ARRAY until the innermost layer is open; + * at that point return the leaf kind so encode targets scalars. + */ + if (c->array_depth < c->ndim) + return CHC_ARRAY; + return chc_type_kind(c->elem_t); + } + if (c->layout == IC_ARRAY_FIXED || c->layout == IC_ARRAY_STRING + || c->layout == IC_ARRAY_NESTED_FIXED || c->layout == IC_ARRAY_NESTED_STRING) + return CHC_ARRAY; + if (c->layout == IC_LC_STRING) + return CHC_STRING; /* PG side targets TEXT */ + return chc_type_kind(c->inner_t); +} + +uint32_t +ch_binary_column_datetime64_precision(const ch_binary_insert_handle * h, size_t col) +{ + if (col >= h->ncols) + return 0; + return h->cols[col].dt64_precision; +} + +/* Dedup map for LowCardinality dict */ +typedef struct lcd_key +{ + const uint8_t *bytes; + size_t len; +} lcd_key; + +typedef struct lcd_entry +{ + uint32 status; + lcd_key key; + uint32 idx; +} lcd_entry; + +#define SH_PREFIX lcd +#define SH_ELEMENT_TYPE lcd_entry +#define SH_KEY_TYPE lcd_key +#define SH_KEY key +#define SH_HASH_KEY(tb, key) hash_bytes((key).bytes, (int) (key).len) +#define SH_EQUAL(tb, a, b) \ + ((a).len == (b).len && memcmp((a).bytes, (b).bytes, (a).len) == 0) +#define SH_SCOPE static inline +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" + +/* Build LC dict (collect unique strings in insertion order) */ +static void +build_lc_dict(ic_col * c, bool nullable, + uint64_t * *out_dict_offs, uint8_t * *out_dict_data, + size_t * out_dict_n, void **out_keys, int *out_key_size, + size_t * out_n_rows) +{ + size_t n_rows = c->body_offs.len; + uint64_t *dict_offs = NULL; + uint8_t *dict_data = NULL; + uint32_t *keys = n_rows ? palloc(n_rows * sizeof(uint32_t)) : NULL; + size_t dict_n = 0; + size_t dict_cap = 0; + size_t data_len = 0; + lcd_hash *ht = n_rows + ? lcd_create(CurrentMemoryContext, + (uint32) Min(n_rows, (size_t) PG_UINT32_MAX), NULL) + : NULL; + + if (nullable) + { + /* dict[0] = "" sentinel. */ + dict_cap = 8; + dict_offs = palloc(dict_cap * sizeof(uint64_t)); + dict_offs[0] = 0; + dict_n = 1; + } + + for (size_t i = 0; i < n_rows; i++) + { + uint64_t start = i == 0 ? 0 : c->body_offs.data[i - 1]; + uint64_t end = c->body_offs.data[i]; + size_t len = (size_t) (end - start); + const uint8_t *bytes = c->body.data + start; + lcd_key k = {bytes, len}; + lcd_entry *entry; + bool found; + + /* If nullable and this row was null (signalled by null bit), key 0. */ + if (nullable && c->nulls.data[i]) + { + keys[i] = 0; + continue; + } + + entry = lcd_insert(ht, k, &found); + if (found) + { + keys[i] = entry->idx; + continue; + } + + if (dict_n == dict_cap) + { + dict_cap = dict_cap ? dict_cap * 2 : 64; + dict_offs = dict_offs + ? repalloc(dict_offs, dict_cap * sizeof(uint64_t)) + : palloc(dict_cap * sizeof(uint64_t)); + } + data_len += len; + dict_offs[dict_n] = data_len; + entry->idx = (uint32) dict_n; + keys[i] = (uint32) dict_n; + dict_n++; + } + + if (data_len) + { + lcd_iterator it; + lcd_entry *e; + + dict_data = MemoryContextAllocExtended(CurrentMemoryContext, data_len, + MCXT_ALLOC_HUGE); + lcd_start_iterate(ht, &it); + while ((e = lcd_iterate(ht, &it)) != NULL) + { + uint64_t s = e->idx == 0 ? 0 : dict_offs[e->idx - 1]; + + memcpy(dict_data + s, e->key.bytes, e->key.len); + } + } + if (ht) + lcd_destroy(ht); + *out_dict_offs = dict_offs; + *out_dict_data = dict_data; + *out_dict_n = dict_n; + *out_keys = keys; + *out_key_size = 4; + *out_n_rows = n_rows; +} + +void +ch_binary_flush_block(ch_binary_insert_handle * h) +{ + MemoryContext old = MemoryContextSwitchTo(h->cxt); + chc_err err = {}; + chc_block_builder *bb = NULL; + int rc = chc_block_builder_init(&bb, &pg_chc_alloc, &err); + + if (rc != CHC_OK) + raise_chc(&err, ERRCODE_FDW_ERROR, "could not append data to column - "); + + for (size_t i = 0; i < h->ncols; i++) + { + ic_col *c = &h->cols[i]; + const char *name = c->info.name ? c->info.name : ""; + size_t nlen = strlen(name); + + switch (c->layout) + { + case IC_FIXED: + { + size_t n_rows = c->elem_size ? c->body.len / c->elem_size : 0; + + if (c->is_nullable) + rc = chc_block_builder_append_nullable_fixed(bb, name, nlen, c->t, + c->nulls.data, c->body.data, + n_rows, &err); + else + rc = chc_block_builder_append_fixed(bb, name, nlen, c->t, + c->body.data, n_rows, &err); + break; + } + case IC_STRING: + { + size_t n_rows = c->body_offs.len; + + if (c->is_nullable) + rc = chc_block_builder_append_nullable_string(bb, name, nlen, c->t, + c->nulls.data, + c->body_offs.data, + c->body.data, + n_rows, &err); + else + rc = chc_block_builder_append_string(bb, name, nlen, + c->body_offs.data, + c->body.data, + n_rows, &err); + break; + } + case IC_JSON_STRING: + { + /* + * JSON columns share IC_STRING's per-row accumulator; the + * library emits the 8-byte SerializationObject::STRING + * prefix and writes rows as String-Binary. + */ + size_t n_rows = c->body_offs.len; + + rc = chc_block_builder_append_json_string(bb, name, nlen, c->t, + c->body_offs.data, + c->body.data, + n_rows, &err); + break; + } + case IC_LC_STRING: + { + bool nullable_inner = chc_type_kind(chc_type_child(c->inner_t, 0)) == CHC_NULLABLE; + size_t dict_n, + n_rows; + int key_size; + uint64_t *lc_offs; + uint8_t *lc_data; + void *lc_keys; + + build_lc_dict(c, nullable_inner, &lc_offs, &lc_data, + &dict_n, &lc_keys, &key_size, &n_rows); + rc = chc_block_builder_append_low_cardinality_string(bb, name, nlen, c->t, + key_size, lc_keys, + lc_offs, lc_data, + dict_n, n_rows, &err); + break; + } + case IC_ARRAY_FIXED: + { + size_t n_rows = c->arr_offs.len; + + rc = chc_block_builder_append_array_fixed(bb, name, nlen, c->t, + c->arr_offs.data, + c->body.data, + n_rows, &err); + break; + } + case IC_ARRAY_STRING: + { + size_t n_rows = c->arr_offs.len; + + rc = chc_block_builder_append_array_string(bb, name, nlen, c->t, + c->arr_offs.data, + c->body_offs.data, + c->body.data, + n_rows, &err); + break; + } + case IC_ARRAY_NESTED_FIXED: + case IC_ARRAY_NESTED_STRING: + { + size_t n_rows = c->arr_offs.len; + const uint64_t **lvl_offsets = palloc((size_t) c->ndim * sizeof(*lvl_offsets)); + size_t *lvl_lens = palloc((size_t) c->ndim * sizeof(*lvl_lens)); + + lvl_offsets[0] = c->arr_offs.data; + lvl_lens[0] = c->arr_offs.len; + for (int lvl = 1; lvl < c->ndim; lvl++) + { + lvl_offsets[lvl] = c->arr_offs_inner[lvl - 1].data; + lvl_lens[lvl] = c->arr_offs_inner[lvl - 1].len; + } + if (c->layout == IC_ARRAY_NESTED_FIXED) + rc = chc_block_builder_append_array_nested_fixed( + bb, name, nlen, c->t, c->ndim, + lvl_offsets, lvl_lens, + c->body.data, n_rows, &err); + else + rc = chc_block_builder_append_array_nested_string( + bb, name, nlen, c->t, c->ndim, + lvl_offsets, lvl_lens, + c->body_offs.data, c->body.data, + n_rows, &err); + break; + } + } + if (rc != CHC_OK) + raise_chc(&err, ERRCODE_FDW_ERROR, + "could not append data to column - "); + } + + rc = chc_client_send_data(h->client, bb, &err); + chc_block_builder_destroy(bb); + if (rc != CHC_OK) + { + if (h->state) + h->state->broken = true; + raise_chc(&err, ERRCODE_FDW_ERROR, "could not insert columns - "); + } + + /* Reset per-column buffers for next batch. */ + for (size_t i = 0; i < h->ncols; i++) + { + ic_col *c = &h->cols[i]; + + dynbuf_reset(&c->body); + dynbuf_reset(&c->nulls); + u64buf_reset(&c->body_offs); + u64buf_reset(&c->arr_offs); + if (c->arr_offs_inner) + { + for (int lvl = 0; lvl + 1 < c->ndim; lvl++) + u64buf_reset(&c->arr_offs_inner[lvl]); + } + c->n_rows = 0; + } + MemoryContextSwitchTo(old); +} + +/* + * Send final empty Data + drain. May ereport on server exception or + * transport failure. Idempotent via h->finalized. Leaves h->cxt alive; + * ch_binary_release_insert deletes it. + */ +void +ch_binary_finalize_insert(ch_binary_insert_handle * h) +{ + if (!h || h->finalized) + return; + + /* + * Set early so an ereport(ERROR) below still leaves h in the "do not + * touch the wire" state for the release callback. + */ + h->finalized = true; + + if (!(h->started && h->client)) + return; + + MemoryContext old = MemoryContextSwitchTo(h->cxt); + char *exc_msg = NULL; + bool broke = false; + chc_err err = {}; + int rc = chc_client_send_data(h->client, NULL, &err); + + if (rc != CHC_OK) + { + broke = true; + exc_msg = pstrdup(err.msg[0] ? err.msg : "send_data failed"); + } + else + { + /* Drain until EOS or exception. */ + for (;;) + { + chc_packet pkt = {}; + + err = (chc_err) + { + 0 + }; + rc = chc_client_recv_packet(h->client, &pkt, &err); + if (rc != CHC_OK) + { + broke = true; + exc_msg = pstrdup(err.msg[0] ? err.msg : "recv_packet failed"); + chc_packet_clear(h->client, &pkt); + break; + } + if (pkt.kind == CHC_PKT_EXCEPTION) + { + const char *msg = "server exception"; + + if (pkt.exception && pkt.exception->display_text) + msg = pkt.exception->display_text; + else if (pkt.exception && pkt.exception->name) + msg = pkt.exception->name; + exc_msg = pstrdup(msg); + broke = true; + chc_packet_clear(h->client, &pkt); + break; + } + chc_packet_clear(h->client, &pkt); + if (pkt.kind == CHC_PKT_END_OF_STREAM) + break; + } + } + + /* + * Server raised mid-INSERT and typically closes the socket; the next op + * would EPIPE. Mark broken so the cache rebuilds. + */ + if (broke && h->state) + h->state->broken = true; + + MemoryContextSwitchTo(old); + + if (exc_msg) + { + /* exc_msg lives in h->cxt; copy into parent before raising. */ + char *parent_msg = pstrdup(exc_msg); + + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: could not finish INSERT - %s", parent_msg))); + } +} + +/* + * Teardown counterpart to finalize. Safe from a MemoryContext reset + * callback: never raises, never talks to the server. If finalize did not + * run (mid-query abort), flags the connection broken so it rebuilds on + * next use. + */ +void +ch_binary_release_insert(ch_binary_insert_handle * h) +{ + if (!h) + return; + + if (!h->finalized && h->started && h->state) + h->state->broken = true; + + MemoryContextDelete(h->cxt); +} diff --git a/src/binary/select.c b/src/binary/select.c new file mode 100644 index 00000000..78386438 --- /dev/null +++ b/src/binary/select.c @@ -0,0 +1,462 @@ +/* + * select.c + * + * Submit a SELECT (chc_client_send_query_ex) and pump Data packets one block + * at a time. decode.c pulls the next block via + * ch_binary_response_fetch_next_block when it exhausts the current one, so + * peak memory is bounded by one block plus what postgres holds for the + * current row. Settings come from the foreign-table KV list; we also add + * output_format_native_write_json_as_string=1 against servers that understand + * it. Cancel polling drives chc_io's per-read callback; server-side + * exceptions flag the connection as broken so the cache drops it. Premature + * close (eg LIMIT, error in decode, transaction abort) sends Cancel + * + drains in ch_binary_response_free so the connection is reusable. + */ + +#include "postgres.h" + +#include +#include +#include + +#include "utils/memutils.h" +#include "utils/palloc.h" + +#include "binary_internal.h" +#include "kv_list.h" + +/* Wall-clock cap on drain_until_eos. A well-behaved server acknowledges + * Cancel by flushing the in-flight block + EndOfStream within a second. */ +#define DRAIN_DEADLINE_US (5 * 1000 * 1000) + +struct ch_binary_response_t +{ + MemoryContext cxt; + struct ch_binary_state *state; /* parent connection state */ + chc_client *client; /* recv_packet target */ + bool (*check_cancel) (void); + + char *error; /* NULL on success */ + bool success; + bool eos; /* END_OF_STREAM or exception seen */ + + size_t columns_count; + chc_block *staged; /* next block to hand out; owned */ + chc_block *prev; /* last block handed out; freed at next fetch */ +}; + +static void +resp_set_error(ch_binary_response_t * resp, const char *msg) +{ + if (resp->error) + return; + resp->error = pstrdup(msg && *msg ? msg : "?"); +} + +static void +resp_set_exception(ch_binary_response_t * resp, const chc_exception * ex) +{ + if (resp->error) + return; + const char *msg = NULL; + + if (ex) + { + if (ex->display_text && ex->display_text[0]) + msg = ex->display_text; + else if (ex->name && ex->name[0]) + msg = ex->name; + } + resp->error = pstrdup(msg ? msg : "server exception"); +} + +/* + * Read one wire packet under resp->cxt and fold it into the response. Sets + * resp->staged (for non-empty Data), resp->columns_count (on first Data with + * columns), resp->error (cancel/exception/transport), or resp->eos + * (END_OF_STREAM / exception / transport failure). Other packet kinds + * (progress, log, profile, ...) are silently consumed. + */ +static void +pump_one(ch_binary_response_t * resp) +{ + MemoryContext old = MemoryContextSwitchTo(resp->cxt); + chc_packet pkt = {}; + chc_err err = {}; + int rc = chc_client_recv_packet(resp->client, &pkt, &err); + + if (rc != CHC_OK) + { + resp_set_error(resp, err.msg); + resp->state->broken = true; + resp->eos = true; + MemoryContextSwitchTo(old); + return; + } + + if (resp->check_cancel && resp->check_cancel()) + resp_set_error(resp, "query was canceled"); + + switch (pkt.kind) + { + case CHC_PKT_DATA: + if (pkt.block && chc_block_n_columns(pkt.block) > 0) + { + if (resp->columns_count == 0) + resp->columns_count = chc_block_n_columns(pkt.block); + if (chc_block_n_rows(pkt.block) > 0 && resp->staged == NULL) + { + size_t ncols = chc_block_n_columns(pkt.block); + chc_err verr = {}; + int vrc = CHC_OK; + + for (size_t i = 0; i < ncols; i++) + { + vrc = chc_column_validate(chc_block_column(pkt.block, i), &verr); + if (vrc != CHC_OK) + break; + } + if (vrc != CHC_OK) + { + resp_set_error(resp, verr.msg); + resp->state->broken = true; + resp->eos = true; + } + else + { + resp->staged = pkt.block; + pkt.block = NULL; + } + } + } + chc_packet_clear(resp->client, &pkt); + break; + + case CHC_PKT_EXCEPTION: + resp_set_exception(resp, pkt.exception); + chc_packet_clear(resp->client, &pkt); + + /* + * Older servers (and some protocol states) close the socket after + * raising an exception, so reusing this connection for a + * follow-up query risks EPIPE. Match the legacy C++ driver (which + * always called Client::ResetConnection) and treat the connection + * as broken. + */ + resp->state->broken = true; + resp->eos = true; + break; + + case CHC_PKT_END_OF_STREAM: + chc_packet_clear(resp->client, &pkt); + resp->eos = true; + break; + + default: + chc_packet_clear(resp->client, &pkt); + break; + } + + MemoryContextSwitchTo(old); +} + +static int64_t +drain_now_us(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return (int64_t) ts.tv_sec * 1000000 + ts.tv_nsec / 1000; +} + +static void +drain_set_deadline(struct ch_binary_state *s, int64_t deadline_us) +{ + if (s->tls) + chc_openssl_io_set_deadline(&s->openssl_state, deadline_us); + else + chc_posix_io_set_deadline(&s->posix_state, deadline_us); +} + +/* + * Best-effort drain of any unconsumed stream so the connection is left clean + * for the next query. Disables cancel polling while draining so + * QueryCancelPending doesn't short-circuit every refill, and caps total wait + * with an IO deadline so a peer that ignores Cancel doesn't cause hang. Sets + * state->broken on transport failure / timeout / send_cancel failure so the + * cache drops the connection. + */ +static void +drain_until_eos(ch_binary_response_t * resp) +{ + if (resp->eos) + return; + + MemoryContext old = MemoryContextSwitchTo(resp->cxt); + chc_err ce = {}; + + if (chc_client_send_cancel(resp->client, &ce) != CHC_OK) + { + resp->state->broken = true; + resp->eos = true; + MemoryContextSwitchTo(old); + return; + } + resp->state->check_cancel_fn = NULL; + resp->check_cancel = NULL; + + drain_set_deadline(resp->state, drain_now_us() + DRAIN_DEADLINE_US); + + for (;;) + { + chc_packet pkt = {}; + int rc; + + ce = (chc_err) + { + 0 + }; + rc = chc_client_recv_packet(resp->client, &pkt, &ce); + + if (rc != CHC_OK) + { + resp->state->broken = true; + resp->eos = true; + break; + } + bool stop = pkt.kind == CHC_PKT_END_OF_STREAM || + pkt.kind == CHC_PKT_EXCEPTION; + + chc_packet_clear(resp->client, &pkt); + if (stop) + { + resp->eos = true; + break; + } + } + + drain_set_deadline(resp->state, 0); + MemoryContextSwitchTo(old); +} + +ch_binary_response_t * +ch_binary_simple_query(ch_binary_connection_t * conn, const ch_query * query, + bool (*check_cancel) (void)) +{ + struct ch_binary_state *s = conn_state(conn); + MemoryContext cxt = AllocSetContextCreate(CurrentMemoryContext, + "pg_clickhouse binary response", + ALLOCSET_DEFAULT_SIZES); + MemoryContext old = MemoryContextSwitchTo(cxt); + ch_binary_response_t *resp = palloc0(sizeof(*resp)); + + resp->cxt = cxt; + resp->state = s; + resp->client = s->client; + resp->check_cancel = check_cancel; + s->check_cancel_fn = check_cancel; + + size_t n_user_settings = 0; + size_t n_params = (size_t) (query->num_params > 0 ? query->num_params : 0); + chc_query_setting *settings = NULL; + chc_query_param *params = NULL; + bool want_json_as_string = server_supports_json_as_string(s->client); + + { + const kv_list *kv = query->settings; + + if (kv) + n_user_settings = (size_t) kv->length; + } + size_t n_settings = n_user_settings + (want_json_as_string ? 1 : 0); + + if (n_settings) + { + settings = palloc0(n_settings * sizeof(*settings)); + size_t i = 0; + + for (kv_iter it = new_kv_iter(query->settings); !kv_iter_done(&it); kv_iter_next(&it), i++) + { + settings[i].name = it.name; + settings[i].value = it.value; + settings[i].important = true; + } + if (want_json_as_string) + { + settings[i].name = "output_format_native_write_json_as_string"; + settings[i].value = "1"; + settings[i].important = true; + } + } + if (n_params) + { + params = palloc0(n_params * sizeof(*params)); + for (size_t i = 0; i < n_params; i++) + { + char nm[32]; + + snprintf(nm, sizeof(nm), "p%zu", i + 1); + params[i].name = pstrdup(nm); + + /* + * Quote & escape the value the way clickhouse-cpp's + * WriteQuotedString did: wrap in single quotes, replace inner + * specials with backslash-escapes the server's + * Field::restoreFromDump understands. Without escaping inner + * quotes the server stops parsing at the first `'` inside the + * value, which breaks Array(String) parameters whose CH literal + * already contains quoted elements. + */ + const char *raw = query->param_values[i]; + + if (raw) + { + size_t rlen = strlen(raw); + size_t cap = rlen * 4 + 3; + char *dst = palloc(cap); + size_t o = 0; + + dst[o++] = '\''; + for (size_t j = 0; j < rlen; j++) + { + unsigned char ch = (unsigned char) raw[j]; + + switch (ch) + { + case '\0': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '0'; + dst[o++] = '0'; + break; + case '\b': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '0'; + dst[o++] = '8'; + break; + case '\t': + dst[o++] = '\\'; + dst[o++] = 't'; + break; + case '\n': + dst[o++] = '\\'; + dst[o++] = 'n'; + break; + case '\'': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '2'; + dst[o++] = '7'; + break; + case '\\': + dst[o++] = '\\'; + dst[o++] = '\\'; + break; + default: + dst[o++] = (char) ch; + } + } + dst[o++] = '\''; + dst[o] = '\0'; + params[i].value = dst; + } + else + params[i].value = "'\\N'"; + } + } + + chc_query_opts opts = { + .settings = settings, + .n_settings = n_settings, + .params = params, + .n_params = n_params, + }; + chc_err err = {}; + int rc = chc_client_send_query_ex(s->client, query->sql, + strlen(query->sql), &opts, &err); + + if (rc != CHC_OK) + { + resp_set_error(resp, err.msg); + s->broken = true; + resp->eos = true; + goto done; + } + + /* + * Pump until schema is known so callers can call + * ch_binary_response_columns immediately. Empty result sets exit on eos + * with columns_count still set from the header Data block. + */ + while (resp->columns_count == 0 && !resp->eos && !resp->error) + pump_one(resp); + +done: + resp->success = (resp->error == NULL); + MemoryContextSwitchTo(old); + return resp; +} + +void +ch_binary_response_free(ch_binary_response_t * resp) +{ + if (!resp) + return; + + /* Avoid raising from a MemoryContextResetCallback. */ + PG_TRY(); + { + drain_until_eos(resp); + } + PG_CATCH(); + { + FlushErrorState(); + resp->state->broken = true; + } + PG_END_TRY(); + + resp->state->check_cancel_fn = NULL; + MemoryContextDelete(resp->cxt); +} + +const char * +ch_binary_response_error(const ch_binary_response_t * resp) +{ + return resp ? resp->error : NULL; +} + +bool +ch_binary_response_success(const ch_binary_response_t * resp) +{ + return resp && resp->success; +} + +size_t +ch_binary_response_columns(const ch_binary_response_t * resp) +{ + return resp ? resp->columns_count : 0; +} + +const chc_block * +ch_binary_response_fetch_next_block(ch_binary_response_t * resp) +{ + if (!resp) + return NULL; + + if (resp->prev) + { + chc_block_destroy(resp->prev, &pg_chc_alloc); + resp->prev = NULL; + } + + while (resp->staged == NULL && !resp->eos && !resp->error) + pump_one(resp); + + if (resp->staged == NULL) + return NULL; + + resp->prev = resp->staged; + resp->staged = NULL; + return resp->prev; +} diff --git a/src/connection.c b/src/connection.c index 86959382..2032431f 100644 --- a/src/connection.c +++ b/src/connection.c @@ -123,10 +123,20 @@ chfdw_get_connection(UserMapping * user) } /* - * We don't check the health of cached connection here, because it would - * require some overhead. Broken connection will be detected when the - * connection is actually used. + * Drop connections that hit an unrecoverable protocol/IO error on the + * previous statement (server raised mid-INSERT and closed the socket, + * write hit EPIPE, etc). Without this, a subsequent statement would write + * to the dead socket and surface a useless "Broken pipe" instead of the + * real error from the next request. */ + if (entry->gate.conn != NULL && + entry->gate.methods->is_broken != NULL && + entry->gate.methods->is_broken(entry->gate.conn)) + { + elog(LOG, "closing broken pg_clickhouse connection"); + entry->gate.methods->disconnect(entry->gate.conn); + entry->gate.conn = NULL; + } /* * If cache entry doesn't have a connection, we have to establish a new diff --git a/src/fdw.c b/src/fdw.c index 20ca5ed6..5a798b88 100644 --- a/src/fdw.c +++ b/src/fdw.c @@ -2,16 +2,18 @@ A PostgreSQL extension for connecting to ClickHouse servers. */ +#include + /* PostgreSQL includes. */ #include "postgres.h" #include "catalog/pg_class_d.h" #include "commands/defrem.h" -#include "commands/explain.h" #include "foreign/fdwapi.h" #include "funcapi.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#include "nodes/parsenodes.h" #include "optimizer/clauses.h" #include "optimizer/cost.h" #include "optimizer/pathnode.h" @@ -32,12 +34,12 @@ #if PG_VERSION_NUM >= 180000 #include "commands/explain_format.h" #include "commands/explain_state.h" +#else +#include "commands/explain.h" #endif /* extension includes. */ #include "utils/builtins.h" -#include "binary.hh" -#include "internal.h" #include "fdw.h" #include "version.h" @@ -1420,6 +1422,15 @@ clickhouseEndForeignInsert(EState * estate, /* flush */ oldcontext = MemoryContextSwitchTo(fmstate->temp_cxt); fmstate->conn.methods->insert_tuple(fmstate->state, NULL); + + /* + * Finalize on the happy path so the binary driver can ereport on a + * server-side INSERT exception without raising from inside the + * MemoryContext reset callback that fires during abort. + */ + if (fmstate->conn.methods->finalize_insert) + fmstate->conn.methods->finalize_insert(fmstate->state); + MemoryContextSwitchTo(oldcontext); MemoryContextReset(fmstate->temp_cxt); diff --git a/src/include/binary.h b/src/include/binary.h new file mode 100644 index 00000000..3ca9b7bd --- /dev/null +++ b/src/include/binary.h @@ -0,0 +1,116 @@ +/* + * binary.h + * + * API exposed by the binary driver. Allocates against caller's + * CurrentMemoryContext at every public entry point and ereports on + * error. Returned objects own a dedicated MemoryContext; explicit + * *_free / *_close calls do MemoryContextDelete. + */ + +#ifndef CLICKHOUSE_BINARY_H +#define CLICKHOUSE_BINARY_H + +#include "postgres.h" + +#include +#include +#include + +#include "access/tupdesc.h" +#include "executor/tuptable.h" +#include "utils/palloc.h" + +#include "clickhouse.h" +#include "engine.h" + +typedef struct ch_binary_connection_t ch_binary_connection_t; +typedef struct ch_binary_response_t ch_binary_response_t; +typedef struct ch_binary_insert_handle ch_binary_insert_handle; + +/* Connection. */ +extern ch_binary_connection_t * ch_binary_connect(ch_connection_details * details); +extern void ch_binary_close(ch_binary_connection_t * conn); + +/* + * Returns true if connection encountered an unrecoverable error + * (server exception, IO failure, mid-protocol break). Callers should + * drop cached connection instead of reusing it. + */ +extern bool ch_binary_is_broken(const ch_binary_connection_t * conn); + +/* SELECT. */ +extern ch_binary_response_t * ch_binary_simple_query(ch_binary_connection_t * conn, + const ch_query * query, + bool (*check_cancel) (void)); +extern void ch_binary_response_free(ch_binary_response_t * resp); +extern const char *ch_binary_response_error(const ch_binary_response_t * resp); +extern bool ch_binary_response_success(const ch_binary_response_t * resp); +extern size_t ch_binary_response_columns(const ch_binary_response_t * resp); + +/* INSERT. */ + +/* + * Finish the insert: send final empty Data, drain the response, ereport + * if the server raised. Idempotent; call exactly once from the FDW happy + * path before tearing down the handle. + */ +extern void ch_binary_finalize_insert(ch_binary_insert_handle * h); + +/* PG-typed surface follows. */ + +typedef struct +{ + ch_binary_response_t *resp; + Oid *coltypes; + Datum *values; + bool *nulls; + + size_t block; /* current block */ + size_t row; /* row in current block */ + const chc_block *cur; /* borrowed from resp; NULL when unloaded */ + void *gc; /* allocated objects while reading */ + char *error; + bool done; +} ch_binary_read_state_t; + +typedef struct +{ + MemoryContext memcxt; /* used for cleanup */ + MemoryContextCallback callback; + + TupleDesc outdesc; + ch_binary_insert_handle *insert_block; + size_t len; + void *conversion_states; + char *table_name; + Oid relid; /* foreign table relid, for column_name + * lookups */ + + Datum *values; + bool *nulls; + bool success; + + ch_binary_connection_t *conn; +} ch_binary_insert_state; + +/* SELECT helpers (decode.c). */ +extern void ch_binary_read_state_init(ch_binary_read_state_t * state, ch_binary_response_t * resp); +extern void ch_binary_read_state_free(ch_binary_read_state_t * state); +extern bool ch_binary_read_row(ch_binary_read_state_t * state); + +/* SELECT/INSERT type conversion (convert.c). */ +extern Datum ch_binary_convert_datum(void *state, Datum val); +extern void *ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype); +extern void ch_binary_free_convert_state(void *state); + +/* INSERT helpers (encode.c). */ +extern void ch_binary_prepare_insert(void *conn, const ch_query * query, + ch_binary_insert_state * state); +extern void ch_binary_insert_columns(ch_binary_insert_state * state); +extern void ch_binary_column_append_data(ch_binary_insert_state * state, size_t colidx); +extern void *ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid); +extern void ch_binary_insert_state_free(void *c); +extern void ch_binary_do_output_conversion(ch_binary_insert_state * state, + TupleTableSlot * slot); + +#endif /* CLICKHOUSE_BINARY_H */ diff --git a/src/include/binary.hh b/src/include/binary.hh deleted file mode 100644 index e3ce0d91..00000000 --- a/src/include/binary.hh +++ /dev/null @@ -1,110 +0,0 @@ -#ifndef CLICKHOUSE_BINARY_H -#define CLICKHOUSE_BINARY_H - -#include "engine.h" - -#ifdef __cplusplus -extern "C" -{ -#endif - - typedef struct ch_binary_connection_t ch_binary_connection_t; - typedef struct ch_insert_block_h ch_insert_block_h; - typedef struct ch_binary_response_t - { - void *values; - size_t columns_count; - size_t blocks_count; - char *error; - bool success; - } ch_binary_response_t; - - typedef struct - { - ch_binary_response_t *resp; - Oid *coltypes; - Datum *values; - bool *nulls; - - size_t block; /* current block */ - size_t row; /* row in current block */ - void *gc; /* allocated objects while reading */ - char *error; - bool done; - } ch_binary_read_state_t; - - typedef struct - { - Datum *datums; - bool *nulls; - size_t len; - Oid *types; - } ch_binary_tuple_t; - - /* - * Holds an array read from ClickHouse. For nested arrays - * (Array(Array(...))) ndim > 1 and datums[i] points to a child - * ch_binary_array_t with ndim-1. item_type is the leaf scalar type, - * array_type is the postgres array type (same at every level since - * postgres uses one array type per element type regardless of - * dimensionality). - */ - typedef struct - { - Datum *datums; - bool *nulls; - size_t len; - int ndim; /* nesting depth, ≥1 (used on selects) */ - Oid item_type; /* leaf element type (used on selects) */ - Oid array_type; /* used on selects */ - } ch_binary_array_t; - - typedef struct - { - MemoryContext memcxt; /* used for cleanup */ - MemoryContextCallback callback; - - TupleDesc outdesc; - ch_insert_block_h *insert_block; /* clickhouse::Block */ - size_t len; - void *conversion_states; - char *table_name; - Oid relid; /* foreign table relid, for column_name - * lookups */ - - Datum *values; - bool *nulls; - bool success; - - ch_binary_connection_t *conn; - } ch_binary_insert_state; - - extern ch_binary_connection_t * ch_binary_connect(ch_connection_details * details, char **error); - extern void ch_binary_close(ch_binary_connection_t * conn); - extern ch_binary_response_t * ch_binary_simple_query(ch_binary_connection_t * conn, - const ch_query * query, bool (*check_cancel) (void)); - extern void ch_binary_response_free(ch_binary_response_t * resp); - -/* reading */ - void ch_binary_read_state_init(ch_binary_read_state_t * state, ch_binary_response_t * resp, const ch_query * query); - void ch_binary_read_state_free(ch_binary_read_state_t * state); - bool ch_binary_read_row(ch_binary_read_state_t * state, TupleDesc tupdesc, List * attrs); - Datum ch_binary_convert_datum(void *state, Datum val); - void *ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype); - void ch_binary_free_convert_state(void *); - -/* insertion */ - void ch_binary_prepare_insert(void *conn, const ch_query * query, - ch_binary_insert_state * state); - void ch_binary_insert_columns(ch_binary_insert_state * state); - void ch_binary_column_append_data(ch_binary_insert_state * state, size_t colidx); - void *ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid); - void ch_binary_insert_state_free(void *c); - void ch_binary_do_output_conversion(ch_binary_insert_state * insert_state, - TupleTableSlot * slot); - -#ifdef __cplusplus -} -#endif - -#endif /* CLICKHOUSE_BINARY_H */ diff --git a/src/include/fdw.h b/src/include/fdw.h index 5fecfc3d..4b711658 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -75,9 +75,11 @@ typedef Datum * (*cursor_fetch_row_method) (ChFdwScanRowContext * ctx); typedef void *(*prepare_insert_method) (void *conn, ResultRelInfo *, List *, const ch_query *, char *); typedef void (*insert_tuple_method) (void *state, TupleTableSlot * slot); +typedef void (*finalize_insert_method) (void *state); typedef ch_cursor * (*streaming_query_method) (void *conn, const ch_query * query, int32 fetch_size); +typedef bool (*is_broken_method) (const void *conn); typedef struct { @@ -86,8 +88,11 @@ typedef struct cursor_fetch_row_method fetch_row; prepare_insert_method prepare_insert; insert_tuple_method insert_tuple; + finalize_insert_method finalize_insert; /* NULL if no explicit finalize + * step needed */ streaming_query_method streaming_query; /* NULL if not supported */ cursor_fetch_row_method streaming_fetch_row; /* NULL if not supported */ + is_broken_method is_broken; /* NULL means connection is never broken */ } libclickhouse_methods; typedef struct diff --git a/src/option.c b/src/option.c index 2ef946b0..f70c9b15 100644 --- a/src/option.c +++ b/src/option.c @@ -303,7 +303,14 @@ chfdw_extract_options(List * defelems, char **driver, char **host, int *port, if (host && strcmp(def->defname, "host") == 0) *host = defGetString(def); else if (port && strcmp(def->defname, "port") == 0) - *port = atoi(defGetString(def)); + { + char *endptr = NULL; + long val = strtol(defGetString(def), &endptr, 10); + + /* Just ignore invalid values. */ + if (val <= INT_MAX && val > 0 && (!*endptr || *endptr == 0)) + *port = val; + } else if (username && strcmp(def->defname, "user") == 0) *username = defGetString(def); else if (password && strcmp(def->defname, "password") == 0) diff --git a/src/pglink.c b/src/pglink.c index 274098d9..56a04c24 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -19,7 +19,7 @@ #include "fdw.h" #include "http.h" #include "http_streaming.h" -#include "binary.hh" +#include "binary.h" #include #include @@ -58,10 +58,12 @@ static libclickhouse_methods http_methods = static void binary_disconnect(void *conn); static ch_cursor * binary_simple_query(void *conn, const ch_query * query); static void binary_cursor_free(void *cursor); +static bool binary_is_broken(const void *conn); /* static void binary_simple_insert(void *conn, const char *query); */ static Datum * binary_fetch_row(ChFdwScanRowContext * ctx); static void binary_insert_tuple(void *, TupleTableSlot * slot); +static void binary_finalize_insert(void *istate); static void *binary_prepare_insert(void *, ResultRelInfo *, List *, const ch_query * query, char *table_name); static char *ch_escape_string(const char *s, size_t len); @@ -76,8 +78,10 @@ static libclickhouse_methods binary_methods = .fetch_row = binary_fetch_row, .prepare_insert = binary_prepare_insert, .insert_tuple = binary_insert_tuple, + .finalize_insert = binary_finalize_insert, .streaming_query = NULL, .streaming_fetch_row = NULL, + .is_broken = binary_is_broken, }; static int @@ -818,31 +822,9 @@ http_insert_tuple(void *istate, TupleTableSlot * slot) ch_connection chfdw_binary_connect(ch_connection_details * details) { - char *ch_error = NULL; ch_connection res; - ch_binary_connection_t *conn = ch_binary_connect(details, &ch_error); - if (conn == NULL) - { - if (ch_error == NULL) - { - ereport(ERROR, - (errcode(ERRCODE_FDW_OUT_OF_MEMORY), - errmsg("out of memory"))); - } - else - { - char *error = pstrdup(ch_error); - - free(ch_error); - - ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: connection error: %s", error))); - } - } - - res.conn = conn; + res.conn = ch_binary_connect(details); res.methods = &binary_methods; res.is_binary = true; return res; @@ -855,6 +837,12 @@ binary_disconnect(void *conn) ch_binary_close((ch_binary_connection_t *) conn); } +static bool +binary_is_broken(const void *conn) +{ + return ch_binary_is_broken((const ch_binary_connection_t *) conn); +} + static ch_cursor * binary_simple_query(void *conn, const ch_query * query) { @@ -865,9 +853,9 @@ binary_simple_query(void *conn, const ch_query * query) ch_binary_response_t *resp = ch_binary_simple_query(conn, query, &is_canceled); - if (!resp->success) + if (!ch_binary_response_success(resp)) { - char *error = pstrdup(resp->error); + char *error = pstrdup(ch_binary_response_error(resp)); ch_binary_response_free(resp); ereport(ERROR, ( @@ -887,10 +875,33 @@ binary_simple_query(void *conn, const ch_query * query) state = (ch_binary_read_state_t *) palloc0(sizeof(ch_binary_read_state_t)); cursor->query = pstrdup(query->sql); cursor->read_state = state; - cursor->columns_count = resp->columns_count; - ch_binary_read_state_init(cursor->read_state, resp, query); + cursor->columns_count = ch_binary_response_columns(resp); + ch_binary_read_state_init(cursor->read_state, resp); cursor->conversion_states = palloc0(sizeof(uintptr_t) * cursor->columns_count); + /* + * CH JSON columns default to JSONBOID in state->coltypes. When foreign + * table column is declared `json` (JSONOID), override so + * binary_make_datum returns json Datum from CH's STRING bytes, skipping + * jsonb_in / jsonb_out round-trip that would reformat CH's emit and break + * expected outputs that pin CH's exact formatting. + */ + if (query->tupdesc && state->coltypes) + { + ListCell *lc; + size_t j = 0; + + foreach(lc, query->attr_nums) + { + int i = lfirst_int(lc); + + if (state->coltypes[j] == JSONBOID && + TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) + state->coltypes[j] = JSONOID; + j++; + } + } + cursor->memcxt = tempcxt; cursor->callback.func = binary_cursor_free; cursor->callback.arg = cursor; @@ -898,12 +909,10 @@ binary_simple_query(void *conn, const ch_query * query) MemoryContextSwitchTo(oldcxt); if (state->error) - { ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: could not initialize read state: %s", - state->error))); - } + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", state->error), + errdetail_internal("Remote Query: %.64000s", query->sql))); return cursor; } @@ -923,6 +932,14 @@ binary_simple_query(void *conn, const ch_query * query) * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, * which only cares about text. */ +static void +binary_fetch_row_errcb(void *arg) +{ + const char *sql = (const char *) arg; + + errdetail_internal("Remote Query: %.64000s", sql); +} + static Datum * binary_fetch_row(ChFdwScanRowContext * ctx) { @@ -933,7 +950,14 @@ binary_fetch_row(ChFdwScanRowContext * ctx) Datum *values = ctx->values; bool *nulls = ctx->nulls; ch_binary_read_state_t *state = cursor->read_state; - bool have_data = ch_binary_read_row(state, tupdesc, attrs); + ErrorContextCallback errcallback; + + errcallback.callback = binary_fetch_row_errcb; + errcallback.arg = (void *) cursor->query; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + bool have_data = ch_binary_read_row(state); size_t attcount = list_length(attrs); if (state->error) @@ -943,11 +967,14 @@ binary_fetch_row(ChFdwScanRowContext * ctx) state->error))); if (!have_data) + { + error_context_stack = errcallback.previous; return NULL; + } if (attcount == 0) { - if (state->resp->columns_count == 1 && state->nulls[0]) + if (ch_binary_response_columns(state->resp) == 1 && state->nulls[0]) { nulls[0] = true; goto ok; @@ -958,14 +985,12 @@ binary_fetch_row(ChFdwScanRowContext * ctx) errmsg("pg_clickhouse: unexpected state: attributes " "count == 0 and haven't got NULL in the response"))); } - else if (attcount != state->resp->columns_count) + else if (attcount != ch_binary_response_columns(state->resp)) { ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal("pg_clickhouse: columns mismatch"), - errdetail("Number of returned columns (%lu) does not match " - "expected column count (%lu).", - state->resp->columns_count, attcount))); + errmsg_internal("pg_clickhouse: returned %lu columns, expected %lu", + ch_binary_response_columns(state->resp), attcount))); } if (tupdesc) @@ -1039,6 +1064,7 @@ binary_fetch_row(ChFdwScanRowContext * ctx) } ok: + error_context_stack = errcallback.previous; return state->values; } @@ -1111,10 +1137,10 @@ binary_insert_tuple(void *istate, TupleTableSlot * slot) /* * Set up conversion states so that Postgres Datums of types * defined by the foreign table can be converted to a well-known - * Postgres type that column_append() in binary.cpp knows how to - * convert to the appropriate ClickHouse type. Binary compatible - * types don't convert; others require a CAST. Fails if there is - * no cast. + * Postgres type that the binary INSERT path knows how to convert + * to the appropriate ClickHouse type. Binary compatible types + * don't convert; others require a CAST. Fails if there is no + * cast. */ state->conversion_states = ch_binary_make_tuple_map(slot->tts_tupleDescriptor, state->outdesc, state->relid); MemoryContextSwitchTo(old_mcxt); @@ -1131,6 +1157,15 @@ binary_insert_tuple(void *istate, TupleTableSlot * slot) } } +static void +binary_finalize_insert(void *istate) +{ + ch_binary_insert_state *state = istate; + + if (state && state->insert_block) + ch_binary_finalize_insert(state->insert_block); +} + /* * Query to generate table for doc/pg_clickhouse.md. Keep in sync with * str_types_map below. On change, re-run and paste the output into @@ -1325,16 +1360,36 @@ chfdw_construct_create_tables(ImportForeignSchemaStmt * stmt, ForeignServer * se NULL }; + /* + * Drain the outer query into private strings before opening the per-table + * column queries: both use the same connection, and binary streaming only + * permits one in-flight response at a time. + */ + List *tables = NIL; + while ((row_values = conn.methods->fetch_row(&tables_ctx)) != NULL) { + List *triple = list_make3(pstrdup(TextDatumGetCString(row_values[0])), + pstrdup(TextDatumGetCString(row_values[1])), + pstrdup(TextDatumGetCString(row_values[2]))); + + CHECK_FOR_INTERRUPTS(); + tables = lappend(tables, triple); + } + MemoryContextDelete(cursor->memcxt); + + ListCell *tlc; + + foreach(tlc, tables) + { + List *triple = (List *) lfirst(tlc); + char *table_name = (char *) linitial(triple); + char *engine = (char *) lsecond(triple); + char *engine_full = (char *) lthird(triple); StringInfoData buf; - char *table_name = TextDatumGetCString(row_values[0]); - char *engine = TextDatumGetCString(row_values[1]); - char *engine_full = TextDatumGetCString(row_values[2]); Datum *dvalues; bool first = true; - CHECK_FOR_INTERRUPTS(); if (table_name == NULL) continue; @@ -1454,7 +1509,6 @@ chfdw_construct_create_tables(ImportForeignSchemaStmt * stmt, ForeignServer * se MemoryContextDelete(cols_ctx.cursor->memcxt); } - MemoryContextDelete(cursor->memcxt); return result; } diff --git a/test/expected/binary.out b/test/expected/binary.out index d73583dc..4185be2b 100644 --- a/test/expected/binary.out +++ b/test/expected/binary.out @@ -309,6 +309,7 @@ SELECT * FROM farrays ORDER BY c1; SELECT * FROM farrays2 ORDER BY c1; ERROR: pg_clickhouse: could not cast value from integer[] to bigint[] +DETAIL: Remote Query: SELECT c1, c2 FROM binary_test.arrays ORDER BY c1 ASC NULLS LAST -- nested arrays SELECT * FROM fnested_arrays ORDER BY c1; c1 | c2 | c3 @@ -319,7 +320,7 @@ SELECT * FROM fnested_arrays ORDER BY c1; SELECT * FROM fragged_arrays ORDER BY c1; ERROR: malformed array literal: "{{"1","2","3"},{"4"}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +DETAIL: Remote Query: SELECT c1, c2 FROM binary_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- tuples SELECT * FROM ftuples ORDER BY c1; c1 | c2 | c3 diff --git a/test/expected/binary_inserts.out b/test/expected/binary_inserts.out index 9dd46567..5620558f 100644 --- a/test/expected/binary_inserts.out +++ b/test/expected/binary_inserts.out @@ -298,7 +298,7 @@ SELECT * FROM nested_arrays ORDER BY c1; /* shape mismatch with column type must error rather than silently corrupt */ INSERT INTO nested_arrays VALUES (4, ARRAY[1,2,3], ARRAY['x']); -ERROR: pg_clickhouse: could not append data to column - pg_clickhouse: insert array has fewer dimensions than column type +ERROR: pg_clickhouse: unexpected PG/CH type pair for column 0 /* Check UUIDs and IPs */ \d addr Foreign table "binary_inserts_test.addr" @@ -490,7 +490,7 @@ INSERT INTO default_vals VALUES( NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); -ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL UInt8 column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL UInt8 column SELECT * FROM default_vals; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c18 | c19 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 ----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- @@ -498,7 +498,7 @@ SELECT * FROM default_vals; -- Test unsupported Nullables. INSERT INTO not_nullable VALUES (1, 'x', 'x', NULL); -ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column DROP USER MAPPING FOR CURRENT_USER SERVER binary_inserts_loopback; SELECT clickhouse_raw_query('DROP DATABASE binary_inserts_test'); clickhouse_raw_query diff --git a/test/expected/binary_inserts_1.out b/test/expected/binary_inserts_1.out index dce76e48..e8b81ed4 100644 --- a/test/expected/binary_inserts_1.out +++ b/test/expected/binary_inserts_1.out @@ -289,7 +289,7 @@ SELECT * FROM nested_arrays ORDER BY c1; /* shape mismatch with column type must error rather than silently corrupt */ INSERT INTO nested_arrays VALUES (4, ARRAY[1,2,3], ARRAY['x']); -ERROR: pg_clickhouse: could not append data to column - pg_clickhouse: insert array has fewer dimensions than column type +ERROR: pg_clickhouse: unexpected PG/CH type pair for column 0 /* Check UUIDs and IPs */ \d addr Foreign table "binary_inserts_test.addr" @@ -481,7 +481,7 @@ INSERT INTO default_vals VALUES( NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); -ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL UInt8 column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL UInt8 column SELECT * FROM default_vals; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c18 | c19 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 ----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- @@ -489,7 +489,7 @@ SELECT * FROM default_vals; -- Test unsupported Nullables. INSERT INTO not_nullable VALUES (1, 'x', 'x', NULL); -ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column DROP USER MAPPING FOR CURRENT_USER SERVER binary_inserts_loopback; SELECT clickhouse_raw_query('DROP DATABASE binary_inserts_test'); clickhouse_raw_query diff --git a/test/expected/binary_queries_5.out b/test/expected/binary_queries_5.out index fa3e305e..8243455e 100644 --- a/test/expected/binary_queries_5.out +++ b/test/expected/binary_queries_5.out @@ -124,7 +124,7 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: DB::Exception: Database `no such database` does not exist +ERROR: pg_clickhouse: connection error: DB::Exception: Database `no such database` doesn't exist ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail ERROR: pg_clickhouse: connection error: DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. diff --git a/test/expected/binary_queries_6.out b/test/expected/binary_queries_6.out deleted file mode 100644 index 7a365603..00000000 --- a/test/expected/binary_queries_6.out +++ /dev/null @@ -1,747 +0,0 @@ -SET datestyle = 'ISO'; -CREATE SERVER binary_queries_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_queries_test', driver 'binary'); -CREATE SERVER binary_queries_loopback2 FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_queries_test', driver 'binary'); -CREATE USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback; -CREATE USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback2; -SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t1 - (c1 Int, c2 Int, c3 String, c4 Date, c5 Date, c6 String, c7 String, c8 String) - ENGINE = MergeTree PARTITION BY c4 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t2 (c1 Int, c2 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t3 (c1 Int, c3 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t4 (c1 Int, c2 Int, c3 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -CREATE SCHEMA binary_queries_test; -SET search_path = binary_queries_test, public; -CREATE FOREIGN TABLE ft1 ( - c0 int, - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER binary_queries_loopback OPTIONS (table_name 't1'); -ALTER FOREIGN TABLE ft1 DROP COLUMN c0; -CREATE FOREIGN TABLE ft2 ( - c1 int NOT NULL, - c2 text NOT NULL -) SERVER binary_queries_loopback OPTIONS (table_name 't2'); -CREATE FOREIGN TABLE ft4 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback OPTIONS (table_name 't4'); -CREATE FOREIGN TABLE ft5 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback OPTIONS (table_name 't4'); -CREATE FOREIGN TABLE ft6 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback2 OPTIONS (table_name 't4'); -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t1 - SELECT number, - number % 10, - toString(number), - toDate('1990-01-01'), - toDate('1990-01-01'), - number % 10, - number % 10, - 'foo' - FROM numbers(1, 110);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t2 - SELECT number, - concat('AAA', toString(number)) - FROM numbers(1, 100);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t4 - SELECT number, - number + 1, - concat('AAA', toString(number)) - FROM numbers(1, 100);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -\set VERBOSITY terse -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work - c3 | c4 -----+------------ - 1 | 1990-01-01 -(1 row) - -ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'no such database'); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: DB::Exception: Database `no such database` doesn't exist -ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (ADD user 'no such user'); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: connection error: DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. -ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'binary_queries_test'); -ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (DROP user); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work again - c3 | c4 -----+------------ - 1 | 1990-01-01 -(1 row) - -\set VERBOSITY default -ANALYZE ft1; -EXPLAIN (COSTS OFF) SELECT * FROM ft1 ORDER BY c1 OFFSET 100 LIMIT 10; - QUERY PLAN ---------------------- - Foreign Scan on ft1 -(1 row) - -SELECT * FROM ft1 ORDER BY c1 OFFSET 100 LIMIT 10; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 ORDER BY t1.c1, t1.tableoid OFFSET 100 LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------------------------------- - Limit - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - -> Sort - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - Sort Key: t1.c1, t1.tableoid - -> Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 -(8 rows) - -SELECT * FROM ft1 t1 ORDER BY t1.c1, t1.tableoid OFFSET 100 LIMIT 10; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1 FROM ft1 t1 ORDER BY t1.c1 OFFSET 100 LIMIT 10; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: t1.*, c1 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 ORDER BY c1 ASC NULLS LAST LIMIT 10 OFFSET 100 -(3 rows) - -SELECT t1 FROM ft1 t1 ORDER BY t1.c1 OFFSET 100 LIMIT 10; - t1 -------------------------------------------- - (101,1,101,1990-01-01,1990-01-01,1,1,foo) - (102,2,102,1990-01-01,1990-01-01,2,2,foo) - (103,3,103,1990-01-01,1990-01-01,3,3,foo) - (104,4,104,1990-01-01,1990-01-01,4,4,foo) - (105,5,105,1990-01-01,1990-01-01,5,5,foo) - (106,6,106,1990-01-01,1990-01-01,6,6,foo) - (107,7,107,1990-01-01,1990-01-01,7,7,foo) - (108,8,108,1990-01-01,1990-01-01,8,8,foo) - (109,9,109,1990-01-01,1990-01-01,9,9,foo) - (110,0,110,1990-01-01,1990-01-01,0,0,foo) -(10 rows) - -SELECT * FROM ft1 WHERE false; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 101 AND t1.c6 = '1' AND t1.c7 >= '1'; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c7 >= '1')) AND ((c1 = 101)) AND ((c6 = '1')) -(3 rows) - -SELECT COUNT(*) FROM ft1 t1; - count -------- - 110 -(1 row) - -SELECT * FROM ft1 t1 WHERE t1.c3 IN (SELECT c2 FROM ft2 t2 WHERE c1 <= 10) ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -SELECT * FROM ft1 t1 WHERE t1.c3 = (SELECT MAX(c2) FROM ft2 t2) ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -WITH t1 AS (SELECT * FROM ft1 WHERE c1 <= 10) SELECT t2.c1, t2.c2, t2.c2 FROM t1, ft2 t2 WHERE t1.c1 = t2.c1 ORDER BY t1.c1; - c1 | c2 | c2 -----+-------+------- - 1 | AAA1 | AAA1 - 2 | AAA2 | AAA2 - 3 | AAA3 | AAA3 - 4 | AAA4 | AAA4 - 5 | AAA5 | AAA5 - 6 | AAA6 | AAA6 - 7 | AAA7 | AAA7 - 8 | AAA8 | AAA8 - 9 | AAA9 | AAA9 - 10 | AAA10 | AAA10 -(10 rows) - -SELECT 'fixed', NULL FROM ft1 t1 WHERE c1 = 1; - ?column? | ?column? -----------+---------- - fixed | -(1 row) - -SET enable_hashjoin TO false; -SET enable_nestloop TO false; -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c1 FROM ft2 t1 JOIN ft1 t2 ON (t1.c1 = t2.c1) OFFSET 100 LIMIT 10; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: t1.c1, t2.c1 - Relations: (ft2 t1) INNER JOIN (ft1 t2) - Remote SQL: SELECT r1.c1, r2.c1 FROM binary_queries_test.t2 r1 ALL INNER JOIN binary_queries_test.t1 r2 ON (((r1.c1 = r2.c1))) LIMIT 10 OFFSET 100 -(4 rows) - -SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - c1 | c1 -----+---- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) OFFSET 100 LIMIT 10; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: t1.c1, t2.c1 - Relations: (ft2 t1) LEFT JOIN (ft1 t2) - Remote SQL: SELECT r1.c1, r2.c1 FROM binary_queries_test.t2 r1 ALL LEFT JOIN binary_queries_test.t1 r2 ON (((r1.c1 = r2.c1))) LIMIT 10 OFFSET 100 -(4 rows) - -EXPLAIN SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - QUERY PLAN ----------------------------------------------------------------- - Limit (cost=1.00..1.05 rows=10 width=32) - -> Unique (cost=1.00..5.60 rows=1000 width=32) - -> Foreign Scan (cost=1.00..0.60 rows=1000 width=32) - Relations: (ft2 t1) LEFT JOIN (ft1 t2) -(4 rows) - -SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - c1 | c1 -----+---- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -RESET enable_hashjoin; -RESET enable_nestloop; -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 1; -- Var, OpExpr(b), Const - QUERY PLAN --------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = 1)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 100 AND t1.c2 = 0; -- BoolExpr - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = 100)) AND ((c2 = 0)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NULL; -- NullTest - QUERY PLAN ------------------------------------------- - Result - Output: c1, c2, c3, c4, c5, c6, c7, c8 - One-Time Filter: false -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL; -- NullTest - QUERY PLAN ---------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE round(abs(c1), 0) = 1; -- FuncExpr - QUERY PLAN ------------------------------------------------------------------------------------------------------------------ - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((round(abs(c1), 0) = 1)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = -c1; -- OpExpr(l) - QUERY PLAN -------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = (- c1))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE 1 = factorial(c1); -- OpExpr(r) - QUERY PLAN -------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((1 = factorial(c1))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE (c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL); -- DistinctExpr - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE (((c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]); -- ScalarArrayOpExpr - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((has([c2, 1, (c1 + 0)],c1))) -(3 rows) - -SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]) ORDER BY c1; -- ScalarArrayOpExpr - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 1 | 1 | 1 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 2 | 2 | 2 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 3 | 3 | 3 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 4 | 4 | 4 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 5 | 5 | 5 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 6 | 6 | 6 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 7 | 7 | 7 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 8 | 8 | 8 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 9 | 9 | 9 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 10 | 0 | 10 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 11 | 1 | 11 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 12 | 2 | 12 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 13 | 3 | 13 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 14 | 4 | 14 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 15 | 5 | 15 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 16 | 6 | 16 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 17 | 7 | 17 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 18 | 8 | 18 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 19 | 9 | 19 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 20 | 0 | 20 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 21 | 1 | 21 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 22 | 2 | 22 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 23 | 3 | 23 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 24 | 4 | 24 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 25 | 5 | 25 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 26 | 6 | 26 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 27 | 7 | 27 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 28 | 8 | 28 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 29 | 9 | 29 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 30 | 0 | 30 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 31 | 1 | 31 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 32 | 2 | 32 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 33 | 3 | 33 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 34 | 4 | 34 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 35 | 5 | 35 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 36 | 6 | 36 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 37 | 7 | 37 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 38 | 8 | 38 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 39 | 9 | 39 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 40 | 0 | 40 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 41 | 1 | 41 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 42 | 2 | 42 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 43 | 3 | 43 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 44 | 4 | 44 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 45 | 5 | 45 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 46 | 6 | 46 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 47 | 7 | 47 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 48 | 8 | 48 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 49 | 9 | 49 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 50 | 0 | 50 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 51 | 1 | 51 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 52 | 2 | 52 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 53 | 3 | 53 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 54 | 4 | 54 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 55 | 5 | 55 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 56 | 6 | 56 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 57 | 7 | 57 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 58 | 8 | 58 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 59 | 9 | 59 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 60 | 0 | 60 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 61 | 1 | 61 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 62 | 2 | 62 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 63 | 3 | 63 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 64 | 4 | 64 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 65 | 5 | 65 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 66 | 6 | 66 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 67 | 7 | 67 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 68 | 8 | 68 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 69 | 9 | 69 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 70 | 0 | 70 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 71 | 1 | 71 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 72 | 2 | 72 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 73 | 3 | 73 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 74 | 4 | 74 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 75 | 5 | 75 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 76 | 6 | 76 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 77 | 7 | 77 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 78 | 8 | 78 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 79 | 9 | 79 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 80 | 0 | 80 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 81 | 1 | 81 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 82 | 2 | 82 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 83 | 3 | 83 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 84 | 4 | 84 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 85 | 5 | 85 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 86 | 6 | 86 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 87 | 7 | 87 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 88 | 8 | 88 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 89 | 9 | 89 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 90 | 0 | 90 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 91 | 1 | 91 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 92 | 2 | 92 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 93 | 3 | 93 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 94 | 4 | 94 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 95 | 5 | 95 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 96 | 6 | 96 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 97 | 7 | 97 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 98 | 8 | 98 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 99 | 9 | 99 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 100 | 0 | 100 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(110 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1]; -- ArrayRef - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = (([c1, c2, 3])[1]))) -(3 rows) - -SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1] ORDER BY c1; -- ArrayRef - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 1 | 1 | 1 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 2 | 2 | 2 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 3 | 3 | 3 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 4 | 4 | 4 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 5 | 5 | 5 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 6 | 6 | 6 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 7 | 7 | 7 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 8 | 8 | 8 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 9 | 9 | 9 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 10 | 0 | 10 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 11 | 1 | 11 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 12 | 2 | 12 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 13 | 3 | 13 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 14 | 4 | 14 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 15 | 5 | 15 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 16 | 6 | 16 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 17 | 7 | 17 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 18 | 8 | 18 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 19 | 9 | 19 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 20 | 0 | 20 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 21 | 1 | 21 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 22 | 2 | 22 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 23 | 3 | 23 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 24 | 4 | 24 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 25 | 5 | 25 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 26 | 6 | 26 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 27 | 7 | 27 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 28 | 8 | 28 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 29 | 9 | 29 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 30 | 0 | 30 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 31 | 1 | 31 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 32 | 2 | 32 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 33 | 3 | 33 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 34 | 4 | 34 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 35 | 5 | 35 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 36 | 6 | 36 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 37 | 7 | 37 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 38 | 8 | 38 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 39 | 9 | 39 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 40 | 0 | 40 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 41 | 1 | 41 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 42 | 2 | 42 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 43 | 3 | 43 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 44 | 4 | 44 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 45 | 5 | 45 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 46 | 6 | 46 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 47 | 7 | 47 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 48 | 8 | 48 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 49 | 9 | 49 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 50 | 0 | 50 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 51 | 1 | 51 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 52 | 2 | 52 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 53 | 3 | 53 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 54 | 4 | 54 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 55 | 5 | 55 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 56 | 6 | 56 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 57 | 7 | 57 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 58 | 8 | 58 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 59 | 9 | 59 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 60 | 0 | 60 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 61 | 1 | 61 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 62 | 2 | 62 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 63 | 3 | 63 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 64 | 4 | 64 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 65 | 5 | 65 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 66 | 6 | 66 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 67 | 7 | 67 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 68 | 8 | 68 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 69 | 9 | 69 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 70 | 0 | 70 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 71 | 1 | 71 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 72 | 2 | 72 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 73 | 3 | 73 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 74 | 4 | 74 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 75 | 5 | 75 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 76 | 6 | 76 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 77 | 7 | 77 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 78 | 8 | 78 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 79 | 9 | 79 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 80 | 0 | 80 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 81 | 1 | 81 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 82 | 2 | 82 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 83 | 3 | 83 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 84 | 4 | 84 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 85 | 5 | 85 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 86 | 6 | 86 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 87 | 7 | 87 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 88 | 8 | 88 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 89 | 9 | 89 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 90 | 0 | 90 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 91 | 1 | 91 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 92 | 2 | 92 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 93 | 3 | 93 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 94 | 4 | 94 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 95 | 5 | 95 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 96 | 6 | 96 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 97 | 7 | 97 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 98 | 8 | 98 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 99 | 9 | 99 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 100 | 0 | 100 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(110 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c6 = E'foo''s\\bar'; -- check special chars - QUERY PLAN --------------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c6 = 'foo''s\\bar')) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c8 = 'foo'; -- can't be sent to remote - QUERY PLAN ------------------------------------------------------------------------------------------------------- - Foreign Scan on binary_queries_test.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c8 = 'foo')) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT (CASE WHEN c1 < 10 THEN 1 WHEN c1 < 50 THEN 2 ELSE 3 END) a, - sum(length(c2)) FROM ft2 GROUP BY a ORDER BY a; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: (CASE WHEN (c1 < 10) THEN 1 WHEN (c1 < 50) THEN 2 ELSE 3 END), (sum(length(c2))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END, sum(lengthUTF8(c2)) FROM binary_queries_test.t2 GROUP BY (CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END) ORDER BY CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END ASC NULLS LAST -(4 rows) - -SELECT (CASE WHEN c1 < 10 THEN 1 WHEN c1 < 50 THEN 2 ELSE 3 END) a, - sum(length(c2)) FROM ft2 GROUP BY a ORDER BY a; - a | sum ----+----- - 1 | 36 - 2 | 200 - 3 | 256 -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT SUM(c1) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN ------------------------------------------------------------------------------- - Foreign Scan - Output: (sum(c1) FILTER (WHERE (c1 < 20))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT sumIf(c1,(((c1 < 20)) > 0)) FROM binary_queries_test.t2 -(4 rows) - -SELECT SUM(c1) FILTER (WHERE c1 < 20) FROM ft2; - sum ------ - 190 -(1 row) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(DISTINCT c1) FROM ft2; - QUERY PLAN ---------------------------------------------------------------------- - Foreign Scan - Output: (count(DISTINCT c1)) - Relations: Aggregate on (ft2) - Remote SQL: SELECT count(DISTINCT c1) FROM binary_queries_test.t2 -(4 rows) - -SELECT COUNT(DISTINCT c1) FROM ft2; - count -------- - 100 -(1 row) - -/* DISTINCT with IF */ -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(DISTINCT c1) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN --------------------------------------------------------------------------------------- - Aggregate - Output: count(DISTINCT c1) FILTER (WHERE (c1 < 20)) - -> Foreign Scan on binary_queries_test.ft2 - Output: c1, c2 - Remote SQL: SELECT c1 FROM binary_queries_test.t2 ORDER BY c1 ASC NULLS LAST -(5 rows) - -SELECT COUNT(DISTINCT c1) FILTER (WHERE c1 < 20) FROM ft2; - count -------- - 19 -(1 row) - -/* https://github.com/ClickHouse/pg_clickhouse/issues/25 */ -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(*) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN ------------------------------------------------------------------------------ - Foreign Scan - Output: (count(*) FILTER (WHERE (c1 < 20))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT countIf((((c1 < 20)) > 0)) FROM binary_queries_test.t2 -(4 rows) - -SELECT COUNT(*) FILTER (WHERE c1 < 10) FROM ft2; - count -------- - 9 -(1 row) - -SELECT clickhouse_raw_query('DROP DATABASE binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -DROP USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback2; -DROP USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback; -DROP SERVER binary_queries_loopback2 CASCADE; -NOTICE: drop cascades to foreign table ft6 -DROP SERVER binary_queries_loopback CASCADE; -NOTICE: drop cascades to 4 other objects -DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft2 -drop cascades to foreign table ft4 -drop cascades to foreign table ft5 diff --git a/test/expected/binary_queries_7.out b/test/expected/binary_queries_7.out deleted file mode 100644 index 6ba4bc34..00000000 --- a/test/expected/binary_queries_7.out +++ /dev/null @@ -1,745 +0,0 @@ -SET datestyle = 'ISO'; -CREATE SERVER binary_queries_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_queries_test', driver 'binary'); -CREATE SERVER binary_queries_loopback2 FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_queries_test', driver 'binary'); -CREATE USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback; -CREATE USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback2; -SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t1 - (c1 Int, c2 Int, c3 String, c4 Date, c5 Date, c6 String, c7 String, c8 String) - ENGINE = MergeTree PARTITION BY c4 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t2 (c1 Int, c2 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t3 (c1 Int, c3 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE binary_queries_test.t4 (c1 Int, c2 Int, c3 String) - ENGINE = MergeTree PARTITION BY c1 % 10000 ORDER BY (c1);'); - clickhouse_raw_query ----------------------- - -(1 row) - -CREATE FOREIGN TABLE ft1 ( - c0 int, - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER binary_queries_loopback OPTIONS (table_name 't1'); -ALTER FOREIGN TABLE ft1 DROP COLUMN c0; -CREATE FOREIGN TABLE ft2 ( - c1 int NOT NULL, - c2 text NOT NULL -) SERVER binary_queries_loopback OPTIONS (table_name 't2'); -CREATE FOREIGN TABLE ft4 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback OPTIONS (table_name 't4'); -CREATE FOREIGN TABLE ft5 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback OPTIONS (table_name 't4'); -CREATE FOREIGN TABLE ft6 ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER binary_queries_loopback2 OPTIONS (table_name 't4'); -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t1 - SELECT number, - number % 10, - toString(number), - toDate('1990-01-01'), - toDate('1990-01-01'), - number % 10, - number % 10, - 'foo' - FROM numbers(1, 110);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t2 - SELECT number, - concat('AAA', toString(number)) - FROM numbers(1, 100);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -select clickhouse_raw_query($$ - INSERT INTO binary_queries_test.t4 - SELECT number, - number + 1, - concat('AAA', toString(number)) - FROM numbers(1, 100);$$); - clickhouse_raw_query ----------------------- - -(1 row) - -\set VERBOSITY terse -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work - c3 | c4 -----+------------ - 1 | 1990-01-01 -(1 row) - -ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'no such database'); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: DB::Exception: Database `no such database` doesn't exist -ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (ADD user 'no such user'); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should fail -ERROR: pg_clickhouse: connection error: DB::Exception: no such user: Authentication failed: password is incorrect or there is no user with such name -ALTER SERVER binary_queries_loopback OPTIONS (SET dbname 'binary_queries_test'); -ALTER USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback OPTIONS (DROP user); -SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work again - c3 | c4 -----+------------ - 1 | 1990-01-01 -(1 row) - -\set VERBOSITY default -ANALYZE ft1; -EXPLAIN (COSTS OFF) SELECT * FROM ft1 ORDER BY c1 OFFSET 100 LIMIT 10; - QUERY PLAN ---------------------- - Foreign Scan on ft1 -(1 row) - -SELECT * FROM ft1 ORDER BY c1 OFFSET 100 LIMIT 10; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 ORDER BY t1.c1, t1.tableoid OFFSET 100 LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------------------------------- - Limit - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - -> Sort - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - Sort Key: t1.c1, t1.tableoid - -> Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8, tableoid - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 -(8 rows) - -SELECT * FROM ft1 t1 ORDER BY t1.c1, t1.tableoid OFFSET 100 LIMIT 10; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1 FROM ft1 t1 ORDER BY t1.c1 OFFSET 100 LIMIT 10; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: t1.*, c1 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 ORDER BY c1 ASC NULLS LAST LIMIT 10 OFFSET 100 -(3 rows) - -SELECT t1 FROM ft1 t1 ORDER BY t1.c1 OFFSET 100 LIMIT 10; - t1 -------------------------------------------- - (101,1,101,1990-01-01,1990-01-01,1,1,foo) - (102,2,102,1990-01-01,1990-01-01,2,2,foo) - (103,3,103,1990-01-01,1990-01-01,3,3,foo) - (104,4,104,1990-01-01,1990-01-01,4,4,foo) - (105,5,105,1990-01-01,1990-01-01,5,5,foo) - (106,6,106,1990-01-01,1990-01-01,6,6,foo) - (107,7,107,1990-01-01,1990-01-01,7,7,foo) - (108,8,108,1990-01-01,1990-01-01,8,8,foo) - (109,9,109,1990-01-01,1990-01-01,9,9,foo) - (110,0,110,1990-01-01,1990-01-01,0,0,foo) -(10 rows) - -SELECT * FROM ft1 WHERE false; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 101 AND t1.c6 = '1' AND t1.c7 >= '1'; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c7 >= '1')) AND ((c1 = 101)) AND ((c6 = '1')) -(3 rows) - -SELECT COUNT(*) FROM ft1 t1; - count -------- - 110 -(1 row) - -SELECT * FROM ft1 t1 WHERE t1.c3 IN (SELECT c2 FROM ft2 t2 WHERE c1 <= 10) ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -SELECT * FROM ft1 t1 WHERE t1.c3 = (SELECT MAX(c2) FROM ft2 t2) ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+----+----+----+----+----+---- -(0 rows) - -WITH t1 AS (SELECT * FROM ft1 WHERE c1 <= 10) SELECT t2.c1, t2.c2, t2.c2 FROM t1, ft2 t2 WHERE t1.c1 = t2.c1 ORDER BY t1.c1; - c1 | c2 | c2 -----+-------+------- - 1 | AAA1 | AAA1 - 2 | AAA2 | AAA2 - 3 | AAA3 | AAA3 - 4 | AAA4 | AAA4 - 5 | AAA5 | AAA5 - 6 | AAA6 | AAA6 - 7 | AAA7 | AAA7 - 8 | AAA8 | AAA8 - 9 | AAA9 | AAA9 - 10 | AAA10 | AAA10 -(10 rows) - -SELECT 'fixed', NULL FROM ft1 t1 WHERE c1 = 1; - ?column? | ?column? -----------+---------- - fixed | -(1 row) - -SET enable_hashjoin TO false; -SET enable_nestloop TO false; -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c1 FROM ft2 t1 JOIN ft1 t2 ON (t1.c1 = t2.c1) OFFSET 100 LIMIT 10; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: t1.c1, t2.c1 - Relations: (ft2 t1) INNER JOIN (ft1 t2) - Remote SQL: SELECT r1.c1, r2.c1 FROM binary_queries_test.t2 r1 ALL INNER JOIN binary_queries_test.t1 r2 ON (((r1.c1 = r2.c1))) LIMIT 10 OFFSET 100 -(4 rows) - -SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - c1 | c1 -----+---- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) OFFSET 100 LIMIT 10; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: t1.c1, t2.c1 - Relations: (ft2 t1) LEFT JOIN (ft1 t2) - Remote SQL: SELECT r1.c1, r2.c1 FROM binary_queries_test.t2 r1 ALL LEFT JOIN binary_queries_test.t1 r2 ON (((r1.c1 = r2.c1))) LIMIT 10 OFFSET 100 -(4 rows) - -EXPLAIN SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - QUERY PLAN ----------------------------------------------------------------- - Limit (cost=1.00..1.05 rows=10 width=32) - -> Unique (cost=1.00..5.60 rows=1000 width=32) - -> Foreign Scan (cost=1.00..0.60 rows=1000 width=32) - Relations: (ft2 t1) LEFT JOIN (ft1 t2) -(4 rows) - -SELECT DISTINCT t1.c1, t2.c1 FROM ft2 t1 LEFT JOIN ft1 t2 ON (t1.c1 = t2.c1) order by t1.c1 LIMIT 10; - c1 | c1 -----+---- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -RESET enable_hashjoin; -RESET enable_nestloop; -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 1; -- Var, OpExpr(b), Const - QUERY PLAN --------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = 1)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.c1 = 100 AND t1.c2 = 0; -- BoolExpr - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = 100)) AND ((c2 = 0)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NULL; -- NullTest - QUERY PLAN ------------------------------------------- - Result - Output: c1, c2, c3, c4, c5, c6, c7, c8 - One-Time Filter: false -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL; -- NullTest - QUERY PLAN ---------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE round(abs(c1), 0) = 1; -- FuncExpr - QUERY PLAN ------------------------------------------------------------------------------------------------------------------ - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((round(abs(c1), 0) = 1)) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = -c1; -- OpExpr(l) - QUERY PLAN -------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = (- c1))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE 1 = factorial(c1); -- OpExpr(r) - QUERY PLAN -------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((1 = factorial(c1))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE (c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL); -- DistinctExpr - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE (((c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL))) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]); -- ScalarArrayOpExpr - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((has([c2, 1, (c1 + 0)],c1))) -(3 rows) - -SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]) ORDER BY c1; -- ScalarArrayOpExpr - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 1 | 1 | 1 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 2 | 2 | 2 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 3 | 3 | 3 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 4 | 4 | 4 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 5 | 5 | 5 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 6 | 6 | 6 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 7 | 7 | 7 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 8 | 8 | 8 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 9 | 9 | 9 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 10 | 0 | 10 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 11 | 1 | 11 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 12 | 2 | 12 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 13 | 3 | 13 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 14 | 4 | 14 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 15 | 5 | 15 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 16 | 6 | 16 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 17 | 7 | 17 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 18 | 8 | 18 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 19 | 9 | 19 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 20 | 0 | 20 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 21 | 1 | 21 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 22 | 2 | 22 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 23 | 3 | 23 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 24 | 4 | 24 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 25 | 5 | 25 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 26 | 6 | 26 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 27 | 7 | 27 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 28 | 8 | 28 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 29 | 9 | 29 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 30 | 0 | 30 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 31 | 1 | 31 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 32 | 2 | 32 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 33 | 3 | 33 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 34 | 4 | 34 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 35 | 5 | 35 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 36 | 6 | 36 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 37 | 7 | 37 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 38 | 8 | 38 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 39 | 9 | 39 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 40 | 0 | 40 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 41 | 1 | 41 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 42 | 2 | 42 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 43 | 3 | 43 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 44 | 4 | 44 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 45 | 5 | 45 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 46 | 6 | 46 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 47 | 7 | 47 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 48 | 8 | 48 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 49 | 9 | 49 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 50 | 0 | 50 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 51 | 1 | 51 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 52 | 2 | 52 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 53 | 3 | 53 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 54 | 4 | 54 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 55 | 5 | 55 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 56 | 6 | 56 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 57 | 7 | 57 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 58 | 8 | 58 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 59 | 9 | 59 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 60 | 0 | 60 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 61 | 1 | 61 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 62 | 2 | 62 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 63 | 3 | 63 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 64 | 4 | 64 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 65 | 5 | 65 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 66 | 6 | 66 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 67 | 7 | 67 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 68 | 8 | 68 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 69 | 9 | 69 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 70 | 0 | 70 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 71 | 1 | 71 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 72 | 2 | 72 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 73 | 3 | 73 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 74 | 4 | 74 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 75 | 5 | 75 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 76 | 6 | 76 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 77 | 7 | 77 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 78 | 8 | 78 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 79 | 9 | 79 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 80 | 0 | 80 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 81 | 1 | 81 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 82 | 2 | 82 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 83 | 3 | 83 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 84 | 4 | 84 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 85 | 5 | 85 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 86 | 6 | 86 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 87 | 7 | 87 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 88 | 8 | 88 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 89 | 9 | 89 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 90 | 0 | 90 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 91 | 1 | 91 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 92 | 2 | 92 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 93 | 3 | 93 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 94 | 4 | 94 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 95 | 5 | 95 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 96 | 6 | 96 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 97 | 7 | 97 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 98 | 8 | 98 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 99 | 9 | 99 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 100 | 0 | 100 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(110 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1]; -- ArrayRef - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c1 = (([c1, c2, 3])[1]))) -(3 rows) - -SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1] ORDER BY c1; -- ArrayRef - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+------------+------------+----+----+----- - 1 | 1 | 1 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 2 | 2 | 2 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 3 | 3 | 3 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 4 | 4 | 4 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 5 | 5 | 5 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 6 | 6 | 6 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 7 | 7 | 7 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 8 | 8 | 8 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 9 | 9 | 9 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 10 | 0 | 10 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 11 | 1 | 11 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 12 | 2 | 12 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 13 | 3 | 13 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 14 | 4 | 14 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 15 | 5 | 15 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 16 | 6 | 16 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 17 | 7 | 17 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 18 | 8 | 18 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 19 | 9 | 19 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 20 | 0 | 20 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 21 | 1 | 21 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 22 | 2 | 22 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 23 | 3 | 23 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 24 | 4 | 24 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 25 | 5 | 25 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 26 | 6 | 26 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 27 | 7 | 27 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 28 | 8 | 28 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 29 | 9 | 29 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 30 | 0 | 30 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 31 | 1 | 31 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 32 | 2 | 32 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 33 | 3 | 33 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 34 | 4 | 34 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 35 | 5 | 35 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 36 | 6 | 36 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 37 | 7 | 37 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 38 | 8 | 38 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 39 | 9 | 39 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 40 | 0 | 40 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 41 | 1 | 41 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 42 | 2 | 42 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 43 | 3 | 43 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 44 | 4 | 44 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 45 | 5 | 45 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 46 | 6 | 46 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 47 | 7 | 47 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 48 | 8 | 48 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 49 | 9 | 49 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 50 | 0 | 50 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 51 | 1 | 51 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 52 | 2 | 52 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 53 | 3 | 53 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 54 | 4 | 54 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 55 | 5 | 55 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 56 | 6 | 56 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 57 | 7 | 57 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 58 | 8 | 58 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 59 | 9 | 59 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 60 | 0 | 60 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 61 | 1 | 61 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 62 | 2 | 62 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 63 | 3 | 63 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 64 | 4 | 64 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 65 | 5 | 65 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 66 | 6 | 66 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 67 | 7 | 67 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 68 | 8 | 68 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 69 | 9 | 69 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 70 | 0 | 70 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 71 | 1 | 71 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 72 | 2 | 72 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 73 | 3 | 73 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 74 | 4 | 74 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 75 | 5 | 75 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 76 | 6 | 76 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 77 | 7 | 77 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 78 | 8 | 78 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 79 | 9 | 79 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 80 | 0 | 80 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 81 | 1 | 81 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 82 | 2 | 82 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 83 | 3 | 83 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 84 | 4 | 84 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 85 | 5 | 85 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 86 | 6 | 86 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 87 | 7 | 87 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 88 | 8 | 88 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 89 | 9 | 89 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 90 | 0 | 90 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 91 | 1 | 91 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 92 | 2 | 92 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 93 | 3 | 93 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 94 | 4 | 94 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 95 | 5 | 95 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 96 | 6 | 96 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 97 | 7 | 97 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 98 | 8 | 98 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 99 | 9 | 99 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 100 | 0 | 100 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo - 101 | 1 | 101 | 1990-01-01 | 1990-01-01 | 1 | 1 | foo - 102 | 2 | 102 | 1990-01-01 | 1990-01-01 | 2 | 2 | foo - 103 | 3 | 103 | 1990-01-01 | 1990-01-01 | 3 | 3 | foo - 104 | 4 | 104 | 1990-01-01 | 1990-01-01 | 4 | 4 | foo - 105 | 5 | 105 | 1990-01-01 | 1990-01-01 | 5 | 5 | foo - 106 | 6 | 106 | 1990-01-01 | 1990-01-01 | 6 | 6 | foo - 107 | 7 | 107 | 1990-01-01 | 1990-01-01 | 7 | 7 | foo - 108 | 8 | 108 | 1990-01-01 | 1990-01-01 | 8 | 8 | foo - 109 | 9 | 109 | 1990-01-01 | 1990-01-01 | 9 | 9 | foo - 110 | 0 | 110 | 1990-01-01 | 1990-01-01 | 0 | 0 | foo -(110 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c6 = E'foo''s\\bar'; -- check special chars - QUERY PLAN --------------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c6 = 'foo''s\\bar')) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c8 = 'foo'; -- can't be sent to remote - QUERY PLAN ------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT c1, c2, c3, c4, c5, c6, c7, c8 FROM binary_queries_test.t1 WHERE ((c8 = 'foo')) -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT (CASE WHEN c1 < 10 THEN 1 WHEN c1 < 50 THEN 2 ELSE 3 END) a, - sum(length(c2)) FROM ft2 GROUP BY a ORDER BY a; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Foreign Scan - Output: (CASE WHEN (c1 < 10) THEN 1 WHEN (c1 < 50) THEN 2 ELSE 3 END), (sum(length(c2))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END, sum(lengthUTF8(c2)) FROM binary_queries_test.t2 GROUP BY (CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END) ORDER BY CASE WHEN (c1 < 10) THEN toInt32(1) WHEN (c1 < 50) THEN toInt32(2) ELSE toInt32(3) END ASC NULLS LAST -(4 rows) - -SELECT (CASE WHEN c1 < 10 THEN 1 WHEN c1 < 50 THEN 2 ELSE 3 END) a, - sum(length(c2)) FROM ft2 GROUP BY a ORDER BY a; - a | sum ----+----- - 1 | 36 - 2 | 200 - 3 | 256 -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT SUM(c1) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN ------------------------------------------------------------------------------- - Foreign Scan - Output: (sum(c1) FILTER (WHERE (c1 < 20))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT sumIf(c1,(((c1 < 20)) > 0)) FROM binary_queries_test.t2 -(4 rows) - -SELECT SUM(c1) FILTER (WHERE c1 < 20) FROM ft2; - sum ------ - 190 -(1 row) - -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(DISTINCT c1) FROM ft2; - QUERY PLAN ---------------------------------------------------------------------- - Foreign Scan - Output: (count(DISTINCT c1)) - Relations: Aggregate on (ft2) - Remote SQL: SELECT count(DISTINCT c1) FROM binary_queries_test.t2 -(4 rows) - -SELECT COUNT(DISTINCT c1) FROM ft2; - count -------- - 100 -(1 row) - -/* DISTINCT with IF */ -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(DISTINCT c1) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN --------------------------------------------------------------------------------------- - Aggregate - Output: count(DISTINCT c1) FILTER (WHERE (c1 < 20)) - -> Foreign Scan on public.ft2 - Output: c1, c2 - Remote SQL: SELECT c1 FROM binary_queries_test.t2 ORDER BY c1 ASC NULLS LAST -(5 rows) - -SELECT COUNT(DISTINCT c1) FILTER (WHERE c1 < 20) FROM ft2; - count -------- - 19 -(1 row) - -/* https://github.com/ClickHouse/pg_clickhouse/issues/25 */ -EXPLAIN (VERBOSE, COSTS OFF) SELECT COUNT(*) FILTER (WHERE c1 < 20) FROM ft2; - QUERY PLAN ------------------------------------------------------------------------------ - Foreign Scan - Output: (count(*) FILTER (WHERE (c1 < 20))) - Relations: Aggregate on (ft2) - Remote SQL: SELECT countIf((((c1 < 20)) > 0)) FROM binary_queries_test.t2 -(4 rows) - -SELECT COUNT(*) FILTER (WHERE c1 < 10) FROM ft2; - count -------- - 9 -(1 row) - -SELECT clickhouse_raw_query('DROP DATABASE binary_queries_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -DROP USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback2; -DROP USER MAPPING FOR CURRENT_USER SERVER binary_queries_loopback; -DROP SERVER binary_queries_loopback2 CASCADE; -NOTICE: drop cascades to foreign table ft6 -DROP SERVER binary_queries_loopback CASCADE; -NOTICE: drop cascades to 4 other objects -DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft2 -drop cascades to foreign table ft4 -drop cascades to foreign table ft5 diff --git a/test/expected/import_schema.out b/test/expected/import_schema.out index c50c4dd1..1d57c9ab 100644 --- a/test/expected/import_schema.out +++ b/test/expected/import_schema.out @@ -21,6 +21,12 @@ SELECT clickhouse_raw_query('CREATE DATABASE import_test'); (1 row) +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test_2'); + clickhouse_raw_query +---------------------- + +(1 row) + SELECT clickhouse_raw_query('CREATE DATABASE import_test_2'); clickhouse_raw_query ---------------------- @@ -703,6 +709,7 @@ SELECT * FROM clickhouse_bin.ints WHERE c1 IN (127, -128) ORDER BY c1; ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; @@ -714,10 +721,10 @@ SELECT c1, c2, c3, c4, c5, c6, c7, NULL, c9, c10 FROM clickhouse_bin.ints WHERE c1 = 127 ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -------+--------+-------------+----------------------+-----+----+------------+----+---------------+------------------------- - -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754942e-38 | 2.2250738585072014e-308 - 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | -1 | 4294967295 | | 3.4028233e+38 | 1.7976931348623157e+308 + c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 +------+--------+-------------+----------------------+-----+-------+------------+----+---------------+------------------------- + -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754942e-38 | 2.2250738585072014e-308 + 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | 65535 | 4294967295 | | 3.4028233e+38 | 1.7976931348623157e+308 (2 rows) SELECT * FROM clickhouse.ints WHERE c1 = -128 diff --git a/test/expected/import_schema_1.out b/test/expected/import_schema_1.out index af315452..1e1a14b3 100644 --- a/test/expected/import_schema_1.out +++ b/test/expected/import_schema_1.out @@ -21,6 +21,12 @@ SELECT clickhouse_raw_query('CREATE DATABASE import_test'); (1 row) +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test_2'); + clickhouse_raw_query +---------------------- + +(1 row) + SELECT clickhouse_raw_query('CREATE DATABASE import_test_2'); clickhouse_raw_query ---------------------- @@ -603,6 +609,7 @@ SELECT * FROM clickhouse_bin.ints WHERE c1 IN (127, -128) ORDER BY c1; ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; @@ -614,10 +621,10 @@ SELECT c1, c2, c3, c4, c5, c6, c7, NULL, c9, c10 FROM clickhouse_bin.ints WHERE c1 = 127 ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -------+--------+-------------+----------------------+-----+----+------------+----+---------------+------------------------- - -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754942e-38 | 2.2250738585072014e-308 - 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | -1 | 4294967295 | | 3.4028233e+38 | 1.7976931348623157e+308 + c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 +------+--------+-------------+----------------------+-----+-------+------------+----+---------------+------------------------- + -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754942e-38 | 2.2250738585072014e-308 + 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | 65535 | 4294967295 | | 3.4028233e+38 | 1.7976931348623157e+308 (2 rows) SELECT * FROM clickhouse.ints WHERE c1 = -128 diff --git a/test/expected/import_schema_2.out b/test/expected/import_schema_2.out deleted file mode 100644 index ed19d783..00000000 --- a/test/expected/import_schema_2.out +++ /dev/null @@ -1,774 +0,0 @@ -SET datestyle = 'ISO'; -CREATE SERVER import_loopback FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'import_test', driver 'http'); -CREATE SERVER import_loopback_bin FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'import_test', driver 'binary'); -CREATE SCHEMA clickhouse; -CREATE SCHEMA clickhouse_bin; -CREATE SCHEMA clickhouse_limit; -CREATE SCHEMA clickhouse_except; -CREATE USER MAPPING FOR CURRENT_USER SERVER import_loopback; -CREATE USER MAPPING FOR CURRENT_USER SERVER import_loopback_bin; -SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE import_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE import_test_2'); - clickhouse_raw_query ----------------------- - -(1 row) - --- integer types -SELECT clickhouse_raw_query('CREATE TABLE import_test.ints ( - c1 Int8, c2 Int16, c3 Int32, c4 Int64, - c5 UInt8, c6 UInt16, c7 UInt32, c8 UInt64, - c9 Float32, c10 Nullable(Float64) -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.ints SELECT - number, number + 1, number + 2, number + 3, number + 4, number + 5, - number + 6, number + 7, number + 8.1, number + 9.2 FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.ints SELECT - number, number + 1, number + 2, number + 3, number + 4, number + 5, - number + 6, number + 7, number + 8.1, NULL FROM numbers(10, 2);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE import_test.types ( - c1 Date, c2 DateTime, c3 String, c4 FixedString(5), c5 UUID, - c6 Enum8(''one'' = 1, ''two'' = 2), - c7 Enum16(''one'' = 1, ''two'' = 2, ''three'' = 3), - c9 Nullable(FixedString(50)), - c8 LowCardinality(String) -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.types SELECT - addDays(toDate(''1990-01-01''), number), - addMinutes(addSeconds(addDays(toDateTime(''1990-01-01 10:00:00''), number), number), number), - format(''number {0}'', toString(number)), - format(''num {0}'', toString(number)), - format(''f4bf890f-f9dc-4332-ad5c-0c18e73f28e{0}'', toString(number)), - ''two'', - ''three'', - toString(number), - format(''cardinal {0}'', toString(number)) - FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE import_test.types2 ( - c1 LowCardinality(Nullable(String)) -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1) SETTINGS allow_nullable_key = 1; -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.types2 SELECT - format(''cardinal {0}'', toString(number + 1)) - FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE TABLE import_test.ip ( - c1 IPv4, - c2 IPv6 -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query($$ - INSERT INTO import_test.ip VALUES - ('116.106.34.242', '2001:44c8:129:2632:33:0:252:2'), - ('116.106.34.243', '2a02:e980:1e::1'), - ('116.106.34.244', '::1'); -$$); - clickhouse_raw_query ----------------------- - -(1 row) - --- array types -SELECT clickhouse_raw_query('CREATE TABLE import_test.arrays ( - c1 Array(Int), c2 Array(String) -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.arrays SELECT - [number, number + 1], - [format(''num{0}'', toString(number)), format(''num{0}'', toString(number + 1))] - FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - --- tuple -SELECT clickhouse_raw_query('CREATE TABLE import_test.tuples ( - c1 Int8, - c2 Tuple(Int, String, Float32), - c3 Nested(a Int, b Int), - c4 Int16 -) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); -'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.tuples SELECT - number, - (number, toString(number), number + 1.0), - [toInt32(number),1,1], - [toInt32(number),2,2], - toInt16(number) - FROM numbers(10);'); - clickhouse_raw_query ----------------------- - -(1 row) - --- datetime with timezones -SELECT clickhouse_raw_query('CREATE TABLE import_test.timezones ( - t1 DateTime64(6,''UTC''), - t2 DateTime64(6,''Europe/Berlin''), - t4 DateTime(''Europe/Berlin''), - t5 DateTime64(6)) - ENGINE = MergeTree ORDER BY (t1) SETTINGS index_granularity=8192;'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.timezones VALUES ( - ''2020-01-01 11:00:00'', - ''2020-01-01 11:00:00'', - ''2020-01-01 11:00:00'', - ''2020-01-01 11:00:00'')'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('INSERT INTO import_test.timezones VALUES ( - ''2020-01-01 12:00:00'', - ''2020-01-01 12:00:00'', - ''2020-01-01 12:00:00'', - ''2020-01-01 12:00:00'')'); - clickhouse_raw_query ----------------------- - -(1 row) - --- Injection attempt should import nothing. -IMPORT FOREIGN SCHEMA "import_test' OR database = 'public" FROM SERVER import_loopback INTO clickhouse; -\det clickhouse.* - List of foreign tables - Schema | Table | Server ---------+-------+-------- -(0 rows) - --- Should succeed. -IMPORT FOREIGN SCHEMA import_test FROM SERVER import_loopback INTO clickhouse; -NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "c2", please create composite type and alter the column if needed -\d+ clickhouse.ints; - Foreign table "clickhouse.ints" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------------------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | smallint | | not null | | | plain | | - c3 | integer | | not null | | | plain | | - c4 | bigint | | not null | | | plain | | - c5 | smallint | | not null | | | plain | | - c6 | integer | | not null | | | plain | | - c7 | bigint | | not null | | | plain | | - c8 | bigint | | not null | | | plain | | - c9 | real | | not null | | | plain | | - c10 | double precision | | | | | plain | | -Not-null constraints: - "ints_c1_not_null" NOT NULL "c1" - "ints_c2_not_null" NOT NULL "c2" - "ints_c3_not_null" NOT NULL "c3" - "ints_c4_not_null" NOT NULL "c4" - "ints_c5_not_null" NOT NULL "c5" - "ints_c6_not_null" NOT NULL "c6" - "ints_c7_not_null" NOT NULL "c7" - "ints_c8_not_null" NOT NULL "c8" - "ints_c9_not_null" NOT NULL "c9" -Server: import_loopback -FDW options: (database 'import_test', table_name 'ints', engine 'MergeTree') - -\d+ clickhouse.types; - Foreign table "clickhouse.types" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | date | | not null | | | plain | | - c2 | timestamp without time zone | | not null | | | plain | | - c3 | text | | not null | | | extended | | - c4 | character varying(5) | | not null | | | extended | | - c5 | uuid | | not null | | | plain | | - c6 | text | | not null | | | extended | | - c7 | text | | not null | | | extended | | - c9 | character varying(50) | | | | | extended | | - c8 | text | | not null | | | extended | | -Not-null constraints: - "types_c1_not_null" NOT NULL "c1" - "types_c2_not_null" NOT NULL "c2" - "types_c3_not_null" NOT NULL "c3" - "types_c4_not_null" NOT NULL "c4" - "types_c5_not_null" NOT NULL "c5" - "types_c6_not_null" NOT NULL "c6" - "types_c7_not_null" NOT NULL "c7" - "types_c8_not_null" NOT NULL "c8" -Server: import_loopback -FDW options: (database 'import_test', table_name 'types', engine 'MergeTree') - -\d+ clickhouse.types2; - Foreign table "clickhouse.types2" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | text | | | | | extended | | -Server: import_loopback -FDW options: (database 'import_test', table_name 'types2', engine 'MergeTree') - -\d+ clickhouse.arrays; - Foreign table "clickhouse.arrays" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | integer[] | | not null | | | extended | | - c2 | text[] | | not null | | | extended | | -Not-null constraints: - "arrays_c1_not_null" NOT NULL "c1" - "arrays_c2_not_null" NOT NULL "c2" -Server: import_loopback -FDW options: (database 'import_test', table_name 'arrays', engine 'MergeTree') - -\d+ clickhouse.tuples; - Foreign table "clickhouse.tuples" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | text | | not null | | | extended | | - c3.a | integer[] | | not null | | | extended | | - c3.b | integer[] | | not null | | | extended | | - c4 | smallint | | not null | | | plain | | -Not-null constraints: - "tuples_c1_not_null" NOT NULL "c1" - "tuples_c2_not_null" NOT NULL "c2" - "tuples_c3.a_not_null" NOT NULL "c3.a" - "tuples_c3.b_not_null" NOT NULL "c3.b" - "tuples_c4_not_null" NOT NULL "c4" -Server: import_loopback -FDW options: (database 'import_test', table_name 'tuples', engine 'MergeTree') - -\d+ clickhouse.timezones; - Foreign table "clickhouse.timezones" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+---------+--------------+------------- - t1 | timestamp without time zone | | not null | | | plain | | - t2 | timestamp without time zone | | not null | | | plain | | - t4 | timestamp without time zone | | not null | | | plain | | - t5 | timestamp without time zone | | not null | | | plain | | -Not-null constraints: - "timezones_t1_not_null" NOT NULL "t1" - "timezones_t2_not_null" NOT NULL "t2" - "timezones_t4_not_null" NOT NULL "t4" - "timezones_t5_not_null" NOT NULL "t5" -Server: import_loopback -FDW options: (database 'import_test', table_name 'timezones', engine 'MergeTree') - -\d+ clickhouse.ip; - Foreign table "clickhouse.ip" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | inet | | not null | | | main | | - c2 | inet | | not null | | | main | | -Not-null constraints: - "ip_c1_not_null" NOT NULL "c1" - "ip_c2_not_null" NOT NULL "c2" -Server: import_loopback -FDW options: (database 'import_test', table_name 'ip', engine 'MergeTree') - -SELECT * FROM clickhouse.ints ORDER BY c1 DESC LIMIT 4; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -----+----+----+----+----+----+----+----+------+------ - 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19.1 | - 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18.1 | - 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17.1 | 18.2 - 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16.1 | 17.2 -(4 rows) - -SELECT * FROM clickhouse.types ORDER BY c1 LIMIT 2; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c9 | c8 -------------+---------------------+----------+-------+--------------------------------------+-----+-------+----+------------ - 1990-01-01 | 1990-01-01 10:00:00 | number 0 | num 0 | f4bf890f-f9dc-4332-ad5c-0c18e73f28e0 | two | three | 0 | cardinal 0 - 1990-01-02 | 1990-01-02 10:01:01 | number 1 | num 1 | f4bf890f-f9dc-4332-ad5c-0c18e73f28e1 | two | three | 1 | cardinal 1 -(2 rows) - -SELECT * FROM clickhouse.types2 ORDER BY c1 LIMIT 2; - c1 -------------- - cardinal 1 - cardinal 10 -(2 rows) - -SELECT * FROM clickhouse.arrays ORDER BY c1 LIMIT 2; - c1 | c2 --------+----------------- - {0,1} | {'num0','num1'} - {1,2} | {'num1','num2'} -(2 rows) - -SELECT * FROM clickhouse.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+-----------+---------+---------+---- - 0 | (0,'0',1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,'1',2) | {1,1,1} | {1,2,2} | 1 -(2 rows) - -SELECT * FROM clickhouse.timezones ORDER BY t1 LIMIT 2; - t1 | t2 | t4 | t5 -------------------------+------------------------+------------------------+------------------------ - 2020-01-01 03:00:00-08 | 2020-01-01 02:00:00-08 | 2020-01-01 02:00:00-08 | 2020-01-01 03:00:00-08 - 2020-01-01 04:00:00-08 | 2020-01-01 03:00:00-08 | 2020-01-01 03:00:00-08 | 2020-01-01 04:00:00-08 -(2 rows) - -SELECT * FROM clickhouse.ip ORDER BY c1; - c1 | c2 -----------------+------------------------------- - 116.106.34.242 | 2001:44c8:129:2632:33:0:252:2 - 116.106.34.243 | 2a02:e980:1e::1 - 116.106.34.244 | ::1 -(3 rows) - -IMPORT FOREIGN SCHEMA "import_test" FROM SERVER import_loopback_bin INTO clickhouse_bin; -NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "c2", please create composite type and alter the column if needed -\d+ clickhouse_bin.ints; - Foreign table "clickhouse_bin.ints" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------------------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | smallint | | not null | | | plain | | - c3 | integer | | not null | | | plain | | - c4 | bigint | | not null | | | plain | | - c5 | smallint | | not null | | | plain | | - c6 | integer | | not null | | | plain | | - c7 | bigint | | not null | | | plain | | - c8 | bigint | | not null | | | plain | | - c9 | real | | not null | | | plain | | - c10 | double precision | | | | | plain | | -Not-null constraints: - "ints_c1_not_null" NOT NULL "c1" - "ints_c2_not_null" NOT NULL "c2" - "ints_c3_not_null" NOT NULL "c3" - "ints_c4_not_null" NOT NULL "c4" - "ints_c5_not_null" NOT NULL "c5" - "ints_c6_not_null" NOT NULL "c6" - "ints_c7_not_null" NOT NULL "c7" - "ints_c8_not_null" NOT NULL "c8" - "ints_c9_not_null" NOT NULL "c9" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'ints', engine 'MergeTree') - -\d+ clickhouse_bin.types; - Foreign table "clickhouse_bin.types" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | date | | not null | | | plain | | - c2 | timestamp without time zone | | not null | | | plain | | - c3 | text | | not null | | | extended | | - c4 | character varying(5) | | not null | | | extended | | - c5 | uuid | | not null | | | plain | | - c6 | text | | not null | | | extended | | - c7 | text | | not null | | | extended | | - c9 | character varying(50) | | | | | extended | | - c8 | text | | not null | | | extended | | -Not-null constraints: - "types_c1_not_null" NOT NULL "c1" - "types_c2_not_null" NOT NULL "c2" - "types_c3_not_null" NOT NULL "c3" - "types_c4_not_null" NOT NULL "c4" - "types_c5_not_null" NOT NULL "c5" - "types_c6_not_null" NOT NULL "c6" - "types_c7_not_null" NOT NULL "c7" - "types_c8_not_null" NOT NULL "c8" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'types', engine 'MergeTree') - -\d+ clickhouse_bin.types2; - Foreign table "clickhouse_bin.types2" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | text | | | | | extended | | -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'types2', engine 'MergeTree') - -\d+ clickhouse_bin.arrays; - Foreign table "clickhouse_bin.arrays" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | integer[] | | not null | | | extended | | - c2 | text[] | | not null | | | extended | | -Not-null constraints: - "arrays_c1_not_null" NOT NULL "c1" - "arrays_c2_not_null" NOT NULL "c2" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'arrays', engine 'MergeTree') - -\d+ clickhouse_bin.tuples; - Foreign table "clickhouse_bin.tuples" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | text | | not null | | | extended | | - c3.a | integer[] | | not null | | | extended | | - c3.b | integer[] | | not null | | | extended | | - c4 | smallint | | not null | | | plain | | -Not-null constraints: - "tuples_c1_not_null" NOT NULL "c1" - "tuples_c2_not_null" NOT NULL "c2" - "tuples_c3.a_not_null" NOT NULL "c3.a" - "tuples_c3.b_not_null" NOT NULL "c3.b" - "tuples_c4_not_null" NOT NULL "c4" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'tuples', engine 'MergeTree') - -\d+ clickhouse_bin.timezones; - Foreign table "clickhouse_bin.timezones" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+---------+--------------+------------- - t1 | timestamp without time zone | | not null | | | plain | | - t2 | timestamp without time zone | | not null | | | plain | | - t4 | timestamp without time zone | | not null | | | plain | | - t5 | timestamp without time zone | | not null | | | plain | | -Not-null constraints: - "timezones_t1_not_null" NOT NULL "t1" - "timezones_t2_not_null" NOT NULL "t2" - "timezones_t4_not_null" NOT NULL "t4" - "timezones_t5_not_null" NOT NULL "t5" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'timezones', engine 'MergeTree') - -\d+ clickhouse_bin.ip; - Foreign table "clickhouse_bin.ip" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | inet | | not null | | | main | | - c2 | inet | | not null | | | main | | -Not-null constraints: - "ip_c1_not_null" NOT NULL "c1" - "ip_c2_not_null" NOT NULL "c2" -Server: import_loopback_bin -FDW options: (database 'import_test', table_name 'ip', engine 'MergeTree') - -SELECT * FROM clickhouse_bin.ints ORDER BY c1 DESC LIMIT 4; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -----+----+----+----+----+----+----+----+------+------ - 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19.1 | - 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18.1 | - 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17.1 | 18.2 - 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16.1 | 17.2 -(4 rows) - -SELECT * FROM clickhouse_bin.types ORDER BY c1 LIMIT 2; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c9 | c8 -------------+---------------------+----------+-------+--------------------------------------+-----+-------+----+------------ - 1990-01-01 | 1990-01-01 10:00:00 | number 0 | num 0 | f4bf890f-f9dc-4332-ad5c-0c18e73f28e0 | two | three | 0 | cardinal 0 - 1990-01-02 | 1990-01-02 10:01:01 | number 1 | num 1 | f4bf890f-f9dc-4332-ad5c-0c18e73f28e1 | two | three | 1 | cardinal 1 -(2 rows) - -SELECT * FROM clickhouse_bin.types2 ORDER BY c1 LIMIT 2; - c1 -------------- - cardinal 1 - cardinal 10 -(2 rows) - -SELECT * FROM clickhouse_bin.arrays ORDER BY c1 LIMIT 2; - c1 | c2 --------+------------- - {0,1} | {num0,num1} - {1,2} | {num1,num2} -(2 rows) - -SELECT * FROM clickhouse_bin.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+---------+---------+---------+---- - 0 | (0,0,1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,1,2) | {1,1,1} | {1,2,2} | 1 -(2 rows) - -SELECT * FROM clickhouse_bin.timezones ORDER BY t1 LIMIT 2; - t1 | t2 | t4 | t5 ----------------------+---------------------+---------------------+--------------------- - 2020-01-01 11:00:00 | 2020-01-01 10:00:00 | 2020-01-01 10:00:00 | 2020-01-01 11:00:00 - 2020-01-01 12:00:00 | 2020-01-01 11:00:00 | 2020-01-01 11:00:00 | 2020-01-01 12:00:00 -(2 rows) - -SELECT * FROM clickhouse.ip ORDER BY c1; - c1 | c2 -----------------+------------------------------- - 116.106.34.242 | 2001:44c8:129:2632:33:0:252:2 - 116.106.34.243 | 2a02:e980:1e::1 - 116.106.34.244 | ::1 -(3 rows) - -IMPORT FOREIGN SCHEMA "import_test" LIMIT TO (ints, types) FROM SERVER import_loopback INTO clickhouse_limit; -\d+ clickhouse_limit.ints; - Foreign table "clickhouse_limit.ints" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+------------------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | smallint | | not null | | | plain | | - c3 | integer | | not null | | | plain | | - c4 | bigint | | not null | | | plain | | - c5 | smallint | | not null | | | plain | | - c6 | integer | | not null | | | plain | | - c7 | bigint | | not null | | | plain | | - c8 | bigint | | not null | | | plain | | - c9 | real | | not null | | | plain | | - c10 | double precision | | | | | plain | | -Not-null constraints: - "ints_c1_not_null" NOT NULL "c1" - "ints_c2_not_null" NOT NULL "c2" - "ints_c3_not_null" NOT NULL "c3" - "ints_c4_not_null" NOT NULL "c4" - "ints_c5_not_null" NOT NULL "c5" - "ints_c6_not_null" NOT NULL "c6" - "ints_c7_not_null" NOT NULL "c7" - "ints_c8_not_null" NOT NULL "c8" - "ints_c9_not_null" NOT NULL "c9" -Server: import_loopback -FDW options: (database 'import_test', table_name 'ints', engine 'MergeTree') - -\d+ clickhouse_limit.types; - Foreign table "clickhouse_limit.types" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------------------------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | date | | not null | | | plain | | - c2 | timestamp without time zone | | not null | | | plain | | - c3 | text | | not null | | | extended | | - c4 | character varying(5) | | not null | | | extended | | - c5 | uuid | | not null | | | plain | | - c6 | text | | not null | | | extended | | - c7 | text | | not null | | | extended | | - c9 | character varying(50) | | | | | extended | | - c8 | text | | not null | | | extended | | -Not-null constraints: - "types_c1_not_null" NOT NULL "c1" - "types_c2_not_null" NOT NULL "c2" - "types_c3_not_null" NOT NULL "c3" - "types_c4_not_null" NOT NULL "c4" - "types_c5_not_null" NOT NULL "c5" - "types_c6_not_null" NOT NULL "c6" - "types_c7_not_null" NOT NULL "c7" - "types_c8_not_null" NOT NULL "c8" -Server: import_loopback -FDW options: (database 'import_test', table_name 'types', engine 'MergeTree') - -\d+ clickhouse_limit.arrays; -\d+ clickhouse_limit.tuples; -IMPORT FOREIGN SCHEMA "import_test" EXCEPT (ints, types) FROM SERVER import_loopback INTO clickhouse_except; -NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "c2", please create composite type and alter the column if needed -\d+ clickhouse_except.ints; -\d+ clickhouse_except.types; -\d+ clickhouse_except.arrays; - Foreign table "clickhouse_except.arrays" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | integer[] | | not null | | | extended | | - c2 | text[] | | not null | | | extended | | -Not-null constraints: - "arrays_c1_not_null" NOT NULL "c1" - "arrays_c2_not_null" NOT NULL "c2" -Server: import_loopback -FDW options: (database 'import_test', table_name 'arrays', engine 'MergeTree') - -\d+ clickhouse_except.tuples; - Foreign table "clickhouse_except.tuples" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+-------------+----------+--------------+------------- - c1 | smallint | | not null | | | plain | | - c2 | text | | not null | | | extended | | - c3.a | integer[] | | not null | | | extended | | - c3.b | integer[] | | not null | | | extended | | - c4 | smallint | | not null | | | plain | | -Not-null constraints: - "tuples_c1_not_null" NOT NULL "c1" - "tuples_c2_not_null" NOT NULL "c2" - "tuples_c3.a_not_null" NOT NULL "c3.a" - "tuples_c3.b_not_null" NOT NULL "c3.b" - "tuples_c4_not_null" NOT NULL "c4" -Server: import_loopback -FDW options: (database 'import_test', table_name 'tuples', engine 'MergeTree') - --- check custom database -SELECT clickhouse_raw_query('CREATE TABLE import_test_2.custom_option (a Int64) ENGINE = MergeTree ORDER BY (a)'); - clickhouse_raw_query ----------------------- - -(1 row) - -IMPORT FOREIGN SCHEMA "import_test_2" FROM SERVER import_loopback INTO clickhouse; -EXPLAIN VERBOSE SELECT * FROM clickhouse.custom_option; - QUERY PLAN ---------------------------------------------------------------------------------- - Foreign Scan on clickhouse.custom_option (cost=10.00..20.00 rows=1000 width=8) - Output: a - Remote SQL: SELECT a FROM import_test_2.custom_option -(3 rows) - -ALTER FOREIGN TABLE clickhouse.custom_option OPTIONS (DROP database); -EXPLAIN VERBOSE SELECT * FROM clickhouse.custom_option; - QUERY PLAN ---------------------------------------------------------------------------------- - Foreign Scan on clickhouse.custom_option (cost=10.00..20.00 rows=1000 width=8) - Output: a - Remote SQL: SELECT a FROM import_test.custom_option -(3 rows) - --- check overflows. -SELECT clickhouse_raw_query($$ - INSERT INTO import_test.ints (c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) VALUES - ( - -- Min values - -128, -32768, -2147483648, -9223372036854775808, - 0, 0, 0, 0, - 1.175494351e-38, 2.2250738585072014e-308 - ), - ( - -- Max values - 127, 32767, 2147483647, 9223372036854775807, - 255, 65535, 4294967295, 18446744073709551615, - 3.402823466e+38, 1.7976931348623158e+308 - ) -$$); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query($$ - SELECT * FROM import_test.ints - WHERE c1 IN (127, -128) - ORDER BY c1; -$$); - clickhouse_raw_query --------------------------------------------------------------------------------------------------------------------------------------------------------- - -128 -32768 -2147483648 -9223372036854775808 0 0 0 0 1.1754944e-38 2.2250738585072014e-308 + - 127 32767 2147483647 9223372036854775807 255 65535 4294967295 18446744073709551615 3.4028235e38 1.7976931348623157e308+ - -(1 row) - --- Error on 18446744073709551615. -SELECT * FROM clickhouse_bin.ints -WHERE c1 IN (127, -128) -ORDER BY c1; -ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint -SELECT * FROM clickhouse.ints -WHERE c1 IN (127, -128) -ORDER BY c1; -ERROR: value "18446744073709551615" is out of range for type bigint --- Ignore 18446744073709551615 -SELECT * FROM clickhouse_bin.ints WHERE c1 = -128 -UNION -SELECT c1, c2, c3, c4, c5, c6, c7, NULL, c9, c10 -FROM clickhouse_bin.ints -WHERE c1 = 127 -ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -------+--------+-------------+----------------------+-----+----+------------+----+---------------+------------------------- - -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754944e-38 | 2.2250738585072014e-308 - 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | -1 | 4294967295 | | 3.4028235e+38 | 1.7976931348623157e+308 -(2 rows) - -SELECT * FROM clickhouse.ints WHERE c1 = -128 -UNION -SELECT c1, c2, c3, c4, c5, c6, c7, NULL, c9, c10 -FROM clickhouse.ints -WHERE c1 = 127 -ORDER BY c1; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 -------+--------+-------------+----------------------+-----+-------+------------+----+---------------+------------------------- - -128 | -32768 | -2147483648 | -9223372036854775808 | 0 | 0 | 0 | 0 | 1.1754944e-38 | 2.2250738585072014e-308 - 127 | 32767 | 2147483647 | 9223372036854775807 | 255 | 65535 | 4294967295 | | 3.4028235e+38 | 1.7976931348623157e+308 -(2 rows) - -DROP USER MAPPING FOR CURRENT_USER SERVER import_loopback; -DROP USER MAPPING FOR CURRENT_USER SERVER import_loopback_bin; -SELECT clickhouse_raw_query('DROP DATABASE import_test'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('DROP DATABASE import_test_2'); - clickhouse_raw_query ----------------------- - -(1 row) - -DROP SERVER import_loopback_bin CASCADE; -NOTICE: drop cascades to 7 other objects -DETAIL: drop cascades to foreign table clickhouse_bin.arrays -drop cascades to foreign table clickhouse_bin.ints -drop cascades to foreign table clickhouse_bin.ip -drop cascades to foreign table clickhouse_bin.timezones -drop cascades to foreign table clickhouse_bin.tuples -drop cascades to foreign table clickhouse_bin.types -drop cascades to foreign table clickhouse_bin.types2 -DROP SERVER import_loopback CASCADE; -NOTICE: drop cascades to 15 other objects -DETAIL: drop cascades to foreign table clickhouse.arrays -drop cascades to foreign table clickhouse.ints -drop cascades to foreign table clickhouse.ip -drop cascades to foreign table clickhouse.timezones -drop cascades to foreign table clickhouse.tuples -drop cascades to foreign table clickhouse.types -drop cascades to foreign table clickhouse.types2 -drop cascades to foreign table clickhouse_limit.ints -drop cascades to foreign table clickhouse_limit.types -drop cascades to foreign table clickhouse_except.arrays -drop cascades to foreign table clickhouse_except.ip -drop cascades to foreign table clickhouse_except.timezones -drop cascades to foreign table clickhouse_except.tuples -drop cascades to foreign table clickhouse_except.types2 -drop cascades to foreign table clickhouse.custom_option diff --git a/test/expected/json_3.out b/test/expected/json_3.out index b7a968f2..ee946eb3 100644 --- a/test/expected/json_3.out +++ b/test/expected/json_3.out @@ -58,7 +58,7 @@ INSERT INTO json_http.things VALUES (4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}') ; SELECT * FROM json_bin.things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.things ORDER BY id; id | data @@ -82,7 +82,7 @@ ERROR: pg_clickhouse: could not finish INSERT - DB::Exception: Invalid version INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; id | data @@ -123,7 +123,7 @@ SELECT data['name'] FROM json_bin.things; (3 rows) SELECT data['name'] FROM json_bin.things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- DISTINCT forces an ORDER BY or HashAgg; the subscript must stay local. EXPLAIN (VERBOSE, COSTS OFF) @@ -158,7 +158,7 @@ SELECT DISTINCT data['size'] FROM json_bin.things; (6 rows) SELECT DISTINCT data['size'] FROM json_bin.things; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT data FROM json_test.things -- GROUP BY with a JSON subscript expression. EXPLAIN (VERBOSE, COSTS OFF) @@ -193,7 +193,7 @@ SELECT data['size'], count(*) FROM json_bin.things GROUP BY data['size']; (6 rows) SELECT data['size'], count(*) FROM json_bin.things GROUP BY data['size']; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT data FROM json_test.things -- The jsonb ->> operator runs locally when used in SELECT target lists. EXPLAIN (VERBOSE, COSTS OFF) @@ -223,7 +223,7 @@ SELECT data ->> 'name' FROM json_bin.things; (3 rows) SELECT data ->> 'name' FROM json_bin.things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- The json ->> operator runs locally when used in SELECT target lists. EXPLAIN (VERBOSE, COSTS OFF) @@ -253,7 +253,7 @@ SELECT data ->> 'name' FROM json_bin.json_things; (3 rows) SELECT data ->> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb ->> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) @@ -454,12 +454,12 @@ SELECT * FROM json_http.things SELECT * FROM json_bin.things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST SELECT * FROM json_bin.things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_things @@ -483,12 +483,12 @@ SELECT * FROM json_http.json_things SELECT * FROM json_bin.json_things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST SELECT * FROM json_bin.json_things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST -- ORDER BY with jsonb ->> pushdown. EXPLAIN (VERBOSE, COSTS OFF) @@ -518,7 +518,7 @@ SELECT * FROM json_bin.things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.things ORDER BY data ->> 'name'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.things ORDER BY data ->> 'name' LIMIT 2; @@ -557,7 +557,7 @@ SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.json_things ORDER BY data ->> 'name' LIMIT 2; @@ -596,7 +596,7 @@ SELECT data -> 'name' FROM json_bin.things; (3 rows) SELECT data -> 'name' FROM json_bin.things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_http.json_things; @@ -625,7 +625,7 @@ SELECT data -> 'name' FROM json_bin.json_things; (3 rows) SELECT data -> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) @@ -711,7 +711,7 @@ SELECT * FROM json_bin.things WHERE data -> 'stocked' = 'true'::jsonb; (3 rows) SELECT * FROM json_bin.things WHERE data -> 'stocked' = 'true'::jsonb ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE ((toJSONString(data.stocked) = 'true')) ORDER BY id ASC NULLS LAST -- WHERE clause with json -> JSON boolean literal must push down. EXPLAIN (VERBOSE, COSTS OFF) @@ -739,7 +739,7 @@ SELECT * FROM json_bin.json_things WHERE (data -> 'stocked')::text = 'true'; (3 rows) SELECT * FROM json_bin.json_things WHERE (data -> 'stocked')::text = 'true' ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE ((CAST(toJSONString(data.stocked) AS String) = 'true')) ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> wraps the dot notation in toJSONString() so the -- result is a proper JSON value (->, unlike ->>, returns jsonb). @@ -852,7 +852,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; @@ -879,7 +879,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) -- Key with mixed case. EXPLAIN (VERBOSE, COSTS OFF) @@ -907,7 +907,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; @@ -934,7 +934,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) -- Key that is a SQL reserved word. EXPLAIN (VERBOSE, COSTS OFF) @@ -962,7 +962,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; @@ -989,7 +989,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) -- Key containing embedded double quotes. EXPLAIN (VERBOSE, COSTS OFF) @@ -1396,7 +1396,7 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) @@ -1425,7 +1425,7 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: jsonb_extract_path (returns jsonb, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) @@ -1454,7 +1454,7 @@ SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level jsonb_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) @@ -1589,7 +1589,7 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) @@ -1618,7 +1618,7 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_event (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events ORDER BY id; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: json_extract_path (returns json, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) @@ -1647,7 +1647,7 @@ SELECT json_extract_path(props, 'address') FROM json_bin.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; -ERROR: pg_clickhouse: Unsupported JSON serialization version. Make sure output_format_native_write_json_as_string=1 is set. +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level json_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) diff --git a/test/expected/json_4.out b/test/expected/json_4.out index 1a2eca07..e9d22520 100644 --- a/test/expected/json_4.out +++ b/test/expected/json_4.out @@ -71,7 +71,8 @@ ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Objec INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; ERROR: invalid input syntax for type json DETAIL: Token "(" is invalid. @@ -179,7 +180,8 @@ SELECT data ->> 'name' FROM json_bin.json_things; (3 rows) SELECT data ->> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb ->> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data ->> 'name' = 'widget'; @@ -439,7 +441,8 @@ SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.json_things ORDER BY data ->> 'name' LIMIT 2; ERROR: pg_clickhouse: Code: 115. DB::Exception: Setting allow_suspicious_types_in_order_by is neither a builtin setting nor started with the prefix 'SQL_' registered for user-defined settings. (UNKNOWN_SETTING) @@ -491,7 +494,8 @@ SELECT data -> 'name' FROM json_bin.json_things; (3 rows) SELECT data -> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data -> 'name' = '"widget"'; @@ -677,7 +681,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; @@ -702,7 +706,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) -- Key with mixed case. EXPLAIN (VERBOSE, COSTS OFF) @@ -728,7 +732,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; @@ -753,7 +757,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) -- Key that is a SQL reserved word. EXPLAIN (VERBOSE, COSTS OFF) @@ -779,7 +783,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; @@ -804,7 +808,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) -- Key containing embedded double quotes. EXPLAIN (VERBOSE, COSTS OFF) @@ -1172,7 +1176,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; @@ -1197,7 +1202,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: jsonb_extract_path (returns jsonb, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; @@ -1222,7 +1228,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level jsonb_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.events WHERE jsonb_extract_path_text(props, 'customerId') = 'C100'; @@ -1353,7 +1360,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events; @@ -1378,7 +1386,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_event (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: json_extract_path (returns json, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_http.json_events; @@ -1403,7 +1412,8 @@ SELECT json_extract_path(props, 'address') FROM json_bin.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level json_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.json_events WHERE json_extract_path_text(props, 'customerId') = 'C100'; diff --git a/test/expected/json_5.out b/test/expected/json_5.out index 716a1df2..05b65248 100644 --- a/test/expected/json_5.out +++ b/test/expected/json_5.out @@ -71,7 +71,8 @@ ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Objec INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; ERROR: invalid input syntax for type json DETAIL: Token "(" is invalid. @@ -179,7 +180,8 @@ SELECT data ->> 'name' FROM json_bin.json_things; (3 rows) SELECT data ->> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb ->> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data ->> 'name' = 'widget'; @@ -439,7 +441,8 @@ SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.json_things ORDER BY data ->> 'name' LIMIT 2; ERROR: pg_clickhouse: Code: 115. DB::Exception: Setting allow_suspicious_types_in_order_by is neither a builtin setting nor started with the prefix 'SQL_' registered for user-defined settings. (UNKNOWN_SETTING) @@ -491,7 +494,8 @@ SELECT data -> 'name' FROM json_bin.json_things; (3 rows) SELECT data -> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data -> 'name' = '"widget"'; @@ -677,7 +681,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; @@ -702,7 +706,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) -- Key with mixed case. EXPLAIN (VERBOSE, COSTS OFF) @@ -728,7 +732,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; @@ -753,7 +757,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) -- Key that is a SQL reserved word. EXPLAIN (VERBOSE, COSTS OFF) @@ -779,7 +783,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; @@ -804,7 +808,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) -- Key containing embedded double quotes. EXPLAIN (VERBOSE, COSTS OFF) @@ -1172,7 +1176,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; @@ -1197,7 +1202,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: jsonb_extract_path (returns jsonb, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; @@ -1222,7 +1228,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level jsonb_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.events WHERE jsonb_extract_path_text(props, 'customerId') = 'C100'; @@ -1353,7 +1360,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events; @@ -1378,7 +1386,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_event (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: json_extract_path (returns json, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_http.json_events; @@ -1403,7 +1412,8 @@ SELECT json_extract_path(props, 'address') FROM json_bin.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level json_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.json_events WHERE json_extract_path_text(props, 'customerId') = 'C100'; diff --git a/test/expected/json_6.out b/test/expected/json_6.out index 3ba355f3..1aa614e4 100644 --- a/test/expected/json_6.out +++ b/test/expected/json_6.out @@ -71,7 +71,8 @@ ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Objec INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; ERROR: invalid input syntax for type json DETAIL: Token "(" is invalid. @@ -179,7 +180,8 @@ SELECT data ->> 'name' FROM json_bin.json_things; (3 rows) SELECT data ->> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb ->> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data ->> 'name' = 'widget'; @@ -439,7 +441,8 @@ SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST SET pg_clickhouse.session_settings TO 'allow_suspicious_types_in_order_by 1'; SELECT * FROM json_http.json_things ORDER BY data ->> 'name' LIMIT 2; ERROR: pg_clickhouse: Code: 115. DB::Exception: Unknown setting allow_suspicious_types_in_order_by. (UNKNOWN_SETTING) @@ -491,7 +494,8 @@ SELECT data -> 'name' FROM json_bin.json_things; (3 rows) SELECT data -> 'name' FROM json_bin.json_things ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- WHERE clause with jsonb -> equality must be pushed down. EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.things WHERE data -> 'name' = '"widget"'; @@ -677,7 +681,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; @@ -702,7 +706,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) -- Key with mixed case. EXPLAIN (VERBOSE, COSTS OFF) @@ -728,7 +732,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; @@ -753,7 +757,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) -- Key that is a SQL reserved word. EXPLAIN (VERBOSE, COSTS OFF) @@ -779,7 +783,7 @@ SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; @@ -804,7 +808,7 @@ SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: pg_clickhouse: unsupported column type: Tuple(CamelCase String, `my field` String, select String) +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) -- Key containing embedded double quotes. EXPLAIN (VERBOSE, COSTS OFF) @@ -1172,7 +1176,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; @@ -1197,7 +1202,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events ORDER BY id; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: jsonb_extract_path (returns jsonb, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; @@ -1222,7 +1228,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; -ERROR: type jsonb is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level jsonb_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.events WHERE jsonb_extract_path_text(props, 'customerId') = 'C100'; @@ -1353,7 +1360,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: multi-level path, still evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events; @@ -1378,7 +1386,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_event (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events ORDER BY id; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST -- Target-list: json_extract_path (returns json, not text), evaluated locally. EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_http.json_events; @@ -1403,7 +1412,8 @@ SELECT json_extract_path(props, 'address') FROM json_bin.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; -ERROR: type json is not composite +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events -- WHERE: single-level json_extract_path_text pushes down as dot notation. EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM json_http.json_events WHERE json_extract_path_text(props, 'customerId') = 'C100'; diff --git a/test/expected/result_map.txt b/test/expected/result_map.txt index de31d92e..9bbe45f2 100644 --- a/test/expected/result_map.txt +++ b/test/expected/result_map.txt @@ -46,7 +46,7 @@ binary_inserts.sql ClickHouse | File ------------|-------------------- - 22-26 | binary_inserts.out + 23+ | binary_inserts.out binary_queries.sql ------------------ @@ -61,10 +61,8 @@ binary_queries.sql ClickHouse | File ------------|---------------------- 26 | binary_queries.out - 25 | binary_queries_4.out - 23-24 | binary_queries_5.out - 23 | binary_queries_6.out - 22 | binary_queries_7.out + 23.8-25 | binary_queries_4.out + 23.3 | binary_queries_5.out deparse_checks.sql ------------------ @@ -76,7 +74,7 @@ deparse_checks.sql ClickHouse | File ------------|-------------------- - 22+ | deparse_checks.out + 23+ | deparse_checks.out engines.sql ----------- @@ -88,7 +86,7 @@ engines.sql ClickHouse | File ------------|------------- - 22+ | engines.out + 23+ | engines.out functions.sql ------------- @@ -158,7 +156,6 @@ import_schema.sql ClickHouse | File ------------|--------------------- 23+ | import_schema.out - 22 | import_schema_2.out json.sql -------- @@ -223,7 +220,7 @@ subquery_pushdown.sql ClickHouse | File ------------|----------------------- - 22+ | subquery_pushdown.out + 23+ | subquery_pushdown.out timezone.sql ------------ @@ -247,7 +244,7 @@ where_sub.sql ClickHouse | File ------------|----------------- - 22+ | where_sub.out + 23+ | where_sub.out window_functions.sql -------------------- diff --git a/test/sql/import_schema.sql b/test/sql/import_schema.sql index 34e6c8e0..211c051f 100644 --- a/test/sql/import_schema.sql +++ b/test/sql/import_schema.sql @@ -12,6 +12,7 @@ CREATE USER MAPPING FOR CURRENT_USER SERVER import_loopback_bin; SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test'); SELECT clickhouse_raw_query('CREATE DATABASE import_test'); +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS import_test_2'); SELECT clickhouse_raw_query('CREATE DATABASE import_test_2'); -- integer types diff --git a/vendor/clickhouse-c b/vendor/clickhouse-c new file mode 160000 index 00000000..5df8f6e8 --- /dev/null +++ b/vendor/clickhouse-c @@ -0,0 +1 @@ +Subproject commit 5df8f6e8ee2c978a808b96c5ccc1142c73fe6d49 diff --git a/vendor/clickhouse-cpp b/vendor/clickhouse-cpp deleted file mode 160000 index f067937e..00000000 --- a/vendor/clickhouse-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f067937e47ff75767709b7471821c6cefbea7666