diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..d96c588 --- /dev/null +++ b/.clang-format @@ -0,0 +1,319 @@ +--- +Language: Cpp +AlignAfterOpenBracket: true +AccessModifierOffset: -2 +AlignArrayOfStructures: Left +AlignConsecutiveAssignments: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionDeclarations: false + AlignFunctionPointers: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionDeclarations: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveDeclarations: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionDeclarations: true + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveMacros: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionDeclarations: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveShortCaseStatements: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCaseArrows: false + AlignCaseColons: false +AlignConsecutiveTableGenBreakingDAGArgColons: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionDeclarations: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveTableGenCondOperatorColons: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionDeclarations: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveTableGenDefinitionColons: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionDeclarations: false + AlignFunctionPointers: false + PadOperators: false +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: + AlignPPAndNotPP: true + Kind: Always + OverEmptyLines: 0 +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowBreakBeforeNoexceptSpecifier: Never +AllowBreakBeforeQtProperty: false +AllowShortBlocksOnASingleLine: Always +AllowShortCaseExpressionOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: false +AllowShortCompoundRequirementOnASingleLine: true +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: false +AllowShortNamespacesOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: TopLevel +AlwaysBreakBeforeMultilineStrings: false +AttributeMacros: + - __capability +BinPackArguments: false +BinPackLongBracedList: true +BinPackParameters: OnePerLine +BitFieldColonSpacing: Both +BracedInitializerIndentWidth: -1 +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterExternBlock: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: true +BreakAdjacentStringLiterals: true +BreakAfterAttributes: Leave +BreakAfterJavaFieldAnnotations: false +BreakAfterOpenBracketBracedList: false +BreakAfterOpenBracketFunction: true +BreakAfterOpenBracketIf: true +BreakAfterOpenBracketLoop: false +BreakAfterOpenBracketSwitch: false +BreakAfterReturnType: TopLevel +BreakArrays: true +BreakBeforeBinaryOperators: None +BreakBeforeCloseBracketBracedList: true +BreakBeforeCloseBracketFunction: true +BreakBeforeCloseBracketIf: true +BreakBeforeCloseBracketLoop: false +BreakBeforeCloseBracketSwitch: false +BreakBeforeConceptDeclarations: Always +BreakBeforeBraces: Custom +BreakBeforeInlineASMColon: OnlyMultiline +BreakBeforeTemplateCloser: false +BreakBeforeTernaryOperators: true +BreakBinaryOperations: Never +BreakConstructorInitializers: BeforeComma +BreakFunctionDefinitionParameters: false +BreakInheritanceList: BeforeComma +BreakStringLiterals: true +BreakTemplateDeclarations: Yes +ColumnLimit: 88 +CommentPragmas: "^ IWYU pragma:" +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: Block +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +EnumTrailingComma: Leave +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: false +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: ".*" + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: "(Test)?$" +IncludeIsMainSourceRegex: "" +IndentAccessModifiers: false +IndentCaseBlocks: false +IndentCaseLabels: false +IndentExportBlock: true +IndentExternBlock: AfterExternBlock +IndentGotoLabels: true +IndentPPDirectives: None +IndentRequiresClause: true +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertBraces: true +InsertNewlineAtEOF: true +InsertTrailingCommas: None +IntegerLiteralSeparator: + Binary: 0 + BinaryMinDigitsInsert: 0 + BinaryMaxDigitsRemove: 0 + Decimal: 0 + DecimalMinDigitsInsert: 0 + DecimalMaxDigitsRemove: 0 + Hex: 0 + HexMinDigitsInsert: 0 + HexMaxDigitsRemove: 0 + BinaryMinDigits: 0 + DecimalMinDigits: 0 + HexMinDigits: 0 +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLines: + AtEndOfFile: false + AtStartOfBlock: true + AtStartOfFile: true +KeepFormFeed: false +LambdaBodyIndentation: Signature +LineEnding: DeriveLF +MacroBlockBegin: "" +MacroBlockEnd: "" +MainIncludeChar: Quote +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +NumericLiteralCase: + ExponentLetter: Leave + HexDigit: Leave + Prefix: Leave + Suffix: Leave +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 4 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: false +OneLineFormatOffRegex: "" +PackConstructorInitializers: BinPack +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakBeforeMemberAccess: 150 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakScopeResolution: 500 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyIndentedWhitespace: 0 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +PPIndentWidth: -1 +QualifierAlignment: Leave +ReferenceAlignment: Pointer +ReflowComments: Always +RemoveBracesLLVM: false +RemoveEmptyLinesInUnwrappedLines: false +RemoveParentheses: Leave +RemoveSemicolon: false +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SkipMacroDefinitionBody: false +SortIncludes: + Enabled: true + IgnoreCase: false + IgnoreExtension: false +SortJavaStaticImport: Before +SortUsingDeclarations: LexicographicNumeric +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterOperatorKeyword: false +SpaceAfterTemplateKeyword: false +SpaceAroundPointerQualifiers: Default +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeJsonColon: false +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterNot: false + AfterOverloadedOperator: false + AfterPlacementOperator: true + AfterRequiresInClause: false + AfterRequiresInExpression: false + BeforeNonEmptyParentheses: false +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBraces: Never +SpacesBeforeTrailingComments: 1 +SpacesInAngles: Never +SpacesInContainerLiterals: true +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParens: Never +SpacesInParensOptions: + ExceptDoubleParentheses: false + InCStyleCasts: false + InConditionalStatements: false + InEmptyParentheses: false + Other: false +SpacesInSquareBrackets: false +Standard: Latest +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TableGenBreakInsideDAGArg: DontBreak +TabWidth: 4 +UseTab: Never +VerilogBreakBetweenInstancePorts: true +WhitespaceSensitiveMacros: + - BOOST_PP_STRINGIZE + - CF_SWIFT_NAME + - NS_SWIFT_NAME + - PP_STRINGIZE + - STRINGIZE +WrapNamespaceBodyWithEmptyLines: Leave diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9668e89..0d01007 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,13 +24,12 @@ repos: rev: v2.4.1 hooks: - id: codespell - - repo: local + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v22.1.5 hooks: - - id: indent - name: indent code - language: system - entry: ./dev/indent.sh - pass_filenames: false + - id: clang-format + name: format code + types: [c] - repo: local hooks: - id: clang-tidy diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eedd49..15ff7f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,12 @@ All notable changes to this project will be documented in this file. It uses the argument is not a constant. * Fixed a memory leak in the http driver when not using streaming ([#281]). +### 🏗️ Build Setup + +* Replaced code formatting, previously performed by `pg_bsd_indent`, with + [clang-format] using a modified "Mozilla style" with an emphasis on + legibility and safety ([#283]). + [#227]: https://github.com/ClickHouse/pg_clickhouse/pull/227 "ClickHouse/pg_clickhouse#227 add three-state secure option for TLS control" [#268]: https://github.com/ClickHouse/pg_clickhouse/pull/268 @@ -52,6 +58,9 @@ All notable changes to this project will be documented in this file. It uses the "ClickHouse/pg_clickhouse#269 Add support for PostgreSQL 19beta1" [#218]: https://github.com/ClickHouse/pg_clickhouse/pull/218 "ClickHouse/pg_clickhouse#218 Properly free Curl memory on palloc error" + [clang-format]: https://clang.llvm.org/docs/ClangFormat.html + [#283]: https://github.com/ClickHouse/pg_clickhouse/pull/283 + "ClickHouse/pg_clickhouse#283 clang-format" ## [v0.3.1] — 2026-05-02 diff --git a/dev/README.md b/dev/README.md index 911e9b8..6d3d49b 100644 --- a/dev/README.md +++ b/dev/README.md @@ -6,7 +6,6 @@ development of pg_clickhouse. * `bear.json`: [Bear Configuration](#bear-configuration) * `bear.yml`: [Bear Configuration](#bear-configuration) * `docker-compose.yml`: [ClickHouse Version Testing Containers](#clickhouse-version-testing-containers) -* `indent.sh`: Run by `make indent`; requires [pg_bsd_indent] * `Makefile`: Automates [ClickHouse Version Testing Containers](#clickhouse-version-testing-containers) * `README.md`: This file * `runch`: [Run ClickHouse Server](#run-clickhouse-server) @@ -87,7 +86,6 @@ There are currently two files: The `compile_commands.json` target determines which to use. - [pg_bsd_indent]: https://github.com/postgres/postgres/tree/master/src/tools/pg_bsd_indent [clickhousectl]: https://clickhouse.com/docs/interfaces/cli "ClickHouse Docs: clickhousectl" [pgxn-tools]: https://github.com/pgxn/docker-pgxn-tools/ [Bear]: https://github.com/rizsotto/Bear "Bear generates a compilation database for Clang tooling" diff --git a/src/binary/binary.c b/src/binary/binary.c index 525a403..3a2d27e 100644 --- a/src/binary/binary.c +++ b/src/binary/binary.c @@ -22,47 +22,55 @@ #define CHC_IMPLEMENTATION #define CHC_NO_ASYNC -#include "clickhouse.h" +#include "clickhouse-client.h" #include "clickhouse-compression.h" -#include "clickhouse-posix-io.h" #include "clickhouse-openssl.h" -#include "clickhouse-client.h" +#include "clickhouse-posix-io.h" +#include "clickhouse.h" #include "binary_internal.h" -static void * -pg_chc_palloc(void *ud pg_attribute_unused(), size_t n) -{ - return MemoryContextAllocExtended(CurrentMemoryContext, n, MCXT_ALLOC_HUGE); +static void* +pg_chc_palloc(void* ud pg_attribute_unused(), size_t n) { + return MemoryContextAllocExtended(CurrentMemoryContext, n, MCXT_ALLOC_HUGE); } -static void * -pg_chc_repalloc(void *ud pg_attribute_unused(), void *p, - size_t old_bytes pg_attribute_unused(), size_t new_bytes) -{ - if (!p) - return MemoryContextAllocExtended(CurrentMemoryContext, new_bytes, MCXT_ALLOC_HUGE); - return repalloc_huge(p, new_bytes); +static void* +pg_chc_repalloc( + void* ud pg_attribute_unused(), + void* p, + size_t old_bytes pg_attribute_unused(), + size_t new_bytes +) { + if (!p) { + return MemoryContextAllocExtended( + CurrentMemoryContext, new_bytes, MCXT_ALLOC_HUGE + ); + } + return repalloc_huge(p, new_bytes); } static void -pg_chc_pfree(void *ud pg_attribute_unused(), void *p, size_t bytes pg_attribute_unused()) -{ - if (p) - pfree(p); +pg_chc_pfree( + void* ud pg_attribute_unused(), + void* p, + size_t bytes pg_attribute_unused() +) { + if (p) { + pfree(p); + } } -const chc_alloc pg_chc_alloc = { - .ud = NULL, - .alloc = pg_chc_palloc, - .realloc = pg_chc_repalloc, - .free = pg_chc_pfree, +const chc_alloc pg_chc_alloc = { + .ud = NULL, + .alloc = pg_chc_palloc, + .realloc = pg_chc_repalloc, + .free = pg_chc_pfree, }; /* power-of-10 lookup; CH bounds DateTime64 / Decimal scale to [0, 9] */ -const int64_t pow10i[10] = { - 1, 10, 100, 1000, 10000, 100000, 1000000, - 10000000, 100000000, 1000000000 +const int64_t pow10i[10] = { + 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, }; /* @@ -71,25 +79,27 @@ const int64_t pow10i[10] = { * fail the query, so gate. */ bool -server_supports_json_as_string(const chc_client * c) -{ - const chc_server_info *info = chc_client_server_info(c); +server_supports_json_as_string(const chc_client* c) { + const chc_server_info* info = chc_client_server_info(c); - if (!info) - return false; - if (info->version_major > 24) - return true; - if (info->version_major == 24 && info->version_minor >= 10) - return true; - return false; + if (!info) { + return false; + } + if (info->version_major > 24) { + return true; + } + if (info->version_major == 24 && info->version_minor >= 10) { + return true; + } + return false; } void -raise_chc(const chc_err * err, int sqlstate, const char *prefix) -{ - const char *m = (err && err->msg[0]) ? err->msg : "unknown error"; +raise_chc(const chc_err* err, int sqlstate, const char* prefix) { + const char* m = (err && err->msg[0]) ? err->msg : "unknown error"; - ereport(ERROR, - (errcode(sqlstate), - errmsg("pg_clickhouse: %s%s", prefix ? prefix : "", m))); + ereport( + ERROR, + (errcode(sqlstate), errmsg("pg_clickhouse: %s%s", prefix ? prefix : "", m)) + ); } diff --git a/src/binary/binary_internal.h b/src/binary/binary_internal.h index 4f751b6..d1fcf83 100644 --- a/src/binary/binary_internal.h +++ b/src/binary/binary_internal.h @@ -12,10 +12,10 @@ #include -#include "clickhouse.h" #include "clickhouse-client.h" -#include "clickhouse-posix-io.h" #include "clickhouse-openssl.h" +#include "clickhouse-posix-io.h" +#include "clickhouse.h" #include "binary.h" #include "internal.h" @@ -24,14 +24,13 @@ #define pg_noreturn pg_attribute_noreturn() #endif -typedef struct -{ - Datum *datums; - bool *nulls; - size_t len; - Oid *types; - const char *ch_type_name; -} ch_binary_tuple_t; +typedef struct { + Datum* datums; + bool* nulls; + size_t len; + Oid* types; + const char* ch_type_name; +} ch_binary_tuple_t; /* * Holds an array decoded from ClickHouse or built for INSERT. For nested @@ -40,23 +39,21 @@ typedef struct * array_type is postgres array type (same across nesting depths since * postgres uses one array type per element type regardless of ndim). */ -typedef struct -{ - Datum *datums; - bool *nulls; - size_t len; - int ndim; /* nesting depth, >=1 */ - Oid item_type; /* leaf scalar PG type */ - Oid array_type; /* PG array type (same at every level) */ -} ch_binary_array_t; +typedef struct { + Datum* datums; + bool* nulls; + size_t len; + int ndim; /* nesting depth, >=1 */ + Oid item_type; /* leaf scalar PG type */ + Oid array_type; /* PG array type (same at every level) */ +} ch_binary_array_t; /* Column metadata returned from ch_binary_begin_insert. */ -typedef struct ch_binary_column_info -{ - const char *name; - const chc_type *type; /* type unwrapped of Nullable + LowCardinality */ - bool is_nullable; -} ch_binary_column_info; +typedef struct ch_binary_column_info { + const char* name; + const chc_type* type; /* type unwrapped of Nullable + LowCardinality */ + bool is_nullable; +} ch_binary_column_info; /* * Pump next non-empty Data block off wire. Returned pointer is borrowed, @@ -64,19 +61,24 @@ typedef struct ch_binary_column_info * stream ends (eos, error, canceled), ch_binary_response_error reports * cause if any. */ -extern const chc_block *ch_binary_response_fetch_next_block(ch_binary_response_t * resp); +extern const chc_block* +ch_binary_response_fetch_next_block(ch_binary_response_t* resp); -extern ch_binary_insert_handle * ch_binary_begin_insert(ch_binary_connection_t * conn, - const ch_query * query, - ch_binary_column_info * *out_cols, - size_t * out_n); +extern ch_binary_insert_handle* +ch_binary_begin_insert( + ch_binary_connection_t* conn, + const ch_query* query, + ch_binary_column_info** out_cols, + size_t* out_n +); /* * Tear down handle. Never raises and never talks to server, safe to call * from a MemoryContext reset callback during transaction abort. Flags * connection broken if finalize did not run. */ -extern void ch_binary_release_insert(ch_binary_insert_handle * h); +extern void +ch_binary_release_insert(ch_binary_insert_handle* h); /* * Per-connection state smuggled through ch_binary_connection_t.client. @@ -86,49 +88,47 @@ extern void ch_binary_release_insert(ch_binary_insert_handle * h); * pg_chc_alloc routes through CurrentMemoryContext, so blocks land in * whichever per-query context the caller has switched to. */ -struct ch_binary_state -{ - /* - * Connection-lifetime context; holds this struct, the chc_client, and - * chc_client's initial read buffer. Deleted by ch_binary_close. - */ - MemoryContext cxt; - - /* - * Registered on cxt; closes fd / SSL on reset so half-built connections - * release OS resources when PG_CATCH deletes cxt. - */ - MemoryContextCallback reset_cb; - - chc_client *client; - /* Transport vtable; backed by posix_state or openssl_state below. */ - chc_io io; - - /* Compression codec used by chc_client. */ - chc_codec codec; - - int fd; /* -1 once closed by reset_cb */ - SSL_CTX *ssl_ctx; - SSL *ssl; - bool tls; - chc_posix_io posix_state; - chc_openssl_io openssl_state; - - /* Per-query cancel callback (no userdata). */ - bool (*check_cancel_fn) (void); - - /* - * Set when an unrecoverable protocol/IO error happened (server raised an - * exception mid-INSERT and closed the socket, write hit EPIPE, etc). - * Cache layer checks via ch_binary_is_broken & drops the entry. - */ - bool broken; +struct ch_binary_state { + /* + * Connection-lifetime context; holds this struct, the chc_client, and + * chc_client's initial read buffer. Deleted by ch_binary_close. + */ + MemoryContext cxt; + + /* + * Registered on cxt; closes fd / SSL on reset so half-built connections + * release OS resources when PG_CATCH deletes cxt. + */ + MemoryContextCallback reset_cb; + + chc_client* client; + /* Transport vtable; backed by posix_state or openssl_state below. */ + chc_io io; + + /* Compression codec used by chc_client. */ + chc_codec codec; + + int fd; /* -1 once closed by reset_cb */ + SSL_CTX* ssl_ctx; + SSL* ssl; + bool tls; + chc_posix_io posix_state; + chc_openssl_io openssl_state; + + /* Per-query cancel callback (no userdata). */ + bool (*check_cancel_fn)(void); + + /* + * Set when an unrecoverable protocol/IO error happened (server raised an + * exception mid-INSERT and closed the socket, write hit EPIPE, etc). + * Cache layer checks via ch_binary_is_broken & drops the entry. + */ + bool broken; }; -static inline struct ch_binary_state * -conn_state(ch_binary_connection_t * conn) -{ - return (struct ch_binary_state *) conn->client; +static inline struct ch_binary_state* +conn_state(ch_binary_connection_t* conn) { + return (struct ch_binary_state*)conn->client; } /* chc allocator wired through palloc; defined in binary.c. */ @@ -141,63 +141,113 @@ extern const Oid ch_scalar_oids[CHC_KIND_COUNT]; extern const int64_t pow10i[10]; /* ereport ERROR carrying chc_err->msg with sqlstate / prefix. */ -pg_noreturn extern void raise_chc(const chc_err * err, int sqlstate, - const char *prefix); +pg_noreturn extern void +raise_chc(const chc_err* err, int sqlstate, const char* prefix); /* True if server advertises output_format_native_write_json_as_string. */ -extern bool server_supports_json_as_string(const chc_client * c); +extern bool +server_supports_json_as_string(const chc_client* c); /* * Per-row, per-column append. Set isnull for NULL values (column must be * Nullable). ereports on type mismatch / NULL-into-NOT-NULL. */ -extern void ch_binary_append_int(ch_binary_insert_handle * h, size_t col, - int64_t val, bool isnull); -extern void ch_binary_append_uint(ch_binary_insert_handle * h, size_t col, - uint64_t val, bool isnull); -extern void ch_binary_append_bool(ch_binary_insert_handle * h, size_t col, - bool val, bool isnull); -extern void ch_binary_append_double(ch_binary_insert_handle * h, size_t col, - double val, bool isnull); -extern void ch_binary_append_float(ch_binary_insert_handle * h, size_t col, - float val, bool isnull); -extern void ch_binary_append_bytes(ch_binary_insert_handle * h, size_t col, - const void *p, size_t n, bool isnull); -extern void ch_binary_append_decimal(ch_binary_insert_handle * h, size_t col, - const char *digits, bool isnull); -extern void ch_binary_append_uuid(ch_binary_insert_handle * h, size_t col, - const uint8_t bytes[16], bool isnull); +extern void +ch_binary_append_int(ch_binary_insert_handle* h, size_t col, int64_t val, bool isnull); +extern void +ch_binary_append_uint( + ch_binary_insert_handle* h, + size_t col, + uint64_t val, + bool isnull +); +extern void +ch_binary_append_bool(ch_binary_insert_handle* h, size_t col, bool val, bool isnull); +extern void +ch_binary_append_double( + ch_binary_insert_handle* h, + size_t col, + double val, + bool isnull +); +extern void +ch_binary_append_float(ch_binary_insert_handle* h, size_t col, float val, bool isnull); +extern void +ch_binary_append_bytes( + ch_binary_insert_handle* h, + size_t col, + const void* p, + size_t n, + bool isnull +); +extern void +ch_binary_append_decimal( + ch_binary_insert_handle* h, + size_t col, + const char* digits, + bool isnull +); +extern void +ch_binary_append_uuid( + ch_binary_insert_handle* h, + size_t col, + const uint8_t bytes[16], + bool isnull +); /* * IPv4: addr_be is 4 BE bytes (matches PG inet ip_addr layout). * IPv6: addr_be is 16 BE bytes. Pass NULL with isnull=true. */ -extern void ch_binary_append_inet(ch_binary_insert_handle * h, size_t col, - const uint8_t * addr_be, size_t addrlen, - bool isnull); +extern void +ch_binary_append_inet( + ch_binary_insert_handle* h, + size_t col, + const uint8_t* addr_be, + size_t addrlen, + bool isnull +); /* * Per-row Date/DateTime/DateTime64 sent as seconds-since-epoch (int64). * For DateTime64 value is wire-level integer at column's scale; * encode.c does scaling. */ -extern void ch_binary_append_date_seconds(ch_binary_insert_handle * h, size_t col, - int64_t seconds, bool isnull); -extern void ch_binary_append_datetime_seconds(ch_binary_insert_handle * h, size_t col, - int64_t seconds, bool isnull); -extern void ch_binary_append_datetime64_raw(ch_binary_insert_handle * h, size_t col, - int64_t raw, bool isnull); +extern void +ch_binary_append_date_seconds( + ch_binary_insert_handle* h, + size_t col, + int64_t seconds, + bool isnull +); +extern void +ch_binary_append_datetime_seconds( + ch_binary_insert_handle* h, + size_t col, + int64_t seconds, + bool isnull +); +extern void +ch_binary_append_datetime64_raw( + ch_binary_insert_handle* h, + size_t col, + int64_t raw, + bool isnull +); /* * Array element append. Open with array_begin. All subsequent * ch_binary_append_* calls target inner items column regardless of * `col` until ch_binary_array_end. */ -extern void ch_binary_array_begin(ch_binary_insert_handle * h, size_t col); -extern void ch_binary_array_end(ch_binary_insert_handle * h); +extern void +ch_binary_array_begin(ch_binary_insert_handle* h, size_t col); +extern void +ch_binary_array_end(ch_binary_insert_handle* h); /* True when handle has an active array context (for assertions). */ -extern bool ch_binary_array_active(const ch_binary_insert_handle * h); +extern bool +ch_binary_array_active(const ch_binary_insert_handle* h); /* * Inspect underlying CH column kind. Used by encode.c to @@ -205,12 +255,13 @@ extern bool ch_binary_array_active(const ch_binary_insert_handle * h); * (between ch_binary_array_begin/_end) returned kind is element kind, * not CHC_ARRAY. */ -extern chc_kind ch_binary_column_kind(const ch_binary_insert_handle * h, - size_t col); -extern uint32_t ch_binary_column_datetime64_precision(const ch_binary_insert_handle * h, - size_t col); +extern chc_kind +ch_binary_column_kind(const ch_binary_insert_handle* h, size_t col); +extern uint32_t +ch_binary_column_datetime64_precision(const ch_binary_insert_handle* h, size_t col); /* Send buffered rows and clear; ready for next batch. */ -extern void ch_binary_flush_block(ch_binary_insert_handle * h); +extern void +ch_binary_flush_block(ch_binary_insert_handle* h); -#endif /* PG_CLICKHOUSE_BINARY_INTERNAL_H */ +#endif /* PG_CLICKHOUSE_BINARY_INTERNAL_H */ diff --git a/src/binary/connection.c b/src/binary/connection.c index 57d74d9..d1cdd3b 100644 --- a/src/binary/connection.c +++ b/src/binary/connection.c @@ -11,13 +11,13 @@ #include #include -#include -#include -#include #include #include +#include +#include #include #include +#include #include "utils/memutils.h" #include "utils/palloc.h" @@ -33,11 +33,10 @@ #define CLICKHOUSE_PLAIN_PORT 9000 static bool -cancel_adapter(void *ud) -{ - struct ch_binary_state *s = ud; +cancel_adapter(void* ud) { + struct ch_binary_state* s = ud; - return s->check_cancel_fn ? s->check_cancel_fn() : false; + return s->check_cancel_fn ? s->check_cancel_fn() : false; } /* @@ -46,19 +45,23 @@ cancel_adapter(void *ud) * frame's method byte, so callers fill both codec families regardless. */ static chc_compression -parse_compression(const char *s) -{ - if (s == NULL || s[0] == '\0' || pg_strcasecmp(s, "lz4") == 0) - return CHC_COMP_LZ4; - if (pg_strcasecmp(s, "none") == 0) - return CHC_COMP_NONE; - if (pg_strcasecmp(s, "zstd") == 0) - return CHC_COMP_ZSTD; - - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), - errmsg("pg_clickhouse: invalid compression \"%s\"", s), - errhint("valid values: none, lz4, zstd"))); +parse_compression(const char* s) { + if (s == NULL || s[0] == '\0' || pg_strcasecmp(s, "lz4") == 0) { + return CHC_COMP_LZ4; + } + if (pg_strcasecmp(s, "none") == 0) { + return CHC_COMP_NONE; + } + if (pg_strcasecmp(s, "zstd") == 0) { + return CHC_COMP_ZSTD; + } + + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("pg_clickhouse: invalid compression \"%s\"", s), + errhint("valid values: none, lz4, zstd")) + ); } /* @@ -67,80 +70,81 @@ parse_compression(const char *s) * MemoryContextDelete; only fd / SSL need an explicit close. */ static void -binary_state_reset_cb(void *arg) -{ - struct ch_binary_state *s = arg; - - if (s->client) - { - chc_client_close(s->client); - s->client = NULL; - } - if (s->ssl) - { - SSL_shutdown(s->ssl); - SSL_free(s->ssl); - s->ssl = NULL; - } - if (s->ssl_ctx) - { - SSL_CTX_free(s->ssl_ctx); - s->ssl_ctx = NULL; - } - if (s->fd >= 0) - { - close(s->fd); - s->fd = -1; - } +binary_state_reset_cb(void* arg) { + struct ch_binary_state* s = arg; + + if (s->client) { + chc_client_close(s->client); + s->client = NULL; + } + if (s->ssl) { + SSL_shutdown(s->ssl); + SSL_free(s->ssl); + s->ssl = NULL; + } + if (s->ssl_ctx) { + SSL_CTX_free(s->ssl_ctx); + s->ssl_ctx = NULL; + } + if (s->fd >= 0) { + close(s->fd); + s->fd = -1; + } } static int -tcp_connect(const char *host, int port) -{ - struct addrinfo hints = {}; - struct addrinfo *res = NULL; - char port_s[16]; - - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - snprintf(port_s, sizeof(port_s), "%d", port); - int rc = getaddrinfo(host, port_s, &hints, &res); - - if (rc != 0) - ereport(ERROR, - (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), - errmsg("pg_clickhouse: getaddrinfo(%s:%d): %s", - host, port, gai_strerror(rc)))); - - int fd = -1; - int save_errno = ECONNREFUSED; - - for (struct addrinfo *ai = res; ai; ai = ai->ai_next) - { - fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); - if (fd < 0) - { - save_errno = errno; - continue; - } - if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) - break; - save_errno = errno; - close(fd); - fd = -1; - } - freeaddrinfo(res); - if (fd < 0) - ereport(ERROR, - (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), - errmsg("pg_clickhouse: connect(%s:%d): %s", - host, port, strerror(save_errno)))); - - int one = 1; - - (void) setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); - (void) setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(one)); - return fd; +tcp_connect(const char* host, int port) { + struct addrinfo hints = {}; + struct addrinfo* res = NULL; + char port_s[16]; + + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + snprintf(port_s, sizeof(port_s), "%d", port); + int rc = getaddrinfo(host, port_s, &hints, &res); + + if (rc != 0) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg( + "pg_clickhouse: getaddrinfo(%s:%d): %s", host, port, gai_strerror(rc) + )) + ); + } + + int fd = -1; + int save_errno = ECONNREFUSED; + + for (struct addrinfo* ai = res; ai; ai = ai->ai_next) { + fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) { + save_errno = errno; + continue; + } + if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) { + break; + } + save_errno = errno; + close(fd); + fd = -1; + } + freeaddrinfo(res); + if (fd < 0) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg( + "pg_clickhouse: connect(%s:%d): %s", host, port, strerror(save_errno) + )) + ); + } + + int one = 1; + + (void)setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + (void)setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(one)); + return fd; } /* @@ -148,206 +152,221 @@ tcp_connect(const char *host, int port) * Returns 0 to leave OpenSSL's default (no minimum forced). */ static int -openssl_min_proto_version(tls_version v) -{ - switch (v) - { - case CH_TLS_V1_0: - return TLS1_VERSION; - case CH_TLS_V1_1: - return TLS1_1_VERSION; - case CH_TLS_V1_2: - return TLS1_2_VERSION; - case CH_TLS_V1_3: - return TLS1_3_VERSION; - default: - return 0; - } +openssl_min_proto_version(tls_version v) { + switch (v) { + case CH_TLS_V1_0: + return TLS1_VERSION; + case CH_TLS_V1_1: + return TLS1_1_VERSION; + case CH_TLS_V1_2: + return TLS1_2_VERSION; + case CH_TLS_V1_3: + return TLS1_3_VERSION; + default: + return 0; + } } static void -tls_connect(struct ch_binary_state *s, const char *host, tls_version min_version) -{ - static int openssl_init_done = 0; - - if (!openssl_init_done) - { - SSL_library_init(); - SSL_load_error_strings(); - OpenSSL_add_all_algorithms(); - openssl_init_done = 1; - } - - s->ssl_ctx = SSL_CTX_new(TLS_client_method()); - if (!s->ssl_ctx) - ereport(ERROR, - (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), - errmsg("pg_clickhouse: failed to initialize opensssl"), - errdetail("SSL_CTX_new failed"))); - - int min_proto = openssl_min_proto_version(min_version); - - if (min_proto != 0 && - SSL_CTX_set_min_proto_version(s->ssl_ctx, min_proto) != 1) - ereport(ERROR, - (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), - errmsg("pg_clickhouse: failed to initialize openssl"), - errdetail("could not set minimum TLS protocol version"))); - /* Authenticate server: verify chain against system CAs and hostname. */ - SSL_CTX_set_verify(s->ssl_ctx, SSL_VERIFY_PEER, NULL); - if (SSL_CTX_set_default_verify_paths(s->ssl_ctx) != 1) - ereport(ERROR, - (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), - errmsg("pg_clickhouse: failed to initialize openssl"), - errdetail("could not load default CA certificates"))); - - s->ssl = SSL_new(s->ssl_ctx); - if (!s->ssl) - ereport(ERROR, - (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), - errmsg("pg_clickhouse: failed to initialize opensssl"), - errdetail("SSL_new failed"))); - SSL_set_tlsext_host_name(s->ssl, host); - SSL_set_hostflags(s->ssl, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); - if (SSL_set1_host(s->ssl, host) != 1) - ereport(ERROR, - (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), - errmsg("pg_clickhouse: failed to initialize openssl"), - errdetail("could not set certificate verification host"))); - SSL_set_fd(s->ssl, s->fd); - if (SSL_connect(s->ssl) != 1) - { - /* verification failures leave error queue empty, use verify result */ - long vr = SSL_get_verify_result(s->ssl); - unsigned long e = ERR_peek_last_error(); - char ebuf[256]; - - if (vr != X509_V_OK) - snprintf(ebuf, sizeof(ebuf), "certificate verify failed: %s", - X509_verify_cert_error_string(vr)); - else if (e) - ERR_error_string_n(e, ebuf, sizeof(ebuf)); - else - snprintf(ebuf, sizeof(ebuf), "SSL_connect failed"); - - ereport(ERROR, - (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), - errmsg("pg_clickhouse: openssl failed to connect"), - errdetail("%s", ebuf))); - } +tls_connect(struct ch_binary_state* s, const char* host, tls_version min_version) { + static int openssl_init_done = 0; + + if (!openssl_init_done) { + SSL_library_init(); + SSL_load_error_strings(); + OpenSSL_add_all_algorithms(); + openssl_init_done = 1; + } + + s->ssl_ctx = SSL_CTX_new(TLS_client_method()); + if (!s->ssl_ctx) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize opensssl"), + errdetail("SSL_CTX_new failed")) + ); + } + + int min_proto = openssl_min_proto_version(min_version); + + if (min_proto != 0 && SSL_CTX_set_min_proto_version(s->ssl_ctx, min_proto) != 1) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize openssl"), + errdetail("could not set minimum TLS protocol version")) + ); + } + /* Authenticate server: verify chain against system CAs and hostname. */ + SSL_CTX_set_verify(s->ssl_ctx, SSL_VERIFY_PEER, NULL); + if (SSL_CTX_set_default_verify_paths(s->ssl_ctx) != 1) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize openssl"), + errdetail("could not load default CA certificates")) + ); + } + + s->ssl = SSL_new(s->ssl_ctx); + if (!s->ssl) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize opensssl"), + errdetail("SSL_new failed")) + ); + } + SSL_set_tlsext_host_name(s->ssl, host); + SSL_set_hostflags(s->ssl, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); + if (SSL_set1_host(s->ssl, host) != 1) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: failed to initialize openssl"), + errdetail("could not set certificate verification host")) + ); + } + SSL_set_fd(s->ssl, s->fd); + if (SSL_connect(s->ssl) != 1) { + /* verification failures leave error queue empty, use verify result */ + long vr = SSL_get_verify_result(s->ssl); + unsigned long e = ERR_peek_last_error(); + char ebuf[256]; + + if (vr != X509_V_OK) { + snprintf( + ebuf, + sizeof(ebuf), + "certificate verify failed: %s", + X509_verify_cert_error_string(vr) + ); + } else if (e) { + ERR_error_string_n(e, ebuf, sizeof(ebuf)); + } else { + snprintf(ebuf, sizeof(ebuf), "SSL_connect failed"); + } + + ereport( + ERROR, + (errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION), + errmsg("pg_clickhouse: openssl failed to connect"), + errdetail("%s", ebuf)) + ); + } } -ch_binary_connection_t * -ch_binary_connect(ch_connection_details * details) -{ - const char *host = details->host ? details->host : "127.0.0.1"; - int port = details->port; - bool tls; - - switch (details->tls) - { - case CH_TLS_ON: - if (port == 0) - port = CLICKHOUSE_SECURE_PORT; - tls = true; - break; - case CH_TLS_OFF: - if (port == 0) - port = CLICKHOUSE_PLAIN_PORT; - tls = false; - break; - default: /* CH_TLS_AUTO */ - if (port == 0) - port = ch_is_cloud_host(host) ? CLICKHOUSE_SECURE_PORT : CLICKHOUSE_PLAIN_PORT; - tls = (port == CLICKHOUSE_SECURE_PORT); - break; - } - - chc_compression comp = parse_compression(details->compression); - - MemoryContext cxt = AllocSetContextCreate(CacheMemoryContext, - "pg_clickhouse binary connection", - ALLOCSET_SMALL_SIZES); - MemoryContext old = MemoryContextSwitchTo(cxt); - struct ch_binary_state *s; - ch_binary_connection_t *conn; - - PG_TRY(); - { - s = palloc0(sizeof(*s)); - conn = palloc0(sizeof(*conn)); - s->cxt = cxt; - s->fd = -1; - s->tls = tls; - s->reset_cb.func = binary_state_reset_cb; - s->reset_cb.arg = s; - MemoryContextRegisterResetCallback(cxt, &s->reset_cb); - conn->client = s; - - s->fd = tcp_connect(host, port); - if (tls) - { - tls_connect(s, host, details->min_tls_version); - chc_openssl_io_init(&s->openssl_state, &s->io, s->ssl, - cancel_adapter, s); - } - else - { - chc_posix_io_init(&s->posix_state, &s->io, s->fd, - cancel_adapter, s); - } - - if (comp != CHC_COMP_NONE) - { - /* both families: server may answer in a different method */ - chc_lz4_codec_init(&s->codec); - chc_zstd_codec_init(&s->codec); - } - - chc_client_opts opts = { - .database = details->dbname ? details->dbname : "default", - .user = details->username ? details->username : "default", - .password = details->password ? details->password : "", - .codec = comp != CHC_COMP_NONE ? &s->codec : NULL, - .compression = comp, - }; - chc_err err = {}; - int rc = chc_client_init(&s->client, &opts, &pg_chc_alloc, &s->io, &err); - - if (rc != CHC_OK) - raise_chc(&err, ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION, - "connection error: "); - } - PG_CATCH(); - { - MemoryContextSwitchTo(old); - MemoryContextDelete(cxt); - PG_RE_THROW(); - } - PG_END_TRY(); - - MemoryContextSwitchTo(old); - return conn; +ch_binary_connection_t* +ch_binary_connect(ch_connection_details* details) { + const char* host = details->host ? details->host : "127.0.0.1"; + int port = details->port; + bool tls; + + switch (details->tls) { + case CH_TLS_ON: + if (port == 0) { + port = CLICKHOUSE_SECURE_PORT; + } + tls = true; + break; + case CH_TLS_OFF: + if (port == 0) { + port = CLICKHOUSE_PLAIN_PORT; + } + tls = false; + break; + default: /* CH_TLS_AUTO */ + if (port == 0) { + port = + ch_is_cloud_host(host) ? CLICKHOUSE_SECURE_PORT : CLICKHOUSE_PLAIN_PORT; + } + tls = (port == CLICKHOUSE_SECURE_PORT); + break; + } + + chc_compression comp = parse_compression(details->compression); + + MemoryContext cxt = AllocSetContextCreate( + CacheMemoryContext, "pg_clickhouse binary connection", ALLOCSET_SMALL_SIZES + ); + MemoryContext old = MemoryContextSwitchTo(cxt); + struct ch_binary_state* s; + ch_binary_connection_t* conn; + + PG_TRY(); + { + s = palloc0(sizeof(*s)); + conn = palloc0(sizeof(*conn)); + s->cxt = cxt; + s->fd = -1; + s->tls = tls; + s->reset_cb.func = binary_state_reset_cb; + s->reset_cb.arg = s; + MemoryContextRegisterResetCallback(cxt, &s->reset_cb); + conn->client = s; + + s->fd = tcp_connect(host, port); + if (tls) { + tls_connect(s, host, details->min_tls_version); + chc_openssl_io_init(&s->openssl_state, &s->io, s->ssl, cancel_adapter, s); + } else { + chc_posix_io_init(&s->posix_state, &s->io, s->fd, cancel_adapter, s); + } + + if (comp != CHC_COMP_NONE) { + /* both families: server may answer in a different method */ + chc_lz4_codec_init(&s->codec); + chc_zstd_codec_init(&s->codec); + } + + chc_client_opts opts = { + .database = details->dbname ? details->dbname : "default", + .user = details->username ? details->username : "default", + .password = details->password ? details->password : "", + .codec = comp != CHC_COMP_NONE ? &s->codec : NULL, + .compression = comp, + }; + chc_err err = {}; + int rc = chc_client_init(&s->client, &opts, &pg_chc_alloc, &s->io, &err); + + if (rc != CHC_OK) { + raise_chc( + &err, ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION, "connection error: " + ); + } + } + PG_CATCH(); + { + MemoryContextSwitchTo(old); + MemoryContextDelete(cxt); + PG_RE_THROW(); + } + PG_END_TRY(); + + MemoryContextSwitchTo(old); + return conn; } bool -ch_binary_is_broken(const ch_binary_connection_t * conn) -{ - if (!conn) - return false; - const struct ch_binary_state *s = (const struct ch_binary_state *) conn->client; +ch_binary_is_broken(const ch_binary_connection_t* conn) { + if (!conn) { + return false; + } + const struct ch_binary_state* s = (const struct ch_binary_state*)conn->client; - return s ? s->broken : false; + return s ? s->broken : false; } void -ch_binary_close(ch_binary_connection_t * conn) -{ - if (!conn) - return; - struct ch_binary_state *s = conn_state(conn); - - if (s) - MemoryContextDelete(s->cxt); - /* conn itself was palloc'd inside s->cxt; freed by the delete */ +ch_binary_close(ch_binary_connection_t* conn) { + if (!conn) { + return; + } + struct ch_binary_state* s = conn_state(conn); + + if (s) { + MemoryContextDelete(s->cxt); + } + /* conn itself was palloc'd inside s->cxt; freed by the delete */ } diff --git a/src/binary/convert.c b/src/binary/convert.c index 9800e37..ea21b48 100644 --- a/src/binary/convert.c +++ b/src/binary/convert.c @@ -2,178 +2,167 @@ #include "postgres.h" -#include "funcapi.h" -#include "access/tupdesc.h" #include "access/tupconvert.h" -#include "catalog/pg_type_d.h" +#include "access/tupdesc.h" #include "catalog/pg_type.h" -#include "utils/typcache.h" +#include "catalog/pg_type_d.h" +#include "executor/tuptable.h" +#include "funcapi.h" +#include "parser/parse_coerce.h" +#include "parser/parse_type.h" +#include "utils/arrayaccess.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/syscache.h" -#include "utils/arrayaccess.h" -#include "parser/parse_coerce.h" -#include "parser/parse_type.h" -#include "executor/tuptable.h" +#include "utils/typcache.h" -#include "fdw.h" #include "binary_internal.h" +#include "fdw.h" #include /* * Scalar CH column kind -> PG type OID, indexed by chc_kind. InvalidOid for * wrapper kinds (Nullable, LowCardinality, Array, Tuple) and unset entries. */ -const Oid ch_scalar_oids[CHC_KIND_COUNT] = { - [CHC_INT8] = INT2OID, - [CHC_INT16] = INT2OID, - [CHC_UINT8] = INT2OID, - [CHC_BOOL] = BOOLOID, - [CHC_INT32] = INT4OID, - [CHC_UINT16] = INT4OID, - [CHC_INT64] = INT8OID, - [CHC_UINT32] = INT8OID, - [CHC_UINT64] = INT8OID, - [CHC_FLOAT32] = FLOAT4OID, - [CHC_FLOAT64] = FLOAT8OID, - [CHC_DECIMAL32] = NUMERICOID, - [CHC_DECIMAL64] = NUMERICOID, - [CHC_DECIMAL128] = NUMERICOID, - [CHC_DECIMAL256] = NUMERICOID, - [CHC_STRING] = TEXTOID, - [CHC_FIXED_STRING] = TEXTOID, - [CHC_ENUM8] = TEXTOID, - [CHC_ENUM16] = TEXTOID, - [CHC_JSON] = JSONBOID, - [CHC_OBJECT] = JSONBOID, - [CHC_DATE] = DATEOID, - [CHC_DATE32] = DATEOID, - [CHC_DATETIME] = TIMESTAMPTZOID, - [CHC_DATETIME64] = TIMESTAMPTZOID, - [CHC_UUID] = UUIDOID, - [CHC_IPV4] = INETOID, - [CHC_IPV6] = INETOID, +const Oid ch_scalar_oids[CHC_KIND_COUNT] = { + [CHC_INT8] = INT2OID, + [CHC_INT16] = INT2OID, + [CHC_UINT8] = INT2OID, + [CHC_BOOL] = BOOLOID, + [CHC_INT32] = INT4OID, + [CHC_UINT16] = INT4OID, + [CHC_INT64] = INT8OID, + [CHC_UINT32] = INT8OID, + [CHC_UINT64] = INT8OID, + [CHC_FLOAT32] = FLOAT4OID, + [CHC_FLOAT64] = FLOAT8OID, + [CHC_DECIMAL32] = NUMERICOID, + [CHC_DECIMAL64] = NUMERICOID, + [CHC_DECIMAL128] = NUMERICOID, + [CHC_DECIMAL256] = NUMERICOID, + [CHC_STRING] = TEXTOID, + [CHC_FIXED_STRING] = TEXTOID, + [CHC_ENUM8] = TEXTOID, + [CHC_ENUM16] = TEXTOID, + [CHC_JSON] = JSONBOID, + [CHC_OBJECT] = JSONBOID, + [CHC_DATE] = DATEOID, + [CHC_DATE32] = DATEOID, + [CHC_DATETIME] = TIMESTAMPTZOID, + [CHC_DATETIME64] = TIMESTAMPTZOID, + [CHC_UUID] = UUIDOID, + [CHC_IPV4] = INETOID, + [CHC_IPV6] = INETOID, }; typedef struct ch_convert_state ch_convert_state; typedef struct ch_convert_output_state ch_convert_output_state; -typedef Datum(*convert_func) (ch_convert_state *, Datum); -typedef Datum(*out_convert_func) (ch_convert_output_state *, Datum); - -typedef struct ch_convert_state -{ - Oid intype; - Oid outtype; - convert_func func; - - /* record */ - TupleConversionMap *tupmap; - CustomObjectDef *cdef; - TupleDesc indesc; /* for RECORD */ - TupleDesc outdesc; /* for RECORD */ - ch_convert_state **conversion_states; - - /* array */ - int16 typlen; - bool typbyval; - char typalign; - - /* text */ - int32 typmod; - Oid typinput; - Oid typioparam; - - /* generic */ - CoercionPathType ctype; - Oid castfunc; -} ch_convert_state; - -typedef struct ch_convert_output_state -{ - Oid intype; - Oid outtype; - AttrNumber attnum; - out_convert_func func; - - /* array */ - Oid innertype; /* if intype is array */ - int16 typlen; - bool typbyval; - char typalign; - - /* generic */ - CoercionPathType ctype; - Oid castfunc; -} ch_convert_output_state; +typedef Datum (*convert_func)(ch_convert_state*, Datum); +typedef Datum (*out_convert_func)(ch_convert_output_state*, Datum); + +typedef struct ch_convert_state { + Oid intype; + Oid outtype; + convert_func func; + + /* record */ + TupleConversionMap* tupmap; + CustomObjectDef* cdef; + TupleDesc indesc; /* for RECORD */ + TupleDesc outdesc; /* for RECORD */ + ch_convert_state** conversion_states; + + /* array */ + int16 typlen; + bool typbyval; + char typalign; + + /* text */ + int32 typmod; + Oid typinput; + Oid typioparam; + + /* generic */ + CoercionPathType ctype; + Oid castfunc; +} ch_convert_state; + +typedef struct ch_convert_output_state { + Oid intype; + Oid outtype; + AttrNumber attnum; + out_convert_func func; + + /* array */ + Oid innertype; /* if intype is array */ + int16 typlen; + bool typbyval; + char typalign; + + /* generic */ + CoercionPathType ctype; + Oid castfunc; +} ch_convert_output_state; static Datum -convert_record(ch_convert_state * state, Datum val) -{ - HeapTuple temptup; - HeapTuple htup; - ch_binary_tuple_t *slot = (ch_binary_tuple_t *) DatumGetPointer(val); - - for (size_t i = 0; i < slot->len; i++) - { - ch_convert_state *s = state->conversion_states[i]; - - if (s) - slot->datums[i] = s->func(s, slot->datums[i]); - } - - htup = heap_form_tuple(state->indesc, slot->datums, slot->nulls); - if (!state->outdesc) - { - val = heap_copy_tuple_as_datum(htup, state->indesc); - - if (state->cdef && state->cdef->rowfunc != InvalidOid) - { - /* there is converter from row to outtype */ - val = OidFunctionCall1(state->cdef->rowfunc, val); - } - else if (state->outtype == TEXTOID) - { - /* a lot of allocations, not so efficient */ - val = CStringGetTextDatum(DatumGetCString( - OidFunctionCall1(F_RECORD_OUT, val))); - } - } - else - { - if (state->tupmap) - temptup = execute_attr_map_tuple(htup, state->tupmap); - else - temptup = htup; - - val = heap_copy_tuple_as_datum(temptup, state->outdesc); - } - - return val; +convert_record(ch_convert_state* state, Datum val) { + HeapTuple temptup; + HeapTuple htup; + ch_binary_tuple_t* slot = (ch_binary_tuple_t*)DatumGetPointer(val); + + for (size_t i = 0; i < slot->len; i++) { + ch_convert_state* s = state->conversion_states[i]; + + if (s) { + slot->datums[i] = s->func(s, slot->datums[i]); + } + } + + htup = heap_form_tuple(state->indesc, slot->datums, slot->nulls); + if (!state->outdesc) { + val = heap_copy_tuple_as_datum(htup, state->indesc); + + if (state->cdef && state->cdef->rowfunc != InvalidOid) { + /* there is converter from row to outtype */ + val = OidFunctionCall1(state->cdef->rowfunc, val); + } else if (state->outtype == TEXTOID) { + /* a lot of allocations, not so efficient */ + val = CStringGetTextDatum( + DatumGetCString(OidFunctionCall1(F_RECORD_OUT, val)) + ); + } + } else { + if (state->tupmap) { + temptup = execute_attr_map_tuple(htup, state->tupmap); + } else { + temptup = htup; + } + + val = heap_copy_tuple_as_datum(temptup, state->outdesc); + } + + return val; } inline static Datum -convert_generic(ch_convert_state * state, Datum val) -{ - if (state->ctype == COERCION_PATH_FUNC) - { - Assert(state->castfunc != InvalidOid); - val = OidFunctionCall1(state->castfunc, val); - } - - return val; +convert_generic(ch_convert_state* state, Datum val) { + if (state->ctype == COERCION_PATH_FUNC) { + Assert(state->castfunc != InvalidOid); + val = OidFunctionCall1(state->castfunc, val); + } + + return val; } inline static Datum -convert_out_generic(ch_convert_output_state * state, Datum val) -{ - if (state->ctype == COERCION_PATH_FUNC) - { - Assert(state->castfunc != InvalidOid); - val = OidFunctionCall1(state->castfunc, val); - } - - return val; +convert_out_generic(ch_convert_output_state* state, Datum val) { + if (state->ctype == COERCION_PATH_FUNC) { + Assert(state->castfunc != InvalidOid); + val = OidFunctionCall1(state->castfunc, val); + } + + return val; } /* @@ -182,32 +171,35 @@ convert_out_generic(ch_convert_output_state * state, Datum val) * shape is jagged so the caller can fall back to a slower path. */ static bool -flatten_nested_array(ch_binary_array_t * slot, int *dims, int level, - Datum * values, bool *nulls, size_t * idx) -{ - if ((int) slot->len != dims[level]) - return false; - - if (slot->ndim == 1) - { - for (size_t i = 0; i < slot->len; i++) - { - values[*idx] = slot->datums[i]; - nulls[*idx] = slot->nulls[i]; - (*idx)++; - } - } - else - { - for (size_t i = 0; i < slot->len; i++) - { - ch_binary_array_t *child = (ch_binary_array_t *) DatumGetPointer(slot->datums[i]); - - if (!flatten_nested_array(child, dims, level + 1, values, nulls, idx)) - return false; - } - } - return true; +flatten_nested_array( + ch_binary_array_t* slot, + int* dims, + int level, + Datum* values, + bool* nulls, + size_t* idx +) { + if ((int)slot->len != dims[level]) { + return false; + } + + if (slot->ndim == 1) { + for (size_t i = 0; i < slot->len; i++) { + values[*idx] = slot->datums[i]; + nulls[*idx] = slot->nulls[i]; + (*idx)++; + } + } else { + for (size_t i = 0; i < slot->len; i++) { + ch_binary_array_t* child = + (ch_binary_array_t*)DatumGetPointer(slot->datums[i]); + + if (!flatten_nested_array(child, dims, level + 1, values, nulls, idx)) { + return false; + } + } + } + return true; } /* @@ -216,129 +208,139 @@ flatten_nested_array(ch_binary_array_t * slot, int *dims, int level, * surfaces the same array_in malformed-literal error as the http path. */ static void -emit_nested_array_text(ch_binary_array_t * slot, FmgrInfo * outfn, StringInfo buf) -{ - appendStringInfoChar(buf, '{'); - for (size_t i = 0; i < slot->len; i++) - { - if (i > 0) - appendStringInfoChar(buf, ','); - - if (slot->ndim > 1) - { - ch_binary_array_t *child = (ch_binary_array_t *) DatumGetPointer(slot->datums[i]); - - emit_nested_array_text(child, outfn, buf); - } - else if (slot->nulls[i]) - appendStringInfoString(buf, "NULL"); - else - { - char *s = OutputFunctionCall(outfn, slot->datums[i]); - - appendStringInfoChar(buf, '"'); - for (char *p = s; *p; p++) - { - if (*p == '"' || *p == '\\') - appendStringInfoChar(buf, '\\'); - appendStringInfoChar(buf, *p); - } - appendStringInfoChar(buf, '"'); - pfree(s); - } - } - appendStringInfoChar(buf, '}'); +emit_nested_array_text(ch_binary_array_t* slot, FmgrInfo* outfn, StringInfo buf) { + appendStringInfoChar(buf, '{'); + for (size_t i = 0; i < slot->len; i++) { + if (i > 0) { + appendStringInfoChar(buf, ','); + } + + if (slot->ndim > 1) { + ch_binary_array_t* child = + (ch_binary_array_t*)DatumGetPointer(slot->datums[i]); + + emit_nested_array_text(child, outfn, buf); + } else if (slot->nulls[i]) { + appendStringInfoString(buf, "NULL"); + } else { + char* s = OutputFunctionCall(outfn, slot->datums[i]); + + appendStringInfoChar(buf, '"'); + for (char* p = s; *p; p++) { + if (*p == '"' || *p == '\\') { + appendStringInfoChar(buf, '\\'); + } + appendStringInfoChar(buf, *p); + } + appendStringInfoChar(buf, '"'); + pfree(s); + } + } + appendStringInfoChar(buf, '}'); } static Datum -convert_array(ch_convert_state * state, Datum val) -{ - ch_binary_array_t *slot = (ch_binary_array_t *) DatumGetPointer(val); - - if (slot->len == 0) - val = PointerGetDatum(construct_empty_array(slot->item_type)); - else if (slot->ndim <= 1) - { - void *arrout = construct_array(slot->datums, slot->len, slot->item_type, - state->typlen, state->typbyval, state->typalign); - - val = PointerGetDatum(arrout); - } - else - { - int dims[MAXDIM] = {}; - int lbs[MAXDIM] = {}; - size_t total = 1; - size_t idx = 0; - Datum *flat; - bool *flatnulls; - ch_binary_array_t *probe = slot; - - if (slot->ndim > MAXDIM) - ereport(ERROR, - (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("pg_clickhouse: nested array depth %d exceeds maximum %d", - slot->ndim, MAXDIM))); - - for (int d = 0; d < slot->ndim; d++) - { - dims[d] = (int) probe->len; - lbs[d] = 1; - total *= probe->len; - if (probe->ndim > 1 && probe->len > 0) - probe = (ch_binary_array_t *) DatumGetPointer(probe->datums[0]); - } - - if (total == 0) - val = PointerGetDatum(construct_empty_array(slot->item_type)); - else - { - flat = palloc(sizeof(Datum) * total); - flatnulls = palloc0(sizeof(bool) * total); - - if (flatten_nested_array(slot, dims, 0, flat, flatnulls, &idx)) - val = PointerGetDatum(construct_md_array(flat, flatnulls, slot->ndim, - dims, lbs, slot->item_type, - state->typlen, state->typbyval, - state->typalign)); - else - { - /* - * Jagged shape: format as text and route through array_in so - * binary surfaces the same malformed-literal error as http. - */ - StringInfoData buf; - FmgrInfo outfn; - Oid out_func; - Oid in_func; - Oid ioparam; - bool varlena; - - pfree(flat); - pfree(flatnulls); - - getTypeOutputInfo(slot->item_type, &out_func, &varlena); - fmgr_info(out_func, &outfn); - - initStringInfo(&buf); - emit_nested_array_text(slot, &outfn, &buf); - - getTypeInputInfo(state->intype, &in_func, &ioparam); - val = OidInputFunctionCall(in_func, buf.data, ioparam, -1); - - pfree(buf.data); - } - } - } - - return convert_generic(state, val); +convert_array(ch_convert_state* state, Datum val) { + ch_binary_array_t* slot = (ch_binary_array_t*)DatumGetPointer(val); + + if (slot->len == 0) { + val = PointerGetDatum(construct_empty_array(slot->item_type)); + } else if (slot->ndim <= 1) { + void* arrout = construct_array( + slot->datums, + slot->len, + slot->item_type, + state->typlen, + state->typbyval, + state->typalign + ); + + val = PointerGetDatum(arrout); + } else { + int dims[MAXDIM] = {}; + int lbs[MAXDIM] = {}; + size_t total = 1; + size_t idx = 0; + Datum* flat; + bool* flatnulls; + ch_binary_array_t* probe = slot; + + if (slot->ndim > MAXDIM) { + ereport( + ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg( + "pg_clickhouse: nested array depth %d exceeds maximum %d", + slot->ndim, + MAXDIM + )) + ); + } + + for (int d = 0; d < slot->ndim; d++) { + dims[d] = (int)probe->len; + lbs[d] = 1; + total *= probe->len; + if (probe->ndim > 1 && probe->len > 0) { + probe = (ch_binary_array_t*)DatumGetPointer(probe->datums[0]); + } + } + + if (total == 0) { + val = PointerGetDatum(construct_empty_array(slot->item_type)); + } else { + flat = palloc(sizeof(Datum) * total); + flatnulls = palloc0(sizeof(bool) * total); + + if (flatten_nested_array(slot, dims, 0, flat, flatnulls, &idx)) { + val = PointerGetDatum(construct_md_array( + flat, + flatnulls, + slot->ndim, + dims, + lbs, + slot->item_type, + state->typlen, + state->typbyval, + state->typalign + )); + } else { + /* + * Jagged shape: format as text and route through array_in so + * binary surfaces the same malformed-literal error as http. + */ + StringInfoData buf; + FmgrInfo outfn; + Oid out_func; + Oid in_func; + Oid ioparam; + bool varlena; + + pfree(flat); + pfree(flatnulls); + + getTypeOutputInfo(slot->item_type, &out_func, &varlena); + fmgr_info(out_func, &outfn); + + initStringInfo(&buf); + emit_nested_array_text(slot, &outfn, &buf); + + getTypeInputInfo(state->intype, &in_func, &ioparam); + val = OidInputFunctionCall(in_func, buf.data, ioparam, -1); + + pfree(buf.data); + } + } + } + + return convert_generic(state, val); } static Datum -convert_remote_text(ch_convert_state * state, Datum val) -{ - return OidInputFunctionCall(state->typinput, TextDatumGetCString(val), - state->typioparam, state->typmod); +convert_remote_text(ch_convert_state* state, Datum val) { + return OidInputFunctionCall( + state->typinput, TextDatumGetCString(val), state->typioparam, state->typmod + ); } /* @@ -346,17 +348,15 @@ convert_remote_text(ch_convert_state * state, Datum val) * SMALLINT and this function covers this case */ static Datum -convert_bool(ch_convert_state * state, Datum val) -{ - int16 dat = DatumGetInt16(val); +convert_bool(ch_convert_state* state, Datum val) { + int16 dat = DatumGetInt16(val); - return BoolGetDatum(dat); + return BoolGetDatum(dat); } Datum -ch_binary_convert_datum(void *state, Datum val) -{ - return state ? ((ch_convert_state *) state)->func(state, val) : val; +ch_binary_convert_datum(void* state, Datum val) { + return state ? ((ch_convert_state*)state)->func(state, val) : val; } /* input */ @@ -365,284 +365,292 @@ ch_binary_convert_datum(void *state, Datum val) * Convert val from intype to outtype. No conversion for binary-compatible * types or when intype and outtype are the same. All others converted via the * appropriate CAST function. -*/ -void * -ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype) -{ - /* make_datum() copies all bytes, no cast needed. */ - if (intype == TEXTOID && outtype == BYTEAOID) - return NULL; + */ +void* +ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype) { + /* make_datum() copies all bytes, no cast needed. */ + if (intype == TEXTOID && outtype == BYTEAOID) { + return NULL; + } - ch_convert_state *state = palloc0(sizeof(ch_convert_state)); + ch_convert_state* state = palloc0(sizeof(ch_convert_state)); - state->intype = intype; - state->outtype = outtype; - state->cdef = chfdw_check_for_custom_type(outtype); - state->typmod = -1; - state->ctype = COERCION_PATH_NONE; + state->intype = intype; + state->outtype = outtype; + state->cdef = chfdw_check_for_custom_type(outtype); + state->typmod = -1; + state->ctype = COERCION_PATH_NONE; - if (intype == ANYARRAYOID) - { - ch_binary_array_t *slot = (ch_binary_array_t *) DatumGetPointer(val); + if (intype == ANYARRAYOID) { + ch_binary_array_t* slot = (ch_binary_array_t*)DatumGetPointer(val); - get_typlenbyvalalign(slot->item_type, &state->typlen, &state->typbyval, - &state->typalign); + get_typlenbyvalalign( + slot->item_type, &state->typlen, &state->typbyval, &state->typalign + ); - /* restore intype */ - state->intype = slot->array_type; - intype = slot->array_type; - state->func = convert_array; - } + /* restore intype */ + state->intype = slot->array_type; + intype = slot->array_type; + state->func = convert_array; + } - if (intype == RECORDOID) - { - ch_binary_tuple_t *slot = (ch_binary_tuple_t *) DatumGetPointer(val); + if (intype == RECORDOID) { + ch_binary_tuple_t* slot = (ch_binary_tuple_t*)DatumGetPointer(val); - state->func = convert_record; + state->func = convert_record; #if PG_VERSION_NUM < 120000 - state->indesc = CreateTemplateTupleDesc(slot->len, false); + state->indesc = CreateTemplateTupleDesc(slot->len, false); #else - state->indesc = CreateTemplateTupleDesc(slot->len); + state->indesc = CreateTemplateTupleDesc(slot->len); #endif - state->conversion_states = palloc(sizeof(void *) * slot->len); + state->conversion_states = palloc(sizeof(void*) * slot->len); - for (size_t i = 0; i < slot->len; ++i) - { - Oid item_type = slot->types[i]; + for (size_t i = 0; i < slot->len; ++i) { + Oid item_type = slot->types[i]; - if (slot->types[i] == ANYARRAYOID) - { - ch_binary_array_t *arr = (ch_binary_array_t *) DatumGetPointer(slot->datums[i]); + if (slot->types[i] == ANYARRAYOID) { + ch_binary_array_t* arr = + (ch_binary_array_t*)DatumGetPointer(slot->datums[i]); - item_type = arr->array_type; - } - state->conversion_states[i] = ch_binary_init_convert_state(slot->datums[i], - slot->types[i], item_type); + item_type = arr->array_type; + } + state->conversion_states[i] = ch_binary_init_convert_state( + slot->datums[i], slot->types[i], item_type + ); - TupleDescInitEntry(state->indesc, (AttrNumber) i + 1, "", - item_type, -1, 0); - } + TupleDescInitEntry(state->indesc, (AttrNumber)i + 1, "", item_type, -1, 0); + } #if PG_VERSION_NUM >= 190000 - TupleDescFinalize(state->indesc); + TupleDescFinalize(state->indesc); #endif - state->indesc = BlessTupleDesc(state->indesc); - - if (!(state->cdef || outtype == RECORDOID || outtype == TEXTOID)) - { - TypeCacheEntry *typentry; - TupleDesc tupdesc; - - typentry = lookup_type_cache(outtype, - TYPECACHE_TUPDESC | - TYPECACHE_DOMAIN_BASE_INFO); - - if (typentry->typtype == TYPTYPE_DOMAIN) - tupdesc = lookup_rowtype_tupdesc_noerror(typentry->domainBaseType, - typentry->domainBaseTypmod, - false); - else - { - if (typentry->tupDesc == NULL) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("pg_clickhouse: cannot return %s as %s", - slot->ch_type_name ? slot->ch_type_name : "?", - format_type_be(outtype)))); - - tupdesc = typentry->tupDesc; - PinTupleDesc(tupdesc); - } - state->outdesc = CreateTupleDescCopy(tupdesc); - state->tupmap = convert_tuples_by_position(state->indesc, state->outdesc, - "pg_clickhouse: could not map tuple to returned type"); - ReleaseTupleDesc(tupdesc); - } - } - else if (intype != outtype) - { - if (!state->func) - state->func = convert_generic; - - if (intype == TEXTOID) - { - Type baseType; - Oid baseTypeId; - Form_pg_type typform; - - baseTypeId = getBaseTypeAndTypmod(outtype, &state->typmod); - if (baseTypeId != INTERVALOID) - state->typmod = -1; - - baseType = typeidType(baseTypeId); - typform = (Form_pg_type) GETSTRUCT(baseType); - state->typinput = typform->typinput; - state->typioparam = getTypeIOParam(baseType); - state->func = convert_remote_text; - ReleaseSysCache(baseType); - } - else if (outtype == BOOLOID && intype == INT2OID) - { - state->func = convert_bool; - } - else - { - /* try to convert */ - state->ctype = find_coercion_pathway(outtype, intype, - COERCION_EXPLICIT, - &state->castfunc); - switch (state->ctype) - { - case COERCION_PATH_FUNC: - break; - case COERCION_PATH_RELABELTYPE: - - /* - * if the conversion func was not previously set, then no - * conversion needed - */ - if (state->func == NULL) - goto no_conversion; - - /* all good */ - break; - default: - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: could not cast value from %s to %s", - format_type_be(intype), format_type_be(outtype)))); - } - } - } - else if (!state->func) - { -no_conversion: - /* no conversion needed */ - pfree(state); - state = NULL; - } - - return state; + state->indesc = BlessTupleDesc(state->indesc); + + if (!(state->cdef || outtype == RECORDOID || outtype == TEXTOID)) { + TypeCacheEntry* typentry; + TupleDesc tupdesc; + + typentry = lookup_type_cache( + outtype, TYPECACHE_TUPDESC | TYPECACHE_DOMAIN_BASE_INFO + ); + + if (typentry->typtype == TYPTYPE_DOMAIN) { + tupdesc = lookup_rowtype_tupdesc_noerror( + typentry->domainBaseType, typentry->domainBaseTypmod, false + ); + } else { + if (typentry->tupDesc == NULL) { + ereport( + ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg( + "pg_clickhouse: cannot return %s as %s", + slot->ch_type_name ? slot->ch_type_name : "?", + format_type_be(outtype) + )) + ); + } + + tupdesc = typentry->tupDesc; + PinTupleDesc(tupdesc); + } + state->outdesc = CreateTupleDescCopy(tupdesc); + state->tupmap = convert_tuples_by_position( + state->indesc, + state->outdesc, + "pg_clickhouse: could not map tuple to returned type" + ); + ReleaseTupleDesc(tupdesc); + } + } else if (intype != outtype) { + if (!state->func) { + state->func = convert_generic; + } + + if (intype == TEXTOID) { + Type baseType; + Oid baseTypeId; + Form_pg_type typform; + + baseTypeId = getBaseTypeAndTypmod(outtype, &state->typmod); + if (baseTypeId != INTERVALOID) { + state->typmod = -1; + } + + baseType = typeidType(baseTypeId); + typform = (Form_pg_type)GETSTRUCT(baseType); + state->typinput = typform->typinput; + state->typioparam = getTypeIOParam(baseType); + state->func = convert_remote_text; + ReleaseSysCache(baseType); + } else if (outtype == BOOLOID && intype == INT2OID) { + state->func = convert_bool; + } else { + /* try to convert */ + state->ctype = find_coercion_pathway( + outtype, intype, COERCION_EXPLICIT, &state->castfunc + ); + switch (state->ctype) { + case COERCION_PATH_FUNC: + break; + case COERCION_PATH_RELABELTYPE: + + /* + * if the conversion func was not previously set, then no + * conversion needed + */ + if (state->func == NULL) { + goto no_conversion; + } + + /* all good */ + break; + default: + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: could not cast value from %s to %s", + format_type_be(intype), + format_type_be(outtype) + )) + ); + } + } + } else if (!state->func) { + no_conversion: + /* no conversion needed */ + pfree(state); + state = NULL; + } + + return state; } void -ch_binary_free_convert_state(void *s) -{ - ch_convert_state *state = s; +ch_binary_free_convert_state(void* s) { + ch_convert_state* state = s; - pfree(state); + pfree(state); } /* output */ static void -init_output_convert_state(ch_convert_output_state * state) -{ - if (state->outtype == state->intype) - return; - - /* column_append() copies all bytes, no cast needed. */ - if (state->intype == BYTEAOID && state->outtype == TEXTOID) - return; - - state->func = convert_out_generic; - - state->ctype = find_coercion_pathway(state->outtype, state->intype, - COERCION_EXPLICIT, &state->castfunc); - - switch (state->ctype) - { - case COERCION_PATH_FUNC: - break; - case COERCION_PATH_RELABELTYPE: - state->func = NULL; - return; - default: - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: could not find a casting path from %s to %s", - format_type_be(state->intype), format_type_be(state->outtype)))); - } +init_output_convert_state(ch_convert_output_state* state) { + if (state->outtype == state->intype) { + return; + } + + /* column_append() copies all bytes, no cast needed. */ + if (state->intype == BYTEAOID && state->outtype == TEXTOID) { + return; + } + + state->func = convert_out_generic; + + state->ctype = find_coercion_pathway( + state->outtype, state->intype, COERCION_EXPLICIT, &state->castfunc + ); + + switch (state->ctype) { + case COERCION_PATH_FUNC: + break; + case COERCION_PATH_RELABELTYPE: + state->func = NULL; + return; + default: + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: could not find a casting path from %s to %s", + format_type_be(state->intype), + format_type_be(state->outtype) + )) + ); + } } -void * -ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid) -{ - ch_convert_output_state *states; - int n; - int i; - - n = outdesc->natts; - states = (ch_convert_output_state *) palloc0(n * sizeof(ch_convert_output_state)); - - for (i = 0; i < n; i++) - { - ch_convert_output_state *curstate = &states[i]; - - Form_pg_attribute attout = TupleDescAttr(outdesc, i); - char *outattname; - int j; - - outattname = NameStr(attout->attname); - curstate->outtype = attout->atttypid; - - if (NameStr(TupleDescAttr(indesc, 0)->attname)[0] == '\0') - { - Form_pg_attribute attin = TupleDescAttr(indesc, i); - - curstate->intype = attin->atttypid; - init_output_convert_state(curstate); - curstate->attnum = (AttrNumber) (i + 1); - } - else - { - for (j = 0; j < indesc->natts; j++) - { - Form_pg_attribute attin = TupleDescAttr(indesc, j); - char *inattname; - CustomColumnInfo *cinfo; - - if (attin->attisdropped) - continue; - - /* Honor column_name FDW option, falls through to attname */ - cinfo = OidIsValid(relid) - ? chfdw_get_custom_column_info(relid, j + 1) : NULL; - inattname = (cinfo && cinfo->colname[0]) - ? cinfo->colname : NameStr(attin->attname); - - curstate->intype = attin->atttypid; - - if (strcmp(outattname, inattname) == 0) - { - init_output_convert_state(curstate); - curstate->attnum = (AttrNumber) (j + 1); - break; - } - } - } - - curstate->innertype = get_element_type(curstate->outtype); - if (curstate->innertype != InvalidOid) - { - curstate->outtype = ANYARRAYOID; - get_typlenbyvalalign(curstate->innertype, &curstate->typlen, - &curstate->typbyval, &curstate->typalign); - } - - - if (curstate->attnum == 0) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal("pg_clickhouse: could not create conversion map"), - errdetail("Attribute \"%s\" of type %s does not exist in type %s.", - outattname, - format_type_be(indesc->tdtypeid), - format_type_be(outdesc->tdtypeid)))); - } - - return states; +void* +ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid) { + ch_convert_output_state* states; + int n; + int i; + + n = outdesc->natts; + states = (ch_convert_output_state*)palloc0(n * sizeof(ch_convert_output_state)); + + for (i = 0; i < n; i++) { + ch_convert_output_state* curstate = &states[i]; + + Form_pg_attribute attout = TupleDescAttr(outdesc, i); + char* outattname; + int j; + + outattname = NameStr(attout->attname); + curstate->outtype = attout->atttypid; + + if (NameStr(TupleDescAttr(indesc, 0)->attname)[0] == '\0') { + Form_pg_attribute attin = TupleDescAttr(indesc, i); + + curstate->intype = attin->atttypid; + init_output_convert_state(curstate); + curstate->attnum = (AttrNumber)(i + 1); + } else { + for (j = 0; j < indesc->natts; j++) { + Form_pg_attribute attin = TupleDescAttr(indesc, j); + char* inattname; + CustomColumnInfo* cinfo; + + if (attin->attisdropped) { + continue; + } + + /* Honor column_name FDW option, falls through to attname */ + cinfo = OidIsValid(relid) ? chfdw_get_custom_column_info(relid, j + 1) + : NULL; + inattname = (cinfo && cinfo->colname[0]) ? cinfo->colname + : NameStr(attin->attname); + + curstate->intype = attin->atttypid; + + if (strcmp(outattname, inattname) == 0) { + init_output_convert_state(curstate); + curstate->attnum = (AttrNumber)(j + 1); + break; + } + } + } + + curstate->innertype = get_element_type(curstate->outtype); + if (curstate->innertype != InvalidOid) { + curstate->outtype = ANYARRAYOID; + get_typlenbyvalalign( + curstate->innertype, + &curstate->typlen, + &curstate->typbyval, + &curstate->typalign + ); + } + + if (curstate->attnum == 0) { + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal("pg_clickhouse: could not create conversion map"), + errdetail( + "Attribute \"%s\" of type %s does not exist in type %s.", + outattname, + format_type_be(indesc->tdtypeid), + format_type_be(outdesc->tdtypeid) + )) + ); + } + } + + return states; } /* @@ -652,40 +660,41 @@ ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid) * ndim>1 with datums[i] = PointerGetDatum(child); leaves carry ndim==1 with * scalar datums copied from the flat buffer. */ -static ch_binary_array_t * -build_nested_binary_array(int level, int ndim, int *dims, Oid item_type, - Datum * flat, bool *flatnulls, size_t * idx) -{ - ch_binary_array_t *arr = palloc(sizeof(ch_binary_array_t)); - - arr->len = dims[level]; - arr->ndim = ndim - level; - arr->item_type = item_type; - arr->array_type = InvalidOid; - arr->datums = palloc(sizeof(Datum) * arr->len); - arr->nulls = palloc0(sizeof(bool) * arr->len); - - if (level + 1 == ndim) - { - for (size_t i = 0; i < arr->len; i++) - { - arr->datums[i] = flat[*idx]; - arr->nulls[i] = flatnulls[*idx]; - (*idx)++; - } - } - else - { - for (size_t i = 0; i < arr->len; i++) - { - ch_binary_array_t *child = build_nested_binary_array(level + 1, ndim, dims, - item_type, flat, - flatnulls, idx); - - arr->datums[i] = PointerGetDatum(child); - } - } - return arr; +static ch_binary_array_t* +build_nested_binary_array( + int level, + int ndim, + int* dims, + Oid item_type, + Datum* flat, + bool* flatnulls, + size_t* idx +) { + ch_binary_array_t* arr = palloc(sizeof(ch_binary_array_t)); + + arr->len = dims[level]; + arr->ndim = ndim - level; + arr->item_type = item_type; + arr->array_type = InvalidOid; + arr->datums = palloc(sizeof(Datum) * arr->len); + arr->nulls = palloc0(sizeof(bool) * arr->len); + + if (level + 1 == ndim) { + for (size_t i = 0; i < arr->len; i++) { + arr->datums[i] = flat[*idx]; + arr->nulls[i] = flatnulls[*idx]; + (*idx)++; + } + } else { + for (size_t i = 0; i < arr->len; i++) { + ch_binary_array_t* child = build_nested_binary_array( + level + 1, ndim, dims, item_type, flat, flatnulls, idx + ); + + arr->datums[i] = PointerGetDatum(child); + } + } + return arr; } /* @@ -697,85 +706,109 @@ build_nested_binary_array(int level, int ndim, int *dims, Oid item_type, * in insert_state->conversion_states) */ void -ch_binary_do_output_conversion(ch_binary_insert_state * insert_state, - TupleTableSlot * slot) -{ - Datum *out_values = insert_state->values; - bool *out_nulls = insert_state->nulls; - - for (size_t i = 0; i < insert_state->outdesc->natts; i++) - { - ch_convert_output_state *cstate = &((ch_convert_output_state *) insert_state->conversion_states)[i]; - AttrNumber attnum = cstate->attnum; - - out_values[i] = slot_getattr(slot, attnum, &out_nulls[i]); - if (!out_nulls[i]) - { - if (cstate->func) - out_values[i] = cstate->func(cstate, out_values[i]); - else if (cstate->outtype == ANYARRAYOID) - { - AnyArrayType *v = DatumGetAnyArrayP(out_values[i]); - ch_binary_array_t *arr; - array_iter iter; - int ndim = AARR_NDIM(v); - int *dims = AARR_DIMS(v); - size_t total = ArrayGetNItems(ndim, dims); - - if (ndim > MAXDIM) - ereport(ERROR, - (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("pg_clickhouse: inserted array depth %d exceeds maximum %d", - ndim, MAXDIM))); - - if (ndim <= 1) - { - arr = palloc(sizeof(ch_binary_array_t)); - arr->len = total; - arr->ndim = 1; - arr->item_type = cstate->innertype; - arr->array_type = InvalidOid; - arr->datums = total ? palloc(sizeof(Datum) * total) : NULL; - arr->nulls = total ? palloc(sizeof(bool) * total) : NULL; +ch_binary_do_output_conversion( + ch_binary_insert_state* insert_state, + TupleTableSlot* slot +) { + Datum* out_values = insert_state->values; + bool* out_nulls = insert_state->nulls; + + for (size_t i = 0; i < insert_state->outdesc->natts; i++) { + ch_convert_output_state* cstate = + &((ch_convert_output_state*)insert_state->conversion_states)[i]; + AttrNumber attnum = cstate->attnum; + + out_values[i] = slot_getattr(slot, attnum, &out_nulls[i]); + if (!out_nulls[i]) { + if (cstate->func) { + out_values[i] = cstate->func(cstate, out_values[i]); + } else if (cstate->outtype == ANYARRAYOID) { + AnyArrayType* v = DatumGetAnyArrayP(out_values[i]); + ch_binary_array_t* arr; + array_iter iter; + int ndim = AARR_NDIM(v); + int* dims = AARR_DIMS(v); + size_t total = ArrayGetNItems(ndim, dims); + + if (ndim > MAXDIM) { + ereport( + ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg( + "pg_clickhouse: inserted array depth %d exceeds maximum " + "%d", + ndim, + MAXDIM + )) + ); + } + + if (ndim <= 1) { + arr = palloc(sizeof(ch_binary_array_t)); + arr->len = total; + arr->ndim = 1; + arr->item_type = cstate->innertype; + arr->array_type = InvalidOid; + arr->datums = total ? palloc(sizeof(Datum) * total) : NULL; + arr->nulls = total ? palloc(sizeof(bool) * total) : NULL; #if PG_VERSION_NUM < 190000 - array_iter_setup(&iter, v); - for (size_t j = 0; j < total; j++) - arr->datums[j] = array_iter_next(&iter, &arr->nulls[j], j, - cstate->typlen, cstate->typbyval, cstate->typalign); + array_iter_setup(&iter, v); + for (size_t j = 0; j < total; j++) { + arr->datums[j] = array_iter_next( + &iter, + &arr->nulls[j], + j, + cstate->typlen, + cstate->typbyval, + cstate->typalign + ); + } #else - array_iter_setup(&iter, v, cstate->typlen, cstate->typbyval, cstate->typalign); - for (size_t j = 0; j < total; j++) - arr->datums[j] = array_iter_next(&iter, &arr->nulls[j], j); + array_iter_setup( + &iter, v, cstate->typlen, cstate->typbyval, cstate->typalign + ); + for (size_t j = 0; j < total; j++) { + arr->datums[j] = array_iter_next(&iter, &arr->nulls[j], j); + } #endif - } - else - { - Datum *flat = palloc(sizeof(Datum) * total); - bool *flatnulls = palloc0(sizeof(bool) * total); - size_t idx = 0; + } else { + Datum* flat = palloc(sizeof(Datum) * total); + bool* flatnulls = palloc0(sizeof(bool) * total); + size_t idx = 0; #if PG_VERSION_NUM < 190000 - array_iter_setup(&iter, v); - for (size_t j = 0; j < total; j++) - flat[j] = array_iter_next(&iter, &flatnulls[j], j, - cstate->typlen, cstate->typbyval, cstate->typalign); + array_iter_setup(&iter, v); + for (size_t j = 0; j < total; j++) { + flat[j] = array_iter_next( + &iter, + &flatnulls[j], + j, + cstate->typlen, + cstate->typbyval, + cstate->typalign + ); + } #else - array_iter_setup(&iter, v, cstate->typlen, cstate->typbyval, cstate->typalign); - for (size_t j = 0; j < total; j++) - flat[j] = array_iter_next(&iter, &flatnulls[j], j); + array_iter_setup( + &iter, v, cstate->typlen, cstate->typbyval, cstate->typalign + ); + for (size_t j = 0; j < total; j++) { + flat[j] = array_iter_next(&iter, &flatnulls[j], j); + } #endif - arr = build_nested_binary_array(0, ndim, dims, cstate->innertype, - flat, flatnulls, &idx); - - pfree(flat); - pfree(flatnulls); - } - out_values[i] = PointerGetDatum(arr); - - /* hack: mark as unified array */ - TupleDescAttr(insert_state->outdesc, i)->atttypid = ANYARRAYOID; - } - } - } + arr = build_nested_binary_array( + 0, ndim, dims, cstate->innertype, flat, flatnulls, &idx + ); + + pfree(flat); + pfree(flatnulls); + } + out_values[i] = PointerGetDatum(arr); + + /* hack: mark as unified array */ + TupleDescAttr(insert_state->outdesc, i)->atttypid = ANYARRAYOID; + } + } + } } diff --git a/src/binary/decode.c b/src/binary/decode.c index f0b8eb2..4ab05a0 100644 --- a/src/binary/decode.c +++ b/src/binary/decode.c @@ -10,7 +10,7 @@ #include "postgres.h" #include -#include /* AF_INET, expanded by PG inet macros */ +#include /* AF_INET, expanded by PG inet macros */ #include "catalog/pg_type_d.h" #include "fmgr.h" @@ -34,31 +34,26 @@ /* Little-endian fixed-width reads at row offset. */ static inline int8_t -rd_i8(const uint8_t * p, uint64_t row) -{ - return (int8_t) p[row]; +rd_i8(const uint8_t* p, uint64_t row) { + return (int8_t)p[row]; } static inline uint8_t -rd_u8(const uint8_t * p, uint64_t row) -{ - return p[row]; +rd_u8(const uint8_t* p, uint64_t row) { + return p[row]; } static inline bool -rd_bool(const bool *p, uint64_t row) -{ - return (bool) p[row]; +rd_bool(const bool* p, uint64_t row) { + return (bool)p[row]; } -#define RD_FIXED(suffix, T) \ - static inline T \ - rd_##suffix(const uint8_t *p, uint64_t row) \ - { \ - T v; \ - memcpy(&v, p + row * sizeof(T), sizeof(T)); \ - return v; \ - } +#define RD_FIXED(suffix, T) \ + static inline T rd_##suffix(const uint8_t* p, uint64_t row) { \ + T v; \ + memcpy(&v, p + row * sizeof(T), sizeof(T)); \ + return v; \ + } RD_FIXED(i16, int16_t) RD_FIXED(u16, uint16_t) @@ -70,50 +65,55 @@ RD_FIXED(f32, float) RD_FIXED(f64, double) static inline void -slice_str(const chc_column * col, uint64_t row, - const char **out_ptr, size_t * out_len) -{ - const uint64_t *offs = chc_column_string_offsets(col); - const uint8_t *data = chc_column_string_data(col); - uint64_t start = row == 0 ? 0 : offs[row - 1]; - uint64_t end = offs[row]; - - *out_ptr = (const char *) data + start; - *out_len = (size_t) (end - start); +slice_str(const chc_column* col, uint64_t row, const char** out_ptr, size_t* out_len) { + const uint64_t* offs = chc_column_string_offsets(col); + const uint8_t* data = chc_column_string_data(col); + uint64_t start = row == 0 ? 0 : offs[row - 1]; + uint64_t end = offs[row]; + + *out_ptr = (const char*)data + start; + *out_len = (size_t)(end - start); } static Oid -ch_kind_to_pg_oid(const chc_type * type) -{ - chc_kind kind = chc_type_kind(type); - Oid oid = ch_scalar_oids[kind]; - - if (OidIsValid(oid)) - return oid; - - switch (kind) - { - case CHC_VOID: - case CHC_NOTHING: - return InvalidOid; - case CHC_NULLABLE: - case CHC_LOW_CARDINALITY: - return ch_kind_to_pg_oid(chc_type_child(type, 0)); - case CHC_ARRAY: - return ANYARRAYOID; - case CHC_TUPLE: - return RECORDOID; - default: - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported column type"))); - } - /* unreachable */ - return InvalidOid; +ch_kind_to_pg_oid(const chc_type* type) { + chc_kind kind = chc_type_kind(type); + Oid oid = ch_scalar_oids[kind]; + + if (OidIsValid(oid)) { + return oid; + } + + switch (kind) { + case CHC_VOID: + case CHC_NOTHING: + return InvalidOid; + case CHC_NULLABLE: + case CHC_LOW_CARDINALITY: + return ch_kind_to_pg_oid(chc_type_child(type, 0)); + case CHC_ARRAY: + return ANYARRAYOID; + case CHC_TUPLE: + return RECORDOID; + default: + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported column type")) + ); + } + /* unreachable */ + return InvalidOid; } -static Datum read_value(const chc_column * col, const chc_type * type, - uint64_t row, Oid * valtype, bool *is_null); +static Datum +read_value( + const chc_column* col, + const chc_type* type, + uint64_t row, + Oid* valtype, + bool* is_null +); /* * Format a ClickHouse Decimal (two's-complement signed integer in LE bytes of @@ -121,140 +121,147 @@ static Datum read_value(const chc_column * col, const chc_type * type, * carried on the column type) into `out`. Returns bytes written, -1 on overflow. */ static int -format_decimal_text(const uint8_t * bytes, size_t width, uint32_t scale, - char *out, size_t out_cap) -{ - uint32_t mag[8]; - size_t nwords = width / 4; - bool neg = false; - - if (width == 0 || width > 32 || width % 4 != 0) - return -1; - memcpy(mag, bytes, width); - /* top bit of MSW is sign; negate two's-complement to get magnitude */ - if (mag[nwords - 1] & 0x80000000u) - { - neg = true; - for (size_t i = 0; i < nwords; i++) - mag[i] = ~mag[i]; - uint64_t carry = 1; - - for (size_t i = 0; i < nwords && carry; i++) - { - uint64_t v = (uint64_t) mag[i] + carry; - - mag[i] = (uint32_t) v; - carry = v >> 32; - } - } - - char buf[80]; - int n = 0; - bool nonzero; - - /* base-10 division of mag yields digits LSB-first */ - do - { - uint64_t rem = 0; - - nonzero = false; - for (ssize_t i = (ssize_t) nwords - 1; i >= 0; i--) - { - uint64_t v = (rem << 32) | mag[i]; - - mag[i] = (uint32_t) (v / 10); - rem = v % 10; - if (mag[i]) - nonzero = true; - } - buf[n++] = (char) ('0' + (uint32_t) rem); - } while (nonzero && n < (int) sizeof(buf)); - - /* pad leading zeros so digit count covers fractional portion */ - while (n <= (int) scale) - buf[n++] = '0'; - - size_t need = (size_t) neg + (size_t) n + (scale ? 1 : 0) + 1; - - if (need > out_cap) - return -1; - char *p = out; - - if (neg) - *p++ = '-'; - - /* emit MSD-first, inserting '.' before `scale` trailing digits */ - for (int i = n - 1; i >= 0; i--) - { - if (i + 1 == scale) - *p++ = '.'; - *p++ = buf[i]; - } - *p = '\0'; - return (int) (p - out); +format_decimal_text( + const uint8_t* bytes, + size_t width, + uint32_t scale, + char* out, + size_t out_cap +) { + uint32_t mag[8]; + size_t nwords = width / 4; + bool neg = false; + + if (width == 0 || width > 32 || width % 4 != 0) { + return -1; + } + memcpy(mag, bytes, width); + /* top bit of MSW is sign; negate two's-complement to get magnitude */ + if (mag[nwords - 1] & 0x80000000u) { + neg = true; + for (size_t i = 0; i < nwords; i++) { + mag[i] = ~mag[i]; + } + uint64_t carry = 1; + + for (size_t i = 0; i < nwords && carry; i++) { + uint64_t v = (uint64_t)mag[i] + carry; + + mag[i] = (uint32_t)v; + carry = v >> 32; + } + } + + char buf[80]; + int n = 0; + bool nonzero; + + /* base-10 division of mag yields digits LSB-first */ + do { + uint64_t rem = 0; + + nonzero = false; + for (ssize_t i = (ssize_t)nwords - 1; i >= 0; i--) { + uint64_t v = (rem << 32) | mag[i]; + + mag[i] = (uint32_t)(v / 10); + rem = v % 10; + if (mag[i]) { + nonzero = true; + } + } + buf[n++] = (char)('0' + (uint32_t)rem); + } while (nonzero && n < (int)sizeof(buf)); + + /* pad leading zeros so digit count covers fractional portion */ + while (n <= (int)scale) { + buf[n++] = '0'; + } + + size_t need = (size_t)neg + (size_t)n + (scale ? 1 : 0) + 1; + + if (need > out_cap) { + return -1; + } + char* p = out; + + if (neg) { + *p++ = '-'; + } + + /* emit MSD-first, inserting '.' before `scale` trailing digits */ + for (int i = n - 1; i >= 0; i--) { + if (i + 1 == scale) { + *p++ = '.'; + } + *p++ = buf[i]; + } + *p = '\0'; + return (int)(p - out); } static Datum -read_decimal(const chc_column * col, const chc_type * type, uint64_t row) -{ - size_t es; - const uint8_t *p = (const uint8_t *) chc_column_fixed_data(col, &es); - uint32_t scale = (uint32_t) chc_type_decimal_scale(type); - char buf[80]; - int rc; +read_decimal(const chc_column* col, const chc_type* type, uint64_t row) { + size_t es; + const uint8_t* p = (const uint8_t*)chc_column_fixed_data(col, &es); + uint32_t scale = (uint32_t)chc_type_decimal_scale(type); + char buf[80]; + int rc; #if PG_VERSION_NUM >= 140000 - /* Decimal32/64 fit in int64; skip the byte-array text path. */ - if (es == 4) - return NumericGetDatum(int64_div_fast_to_numeric(rd_i32(p, row), (int) scale)); - if (es == 8) - return NumericGetDatum(int64_div_fast_to_numeric(rd_i64(p, row), (int) scale)); + /* Decimal32/64 fit in int64; skip the byte-array text path. */ + if (es == 4) { + return NumericGetDatum(int64_div_fast_to_numeric(rd_i32(p, row), (int)scale)); + } + if (es == 8) { + return NumericGetDatum(int64_div_fast_to_numeric(rd_i64(p, row), (int)scale)); + } #endif - rc = format_decimal_text(p + row * es, es, scale, buf, sizeof(buf)); - if (rc < 0) - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: decimal too wide"))); - return DirectFunctionCall3(numeric_in, CStringGetDatum(buf), - ObjectIdGetDatum(0), Int32GetDatum(-1)); + rc = format_decimal_text(p + row * es, es, scale, buf, sizeof(buf)); + if (rc < 0) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), errmsg("pg_clickhouse: decimal too wide")) + ); + } + return DirectFunctionCall3( + numeric_in, CStringGetDatum(buf), ObjectIdGetDatum(0), Int32GetDatum(-1) + ); } static Datum -read_string_as_text(const chc_column * col, uint64_t row) -{ - const char *p; - size_t len; +read_string_as_text(const chc_column* col, uint64_t row) { + const char* p; + size_t len; - slice_str(col, row, &p, &len); - return PointerGetDatum(cstring_to_text_with_len(p, len)); + slice_str(col, row, &p, &len); + return PointerGetDatum(cstring_to_text_with_len(p, len)); } static Datum -read_fixedstring_as_text(const chc_column * col, uint64_t row) -{ - size_t width; - const uint8_t *base = chc_column_fixed_data(col, &width); +read_fixedstring_as_text(const chc_column* col, uint64_t row) { + size_t width; + const uint8_t* base = chc_column_fixed_data(col, &width); - return PointerGetDatum(cstring_to_text_with_len((const char *) base + row * width, - width)); + return PointerGetDatum( + cstring_to_text_with_len((const char*)base + row * width, width) + ); } static Datum -read_uuid(const chc_column * col, uint64_t row) -{ - pg_uuid_t *u = (pg_uuid_t *) palloc(sizeof(pg_uuid_t)); - const uint8_t *p = (uint8_t *) chc_column_fixed_data(col, NULL) + row * 16; - uint64_t a, - b; - - memcpy(&a, p, 8); - memcpy(&b, p + 8, 8); - a = pg_hton64(a); - b = pg_hton64(b); - memcpy(u->data, &a, 8); - memcpy(u->data + 8, &b, 8); - return UUIDPGetDatum(u); +read_uuid(const chc_column* col, uint64_t row) { + pg_uuid_t* u = (pg_uuid_t*)palloc(sizeof(pg_uuid_t)); + const uint8_t* p = (uint8_t*)chc_column_fixed_data(col, NULL) + row * 16; + uint64_t a, b; + + memcpy(&a, p, 8); + memcpy(&b, p + 8, 8); + a = pg_hton64(a); + b = pg_hton64(b); + memcpy(u->data, &a, 8); + memcpy(u->data + 8, &b, 8); + return UUIDPGetDatum(u); } /* @@ -262,63 +269,61 @@ read_uuid(const chc_column * col, uint64_t row) * BE bytes of dotted-quad, so pg_hton32 the value into ip_addr directly. */ static Datum -read_ipv4(const chc_column * col, uint64_t row) -{ - inet *res = (inet *) palloc0(sizeof(inet)); - uint32_t addr; - - memcpy(&addr, (const uint8_t *) chc_column_fixed_data(col, NULL) + row * 4, 4); - addr = pg_hton32(addr); - ip_family(res) = PGSQL_AF_INET; - ip_bits(res) = 32; - memcpy(ip_addr(res), &addr, 4); - SET_INET_VARSIZE(res); - return InetPGetDatum(res); +read_ipv4(const chc_column* col, uint64_t row) { + inet* res = (inet*)palloc0(sizeof(inet)); + uint32_t addr; + + memcpy(&addr, (const uint8_t*)chc_column_fixed_data(col, NULL) + row * 4, 4); + addr = pg_hton32(addr); + ip_family(res) = PGSQL_AF_INET; + ip_bits(res) = 32; + memcpy(ip_addr(res), &addr, 4); + SET_INET_VARSIZE(res); + return InetPGetDatum(res); } /* IPv6 wire is already network order; same layout as PG inet ip_addr. */ static Datum -read_ipv6(const chc_column * col, uint64_t row) -{ - inet *res = (inet *) palloc0(sizeof(inet)); - - ip_family(res) = PGSQL_AF_INET6; - ip_bits(res) = 128; - memcpy(ip_addr(res), (const uint8_t *) chc_column_fixed_data(col, NULL) + row * 16, 16); - SET_INET_VARSIZE(res); - return InetPGetDatum(res); +read_ipv6(const chc_column* col, uint64_t row) { + inet* res = (inet*)palloc0(sizeof(inet)); + + ip_family(res) = PGSQL_AF_INET6; + ip_bits(res) = 128; + memcpy( + ip_addr(res), (const uint8_t*)chc_column_fixed_data(col, NULL) + row * 16, 16 + ); + SET_INET_VARSIZE(res); + return InetPGetDatum(res); } static Datum -read_enum_as_text(const chc_column * col, const chc_type * type, uint64_t row) -{ - size_t es; - const uint8_t *p = (const uint8_t *) chc_column_fixed_data(col, &es); - int64_t v = 0; - - if (es == 1) - v = (int8_t) p[row]; - else - { - int16_t t; - - memcpy(&t, p + row * 2, 2); - v = t; - } - - size_t n = chc_type_enum_count(type); - - for (size_t i = 0; i < n; i++) - { - const char *en; - size_t el; - int64_t ev; - - chc_type_enum_at(type, i, &en, &el, &ev); - if (ev == v) - return PointerGetDatum(cstring_to_text_with_len(en ? en : "", el)); - } - return PointerGetDatum(cstring_to_text_with_len("", 0)); +read_enum_as_text(const chc_column* col, const chc_type* type, uint64_t row) { + size_t es; + const uint8_t* p = (const uint8_t*)chc_column_fixed_data(col, &es); + int64_t v = 0; + + if (es == 1) { + v = (int8_t)p[row]; + } else { + int16_t t; + + memcpy(&t, p + row * 2, 2); + v = t; + } + + size_t n = chc_type_enum_count(type); + + for (size_t i = 0; i < n; i++) { + const char* en; + size_t el; + int64_t ev; + + chc_type_enum_at(type, i, &en, &el, &ev); + if (ev == v) { + return PointerGetDatum(cstring_to_text_with_len(en ? en : "", el)); + } + } + return PointerGetDatum(cstring_to_text_with_len("", 0)); } /* @@ -328,21 +333,21 @@ read_enum_as_text(const chc_column * col, const chc_type * type, uint64_t row) * table declares the column as `json`). */ static Datum -read_json(const chc_column * col, uint64_t row, Oid valtype) -{ - const char *p; - size_t len; - char *cstr; - Datum ret; - - slice_str(col, row, &p, &len); - cstr = palloc(len + 1); - memcpy(cstr, p, len); - cstr[len] = '\0'; - ret = DirectFunctionCall1(valtype == JSONOID ? json_in : jsonb_in, - CStringGetDatum(cstr)); - pfree(cstr); - return ret; +read_json(const chc_column* col, uint64_t row, Oid valtype) { + const char* p; + size_t len; + char* cstr; + Datum ret; + + slice_str(col, row, &p, &len); + cstr = palloc(len + 1); + memcpy(cstr, p, len); + cstr[len] = '\0'; + ret = DirectFunctionCall1( + valtype == JSONOID ? json_in : jsonb_in, CStringGetDatum(cstr) + ); + pfree(cstr); + return ret; } /* @@ -351,453 +356,478 @@ read_json(const chc_column * col, uint64_t row, Oid valtype) * sentinel for the Nullable variant. */ static Datum -read_lc_string(const chc_column * col, const chc_type * type, uint64_t row, - Oid * valtype, bool *is_null) -{ - int ks = chc_column_lc_key_size(col); - const uint8_t *kp = (const uint8_t *) chc_column_lc_keys(col) + (size_t) row * ks; - uint64_t k = 0; - - switch (ks) - { - case 1: - k = kp[0]; - break; - case 2: - { - uint16_t v; - - memcpy(&v, kp, 2); - k = v; - break; - } - case 4: - { - uint32_t v; - - memcpy(&v, kp, 4); - k = v; - break; - } - case 8: - memcpy(&k, kp, 8); - break; - default: - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: unexpected LowCardinality key size %d", ks))); - } - - const chc_column *dict = chc_column_lc_dict(col); - const chc_type *inner_t = chc_type_child(type, 0); - - if (chc_type_kind(inner_t) == CHC_NULLABLE - && chc_column_layout(dict) == CHC_COL_NULLABLE) - { - const uint8_t *dnm = chc_column_null_map(dict); - - if (dnm && dnm[k]) - { - *valtype = TEXTOID; - *is_null = true; - return (Datum) 0; - } - dict = chc_column_nullable_inner(dict); - } - - *valtype = TEXTOID; - *is_null = false; - if (chc_column_layout(dict) != CHC_COL_STRING) - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported LowCardinality inner type"))); - return read_string_as_text(dict, k); +read_lc_string( + const chc_column* col, + const chc_type* type, + uint64_t row, + Oid* valtype, + bool* is_null +) { + int ks = chc_column_lc_key_size(col); + const uint8_t* kp = (const uint8_t*)chc_column_lc_keys(col) + (size_t)row * ks; + uint64_t k = 0; + + switch (ks) { + case 1: + k = kp[0]; + break; + case 2: { + uint16_t v; + + memcpy(&v, kp, 2); + k = v; + break; + } + case 4: { + uint32_t v; + + memcpy(&v, kp, 4); + k = v; + break; + } + case 8: + memcpy(&k, kp, 8); + break; + default: + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: unexpected LowCardinality key size %d", ks)) + ); + } + + const chc_column* dict = chc_column_lc_dict(col); + const chc_type* inner_t = chc_type_child(type, 0); + + if (chc_type_kind(inner_t) == CHC_NULLABLE && + chc_column_layout(dict) == CHC_COL_NULLABLE) { + const uint8_t* dnm = chc_column_null_map(dict); + + if (dnm && dnm[k]) { + *valtype = TEXTOID; + *is_null = true; + return (Datum)0; + } + dict = chc_column_nullable_inner(dict); + } + + *valtype = TEXTOID; + *is_null = false; + if (chc_column_layout(dict) != CHC_COL_STRING) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported LowCardinality inner type")) + ); + } + return read_string_as_text(dict, k); } static Datum -read_array(const chc_column * col, const chc_type * type, uint64_t row, - Oid * valtype, bool *is_null) -{ - const uint64_t *offs = chc_column_array_offsets(col); - uint64_t start = row == 0 ? 0 : offs[row - 1]; - uint64_t end = offs[row]; - uint64_t len = end - start; - const chc_type *inner_t = chc_type_child(type, 0); - const chc_column *inner = chc_column_array_values(col); - ch_binary_array_t *slot = (ch_binary_array_t *) palloc(sizeof(ch_binary_array_t)); - const chc_type *leaf = type; - int ndim = 0; - - /* - * postgres uses one array type per element type regardless of nesting, so - * walk past nested Array layers to the leaf scalar type. - */ - while (chc_type_kind(leaf) == CHC_ARRAY) - { - ndim++; - leaf = chc_type_child(leaf, 0); - } - - slot->len = len; - slot->ndim = ndim; - slot->item_type = ch_kind_to_pg_oid(leaf); - slot->array_type = get_array_type(slot->item_type); - if (slot->array_type == InvalidOid) - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: could not find array type for column type \"%s\"", - chc_type_name(leaf, NULL)))); - - if (len > 0) - { - Oid scratch = slot->item_type; - - slot->datums = (Datum *) palloc0(sizeof(Datum) * len); - slot->nulls = (bool *) palloc0(sizeof(bool) * len); - - /* - * For ndim==1 read_value returns leaf scalars; for ndim>1 inner_t is - * itself CHC_ARRAY so recursion produces nested ch_binary_array_t*. - * Use a scratch valtype to avoid clobbering slot->item_type. - */ - for (uint64_t i = 0; i < len; ++i) - slot->datums[i] = read_value(inner, inner_t, start + i, - &scratch, &slot->nulls[i]); - } - else - { - slot->datums = NULL; - slot->nulls = NULL; - } - - *valtype = ANYARRAYOID; - *is_null = false; - return PointerGetDatum(slot); +read_array( + const chc_column* col, + const chc_type* type, + uint64_t row, + Oid* valtype, + bool* is_null +) { + const uint64_t* offs = chc_column_array_offsets(col); + uint64_t start = row == 0 ? 0 : offs[row - 1]; + uint64_t end = offs[row]; + uint64_t len = end - start; + const chc_type* inner_t = chc_type_child(type, 0); + const chc_column* inner = chc_column_array_values(col); + ch_binary_array_t* slot = (ch_binary_array_t*)palloc(sizeof(ch_binary_array_t)); + const chc_type* leaf = type; + int ndim = 0; + + /* + * postgres uses one array type per element type regardless of nesting, so + * walk past nested Array layers to the leaf scalar type. + */ + while (chc_type_kind(leaf) == CHC_ARRAY) { + ndim++; + leaf = chc_type_child(leaf, 0); + } + + slot->len = len; + slot->ndim = ndim; + slot->item_type = ch_kind_to_pg_oid(leaf); + slot->array_type = get_array_type(slot->item_type); + if (slot->array_type == InvalidOid) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: could not find array type for column type \"%s\"", + chc_type_name(leaf, NULL) + )) + ); + } + + if (len > 0) { + Oid scratch = slot->item_type; + + slot->datums = (Datum*)palloc0(sizeof(Datum) * len); + slot->nulls = (bool*)palloc0(sizeof(bool) * len); + + /* + * For ndim==1 read_value returns leaf scalars; for ndim>1 inner_t is + * itself CHC_ARRAY so recursion produces nested ch_binary_array_t*. + * Use a scratch valtype to avoid clobbering slot->item_type. + */ + for (uint64_t i = 0; i < len; ++i) { + slot->datums[i] = + read_value(inner, inner_t, start + i, &scratch, &slot->nulls[i]); + } + } else { + slot->datums = NULL; + slot->nulls = NULL; + } + + *valtype = ANYARRAYOID; + *is_null = false; + return PointerGetDatum(slot); } static Datum -read_tuple(const chc_column * col, const chc_type * type, uint64_t row, - Oid * valtype, bool *is_null) -{ - size_t n = chc_type_n_children(type); - ch_binary_tuple_t *slot; - - if (n == 0) - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: returned tuple is empty"))); - - slot = (ch_binary_tuple_t *) palloc(sizeof(ch_binary_tuple_t)); - slot->datums = (Datum *) palloc(sizeof(Datum) * n); - slot->nulls = (bool *) palloc0(sizeof(bool) * n); - slot->types = (Oid *) palloc0(sizeof(Oid) * n); - slot->len = n; - slot->ch_type_name = chc_type_name(type, NULL); - - for (size_t i = 0; i < n; ++i) - { - const chc_type *ft = chc_type_child(type, i); - const chc_column *fc = chc_column_tuple_child(col, i); - - slot->datums[i] = read_value(fc, ft, row, &slot->types[i], &slot->nulls[i]); - } - - *valtype = RECORDOID; - *is_null = false; - return PointerGetDatum(slot); +read_tuple( + const chc_column* col, + const chc_type* type, + uint64_t row, + Oid* valtype, + bool* is_null +) { + size_t n = chc_type_n_children(type); + ch_binary_tuple_t* slot; + + if (n == 0) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: returned tuple is empty")) + ); + } + + slot = (ch_binary_tuple_t*)palloc(sizeof(ch_binary_tuple_t)); + slot->datums = (Datum*)palloc(sizeof(Datum) * n); + slot->nulls = (bool*)palloc0(sizeof(bool) * n); + slot->types = (Oid*)palloc0(sizeof(Oid) * n); + slot->len = n; + slot->ch_type_name = chc_type_name(type, NULL); + + for (size_t i = 0; i < n; ++i) { + const chc_type* ft = chc_type_child(type, i); + const chc_column* fc = chc_column_tuple_child(col, i); + + slot->datums[i] = read_value(fc, ft, row, &slot->types[i], &slot->nulls[i]); + } + + *valtype = RECORDOID; + *is_null = false; + return PointerGetDatum(slot); } static Datum -read_value(const chc_column * col, const chc_type * type, uint64_t row, - Oid * valtype, bool *is_null) -{ - /* Unwrap outer Nullable, handling nulls here. */ - if (chc_type_kind(type) == CHC_NULLABLE) - { - const chc_type *inner_t = chc_type_child(type, 0); - - if (chc_column_layout(col) == CHC_COL_NULLABLE) - { - const uint8_t *nm = chc_column_null_map(col); - - if (nm && nm[row]) - { - *valtype = ch_kind_to_pg_oid(inner_t); - *is_null = true; - return (Datum) 0; - } - col = chc_column_nullable_inner(col); - } - type = inner_t; - } - *is_null = false; - - switch (chc_type_kind(type)) - { - case CHC_VOID: - case CHC_NOTHING: - *valtype = InvalidOid; - *is_null = true; - return (Datum) 0; - case CHC_UINT8: - *valtype = INT2OID; - return (Datum) rd_u8((const uint8_t *) chc_column_fixed_data(col, NULL), row); - case CHC_BOOL: - *valtype = BOOLOID; - return (Datum) rd_bool((const bool *) chc_column_fixed_data(col, NULL), row); - case CHC_INT8: - *valtype = INT2OID; - return (Datum) rd_i8((const uint8_t *) chc_column_fixed_data(col, NULL), row); - case CHC_INT16: - *valtype = INT2OID; - return (Datum) rd_i16((const uint8_t *) chc_column_fixed_data(col, NULL), row); - case CHC_UINT16: - *valtype = INT4OID; - return (Datum) rd_u16((const uint8_t *) chc_column_fixed_data(col, NULL), row); - case CHC_INT32: - *valtype = INT4OID; - return (Datum) rd_i32((const uint8_t *) chc_column_fixed_data(col, NULL), row); - case CHC_UINT32: - *valtype = INT8OID; - return Int64GetDatum((int64) rd_u32((const uint8_t *) chc_column_fixed_data(col, NULL), row)); - case CHC_INT64: - *valtype = INT8OID; - return Int64GetDatum(rd_i64((const uint8_t *) chc_column_fixed_data(col, NULL), row)); - case CHC_UINT64: - { - uint64_t v = rd_u64((const uint8_t *) chc_column_fixed_data(col, NULL), row); - - if (v > (uint64_t) PG_INT64_MAX) - ereport(ERROR, - (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), - errmsg("value %" PRIu64 " is out of range of bigint", - v))); - *valtype = INT8OID; - return Int64GetDatum((int64) v); - } - case CHC_FLOAT32: - *valtype = FLOAT4OID; - return Float4GetDatum(rd_f32((const uint8_t *) chc_column_fixed_data(col, NULL), row)); - case CHC_FLOAT64: - *valtype = FLOAT8OID; - return Float8GetDatum(rd_f64((const uint8_t *) chc_column_fixed_data(col, NULL), row)); - case CHC_DECIMAL32: - case CHC_DECIMAL64: - case CHC_DECIMAL128: - case CHC_DECIMAL256: - *valtype = NUMERICOID; - return read_decimal(col, type, row); - case CHC_STRING: - *valtype = TEXTOID; - return read_string_as_text(col, row); - case CHC_ENUM8: - case CHC_ENUM16: - *valtype = TEXTOID; - return read_enum_as_text(col, type, row); - case CHC_JSON: - case CHC_OBJECT: - { - /* - * *valtype arrives set to JSONBOID by default (from - * ch_kind_to_pg_oid) but the caller's foreign-table - * declaration may have requested JSONOID; honor it. - */ - Oid target = (*valtype == JSONOID) ? JSONOID : JSONBOID; - - *valtype = target; - return read_json(col, row, target); - } - case CHC_FIXED_STRING: - *valtype = TEXTOID; - return read_fixedstring_as_text(col, row); - case CHC_DATE: - *valtype = DATEOID; - return DateADTGetDatum((DateADT) rd_u16((const uint8_t *) chc_column_fixed_data(col, NULL), row) - - CH_TO_PG_DATE_OFFSET); - case CHC_DATE32: - *valtype = DATEOID; - return DateADTGetDatum((DateADT) rd_i32((const uint8_t *) chc_column_fixed_data(col, NULL), row) - - CH_TO_PG_DATE_OFFSET); - case CHC_DATETIME: - { - uint32_t secs = rd_u32((const uint8_t *) chc_column_fixed_data(col, NULL), row); - - *valtype = TIMESTAMPTZOID; - return TimestampTzGetDatum(time_t_to_timestamptz((pg_time_t) secs)); - } - case CHC_DATETIME64: - { - int64 raw = rd_i64((const uint8_t *) chc_column_fixed_data(col, NULL), row); - uint32_t scale = chc_type_datetime64_scale(type); - int64 power = pow10i[scale]; - - *valtype = TIMESTAMPTZOID; - /* multiply before divide so scale > 6 keeps sub-second us */ - return TimestampTzGetDatum(time_t_to_timestamptz(raw / power) - + (raw % power) * USECS_PER_SEC / power); - } - case CHC_UUID: - *valtype = UUIDOID; - return read_uuid(col, row); - case CHC_IPV4: - *valtype = INETOID; - return read_ipv4(col, row); - case CHC_IPV6: - *valtype = INETOID; - return read_ipv6(col, row); - case CHC_LOW_CARDINALITY: - return read_lc_string(col, type, row, valtype, is_null); - case CHC_ARRAY: - return read_array(col, type, row, valtype, is_null); - case CHC_TUPLE: - return read_tuple(col, type, row, valtype, is_null); - default: - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported type in binary protocol"))); - } - /* unreachable */ - return (Datum) 0; +read_value( + const chc_column* col, + const chc_type* type, + uint64_t row, + Oid* valtype, + bool* is_null +) { + /* Unwrap outer Nullable, handling nulls here. */ + if (chc_type_kind(type) == CHC_NULLABLE) { + const chc_type* inner_t = chc_type_child(type, 0); + + if (chc_column_layout(col) == CHC_COL_NULLABLE) { + const uint8_t* nm = chc_column_null_map(col); + + if (nm && nm[row]) { + *valtype = ch_kind_to_pg_oid(inner_t); + *is_null = true; + return (Datum)0; + } + col = chc_column_nullable_inner(col); + } + type = inner_t; + } + *is_null = false; + + switch (chc_type_kind(type)) { + case CHC_VOID: + case CHC_NOTHING: + *valtype = InvalidOid; + *is_null = true; + return (Datum)0; + case CHC_UINT8: + *valtype = INT2OID; + return (Datum)rd_u8((const uint8_t*)chc_column_fixed_data(col, NULL), row); + case CHC_BOOL: + *valtype = BOOLOID; + return (Datum)rd_bool((const bool*)chc_column_fixed_data(col, NULL), row); + case CHC_INT8: + *valtype = INT2OID; + return (Datum)rd_i8((const uint8_t*)chc_column_fixed_data(col, NULL), row); + case CHC_INT16: + *valtype = INT2OID; + return (Datum)rd_i16((const uint8_t*)chc_column_fixed_data(col, NULL), row); + case CHC_UINT16: + *valtype = INT4OID; + return (Datum)rd_u16((const uint8_t*)chc_column_fixed_data(col, NULL), row); + case CHC_INT32: + *valtype = INT4OID; + return (Datum)rd_i32((const uint8_t*)chc_column_fixed_data(col, NULL), row); + case CHC_UINT32: + *valtype = INT8OID; + return Int64GetDatum( + (int64)rd_u32((const uint8_t*)chc_column_fixed_data(col, NULL), row) + ); + case CHC_INT64: + *valtype = INT8OID; + return Int64GetDatum( + rd_i64((const uint8_t*)chc_column_fixed_data(col, NULL), row) + ); + case CHC_UINT64: { + uint64_t v = rd_u64((const uint8_t*)chc_column_fixed_data(col, NULL), row); + + if (v > (uint64_t)PG_INT64_MAX) { + ereport( + ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("value %" PRIu64 " is out of range of bigint", v)) + ); + } + *valtype = INT8OID; + return Int64GetDatum((int64)v); + } + case CHC_FLOAT32: + *valtype = FLOAT4OID; + return Float4GetDatum( + rd_f32((const uint8_t*)chc_column_fixed_data(col, NULL), row) + ); + case CHC_FLOAT64: + *valtype = FLOAT8OID; + return Float8GetDatum( + rd_f64((const uint8_t*)chc_column_fixed_data(col, NULL), row) + ); + case CHC_DECIMAL32: + case CHC_DECIMAL64: + case CHC_DECIMAL128: + case CHC_DECIMAL256: + *valtype = NUMERICOID; + return read_decimal(col, type, row); + case CHC_STRING: + *valtype = TEXTOID; + return read_string_as_text(col, row); + case CHC_ENUM8: + case CHC_ENUM16: + *valtype = TEXTOID; + return read_enum_as_text(col, type, row); + case CHC_JSON: + case CHC_OBJECT: { + /* + * *valtype arrives set to JSONBOID by default (from + * ch_kind_to_pg_oid) but the caller's foreign-table + * declaration may have requested JSONOID; honor it. + */ + Oid target = (*valtype == JSONOID) ? JSONOID : JSONBOID; + + *valtype = target; + return read_json(col, row, target); + } + case CHC_FIXED_STRING: + *valtype = TEXTOID; + return read_fixedstring_as_text(col, row); + case CHC_DATE: + *valtype = DATEOID; + return DateADTGetDatum( + (DateADT)rd_u16((const uint8_t*)chc_column_fixed_data(col, NULL), row) - + CH_TO_PG_DATE_OFFSET + ); + case CHC_DATE32: + *valtype = DATEOID; + return DateADTGetDatum( + (DateADT)rd_i32((const uint8_t*)chc_column_fixed_data(col, NULL), row) - + CH_TO_PG_DATE_OFFSET + ); + case CHC_DATETIME: { + uint32_t secs = rd_u32((const uint8_t*)chc_column_fixed_data(col, NULL), row); + + *valtype = TIMESTAMPTZOID; + return TimestampTzGetDatum(time_t_to_timestamptz((pg_time_t)secs)); + } + case CHC_DATETIME64: { + int64 raw = rd_i64((const uint8_t*)chc_column_fixed_data(col, NULL), row); + uint32_t scale = chc_type_datetime64_scale(type); + int64 power = pow10i[scale]; + + *valtype = TIMESTAMPTZOID; + /* multiply before divide so scale > 6 keeps sub-second us */ + return TimestampTzGetDatum( + time_t_to_timestamptz(raw / power) + (raw % power) * USECS_PER_SEC / power + ); + } + case CHC_UUID: + *valtype = UUIDOID; + return read_uuid(col, row); + case CHC_IPV4: + *valtype = INETOID; + return read_ipv4(col, row); + case CHC_IPV6: + *valtype = INETOID; + return read_ipv6(col, row); + case CHC_LOW_CARDINALITY: + return read_lc_string(col, type, row, valtype, is_null); + case CHC_ARRAY: + return read_array(col, type, row, valtype, is_null); + case CHC_TUPLE: + return read_tuple(col, type, row, valtype, is_null); + default: + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: unsupported type in binary protocol")) + ); + } + /* unreachable */ + return (Datum)0; } /* ---- read state ----------------------------------------------------- */ static bool -load_block(ch_binary_read_state_t * state) -{ - state->cur = ch_binary_response_fetch_next_block(state->resp); - if (state->cur == NULL) - { - const char *resp_err = ch_binary_response_error(state->resp); - - if (resp_err) - state->error = pstrdup(resp_err); - state->done = true; - return false; - } - return true; +load_block(ch_binary_read_state_t* state) { + state->cur = ch_binary_response_fetch_next_block(state->resp); + if (state->cur == NULL) { + const char* resp_err = ch_binary_response_error(state->resp); + + if (resp_err) { + state->error = pstrdup(resp_err); + } + state->done = true; + return false; + } + return true; } void -ch_binary_read_state_init(ch_binary_read_state_t * state, ch_binary_response_t * resp) -{ - const char *resp_err; - size_t ncols; - - state->resp = resp; - state->block = 0; - state->row = 0; - state->done = false; - state->error = NULL; - state->coltypes = NULL; - state->values = NULL; - state->nulls = NULL; - state->cur = NULL; - - resp_err = ch_binary_response_error(resp); - if (resp_err) - { - state->done = true; - state->error = pstrdup(resp_err); - return; - } - - ncols = ch_binary_response_columns(resp); - if (ncols == 0) - { - state->done = true; - return; - } - - state->coltypes = palloc0(sizeof(Oid) * ncols); - state->values = palloc0(sizeof(Datum) * ncols); - state->nulls = palloc0(sizeof(bool) * ncols); - - if (!load_block(state)) - return; - - for (size_t i = 0; i < ncols; i++) - state->coltypes[i] = ch_kind_to_pg_oid(chc_block_column_type(state->cur, i)); +ch_binary_read_state_init(ch_binary_read_state_t* state, ch_binary_response_t* resp) { + const char* resp_err; + size_t ncols; + + state->resp = resp; + state->block = 0; + state->row = 0; + state->done = false; + state->error = NULL; + state->coltypes = NULL; + state->values = NULL; + state->nulls = NULL; + state->cur = NULL; + + resp_err = ch_binary_response_error(resp); + if (resp_err) { + state->done = true; + state->error = pstrdup(resp_err); + return; + } + + ncols = ch_binary_response_columns(resp); + if (ncols == 0) { + state->done = true; + return; + } + + state->coltypes = palloc0(sizeof(Oid) * ncols); + state->values = palloc0(sizeof(Datum) * ncols); + state->nulls = palloc0(sizeof(bool) * ncols); + + if (!load_block(state)) { + return; + } + + for (size_t i = 0; i < ncols; i++) { + state->coltypes[i] = ch_kind_to_pg_oid(chc_block_column_type(state->cur, i)); + } } bool -ch_binary_read_row(ch_binary_read_state_t * state) -{ - size_t ncols; +ch_binary_read_row(ch_binary_read_state_t* state) { + size_t ncols; - if (state->done || state->coltypes == NULL || state->error) - return false; + if (state->done || state->coltypes == NULL || state->error) { + return false; + } - ncols = ch_binary_response_columns(state->resp); + ncols = ch_binary_response_columns(state->resp); again: - if (state->cur == NULL) - { - if (!load_block(state)) - return false; - } - - if (state->row >= chc_block_n_rows(state->cur)) - { - state->row = 0; - state->cur = NULL; - goto again; - } - - PG_TRY(); - { - for (size_t i = 0; i < ncols; i++) - { - /* - * Currently, read_value overwrites *valtype for most types - * canonical PG type for that CH kind. For CHC_JSON we honor the - * incoming value so callers (binary_fetch_row) can pin the Datum - * type to JSONOID for `data json` foreign columns and avoid the - * jsonb_in -> jsonb_out round-trip that would strip CH's verbatim - * STRING- serialized formatting. - */ - Oid t = state->coltypes[i]; - const chc_column *col = chc_block_column(state->cur, i); - const chc_type *ct = chc_block_column_type(state->cur, i); - - state->values[i] = read_value(col, ct, state->row, &t, &state->nulls[i]); - } - } - PG_CATCH(); - { - MemoryContext mcxt = GetMemoryChunkContext(state); - MemoryContext oldcxt; - ErrorData *edata; - const char *msg; - static const char prefix[] = "pg_clickhouse: "; - - oldcxt = MemoryContextSwitchTo(mcxt); - edata = CopyErrorData(); - msg = edata->message ? edata->message : "unknown error"; - - /* - * binary_fetch_row re-prefixes with "pg_clickhouse: error while - * reading row:"; drop a leading "pg_clickhouse: " here so the final - * message doesn't carry it twice. - */ - if (strncmp(msg, prefix, sizeof(prefix) - 1) == 0) - msg += sizeof(prefix) - 1; - state->error = pstrdup(msg); - FlushErrorState(); - FreeErrorData(edata); - MemoryContextSwitchTo(oldcxt); - state->done = true; - return false; - } - PG_END_TRY(); - - state->row++; - return true; + if (state->cur == NULL) { + if (!load_block(state)) { + return false; + } + } + + if (state->row >= chc_block_n_rows(state->cur)) { + state->row = 0; + state->cur = NULL; + goto again; + } + + PG_TRY(); + { + for (size_t i = 0; i < ncols; i++) { + /* + * Currently, read_value overwrites *valtype for most types + * canonical PG type for that CH kind. For CHC_JSON we honor the + * incoming value so callers (binary_fetch_row) can pin the Datum + * type to JSONOID for `data json` foreign columns and avoid the + * jsonb_in -> jsonb_out round-trip that would strip CH's verbatim + * STRING- serialized formatting. + */ + Oid t = state->coltypes[i]; + const chc_column* col = chc_block_column(state->cur, i); + const chc_type* ct = chc_block_column_type(state->cur, i); + + state->values[i] = read_value(col, ct, state->row, &t, &state->nulls[i]); + } + } + PG_CATCH(); + { + MemoryContext mcxt = GetMemoryChunkContext(state); + MemoryContext oldcxt; + ErrorData* edata; + const char* msg; + static const char prefix[] = "pg_clickhouse: "; + + oldcxt = MemoryContextSwitchTo(mcxt); + edata = CopyErrorData(); + msg = edata->message ? edata->message : "unknown error"; + + /* + * binary_fetch_row re-prefixes with "pg_clickhouse: error while + * reading row:"; drop a leading "pg_clickhouse: " here so the final + * message doesn't carry it twice. + */ + if (strncmp(msg, prefix, sizeof(prefix) - 1) == 0) { + msg += sizeof(prefix) - 1; + } + state->error = pstrdup(msg); + FlushErrorState(); + FreeErrorData(edata); + MemoryContextSwitchTo(oldcxt); + state->done = true; + return false; + } + PG_END_TRY(); + + state->row++; + return true; } void -ch_binary_read_state_free(ch_binary_read_state_t * state) -{ - /* state->error is palloc'd; freed with surrounding memory context. */ - state->error = NULL; +ch_binary_read_state_free(ch_binary_read_state_t* state) { + /* state->error is palloc'd; freed with surrounding memory context. */ + state->error = NULL; } diff --git a/src/binary/encode.c b/src/binary/encode.c index 3d761ef..160550e 100644 --- a/src/binary/encode.c +++ b/src/binary/encode.c @@ -8,12 +8,12 @@ #include "postgres.h" #include -#include /* AF_INET, expanded by PG inet macros */ +#include /* AF_INET, expanded by PG inet macros */ #include "access/tupdesc.h" #include "catalog/pg_type_d.h" -#include "nodes/pg_list.h" #include "fmgr.h" +#include "nodes/pg_list.h" #include "utils/builtins.h" #include "utils/date.h" #include "utils/inet.h" @@ -35,92 +35,107 @@ * constructing the TupleDesc for INSERT...VALUES. */ static Oid -ch_kind_to_pg_oid_for_insert(const chc_type * type, const char *colname) -{ - chc_kind kind = chc_type_kind(type); - Oid oid = ch_scalar_oids[kind]; - - if (OidIsValid(oid)) - return oid; - - switch (kind) - { - case CHC_ARRAY: - { - /* - * postgres uses one array type per element type regardless of - * nesting, so unwrap nested Array layers before looking up. - */ - const chc_type *leaf = type; - Oid item_type; - Oid array_type; - - while (chc_type_kind(leaf) == CHC_ARRAY) - leaf = chc_type_child(leaf, 0); - item_type = ch_kind_to_pg_oid_for_insert(leaf, colname); - array_type = get_array_type(item_type); - - if (array_type == InvalidOid) - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: could not find array type for column \"%s\"", - colname ? colname : "?"))); - return array_type; - } - case CHC_TUPLE: - return RECORDOID; - case CHC_NULLABLE: - case CHC_LOW_CARDINALITY: - return ch_kind_to_pg_oid_for_insert(chc_type_child(type, 0), colname); - default: - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported column type \"%s\" for \"%s\"", - chc_type_name(type, NULL), - colname ? colname : "?"))); - } - return InvalidOid; +ch_kind_to_pg_oid_for_insert(const chc_type* type, const char* colname) { + chc_kind kind = chc_type_kind(type); + Oid oid = ch_scalar_oids[kind]; + + if (OidIsValid(oid)) { + return oid; + } + + switch (kind) { + case CHC_ARRAY: { + /* + * postgres uses one array type per element type regardless of + * nesting, so unwrap nested Array layers before looking up. + */ + const chc_type* leaf = type; + Oid item_type; + Oid array_type; + + while (chc_type_kind(leaf) == CHC_ARRAY) { + leaf = chc_type_child(leaf, 0); + } + item_type = ch_kind_to_pg_oid_for_insert(leaf, colname); + array_type = get_array_type(item_type); + + if (array_type == InvalidOid) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: could not find array type for column \"%s\"", + colname ? colname : "?" + )) + ); + } + return array_type; + } + case CHC_TUPLE: + return RECORDOID; + case CHC_NULLABLE: + case CHC_LOW_CARDINALITY: + return ch_kind_to_pg_oid_for_insert(chc_type_child(type, 0), colname); + default: + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: unsupported column type \"%s\" for \"%s\"", + chc_type_name(type, NULL), + colname ? colname : "?" + )) + ); + } + return InvalidOid; } void -ch_binary_prepare_insert(void *conn, const ch_query * query, - ch_binary_insert_state * state) -{ - ch_binary_column_info *cols = NULL; - size_t n = 0; - ch_binary_insert_handle *h; - - h = ch_binary_begin_insert((ch_binary_connection_t *) conn, query, &cols, &n); - - state->len = n; - state->insert_block = h; - - if (n == 0) - return; - - state->outdesc = CreateTemplateTupleDesc(n); - for (size_t i = 0; i < n; i++) - { - Oid pg_type = ch_kind_to_pg_oid_for_insert(cols[i].type, cols[i].name); - - /* - * CHC_JSON defaults to JSONBOID; honor foreign-table's JSONOID - * declaration when present so outdesc matches the source slot and no - * json↔jsonb conversion is needed. - */ - if (pg_type == JSONBOID && query->tupdesc && query->attr_nums) - { - AttrNumber attnum = list_nth_int((List *) query->attr_nums, i); - - if (attnum >= 1 && attnum <= query->tupdesc->natts && - TupleDescAttr(query->tupdesc, attnum - 1)->atttypid == JSONOID) - pg_type = JSONOID; - } - - TupleDescInitEntry(state->outdesc, (AttrNumber) (i + 1), - cols[i].name ? cols[i].name : "", - pg_type, -1, 0); - } +ch_binary_prepare_insert( + void* conn, + const ch_query* query, + ch_binary_insert_state* state +) { + ch_binary_column_info* cols = NULL; + size_t n = 0; + ch_binary_insert_handle* h; + + h = ch_binary_begin_insert((ch_binary_connection_t*)conn, query, &cols, &n); + + state->len = n; + state->insert_block = h; + + if (n == 0) { + return; + } + + state->outdesc = CreateTemplateTupleDesc(n); + for (size_t i = 0; i < n; i++) { + Oid pg_type = ch_kind_to_pg_oid_for_insert(cols[i].type, cols[i].name); + + /* + * CHC_JSON defaults to JSONBOID; honor foreign-table's JSONOID + * declaration when present so outdesc matches the source slot and no + * json↔jsonb conversion is needed. + */ + if (pg_type == JSONBOID && query->tupdesc && query->attr_nums) { + AttrNumber attnum = list_nth_int((List*)query->attr_nums, i); + + if (attnum >= 1 && attnum <= query->tupdesc->natts && + TupleDescAttr(query->tupdesc, attnum - 1)->atttypid == JSONOID) { + pg_type = JSONOID; + } + } + + TupleDescInitEntry( + state->outdesc, + (AttrNumber)(i + 1), + cols[i].name ? cols[i].name : "", + pg_type, + -1, + 0 + ); + } } /* @@ -128,269 +143,272 @@ ch_binary_prepare_insert(void *conn, const ch_query * query, * current column, dispatching on (PG Oid, CH kind). ereports on mismatch. */ static void -append_one(ch_binary_insert_handle * h, size_t colidx, - chc_kind kind, Datum val, Oid valtype, bool isnull) -{ - switch (valtype) - { - case INT2OID: - case INT4OID: - case INT8OID: - { - int64_t v = 0; - - /* Support mixing integer types. */ - if (!(kind == CHC_BOOL - || (kind >= CHC_INT8 && kind <= CHC_INT64) - || (kind >= CHC_UINT8 && kind <= CHC_UINT64))) - goto type_mismatch; - if (!isnull) - { - if (valtype == INT2OID) - v = (int64_t) DatumGetInt16(val); - else if (valtype == INT4OID) - v = (int64_t) DatumGetInt32(val); - else - v = DatumGetInt64(val); - } - ch_binary_append_int(h, colidx, v, isnull); - return; - } - case BOOLOID: - if (kind != CHC_BOOL && kind != CHC_UINT8) - goto type_mismatch; - ch_binary_append_bool(h, colidx, (bool) val, isnull); - return; - case FLOAT4OID: - if (kind != CHC_FLOAT32) - goto type_mismatch; - ch_binary_append_float(h, colidx, DatumGetFloat4(val), isnull); - return; - case FLOAT8OID: - if (kind != CHC_FLOAT64) - goto type_mismatch; - ch_binary_append_double(h, colidx, DatumGetFloat8(val), isnull); - return; - case NUMERICOID: - { - char *s = NULL; - - if (kind != CHC_DECIMAL32 && kind != CHC_DECIMAL64 - && kind != CHC_DECIMAL128 && kind != CHC_DECIMAL256) - goto type_mismatch; - if (!isnull) - s = DatumGetCString(DirectFunctionCall1(numeric_out, val)); - ch_binary_append_decimal(h, colidx, s, isnull); - if (s) - pfree(s); - return; - } - case TEXTOID: - { - const char *p = NULL; - size_t len = 0; - text *string = NULL; - - if (!isnull) - { - string = PG_DETOAST_DATUM(val); - p = VARDATA(string); - len = VARSIZE_ANY_EXHDR(string); - } - switch (kind) - { - case CHC_FIXED_STRING: - case CHC_STRING: - case CHC_ENUM8: - case CHC_ENUM16: - ch_binary_append_bytes(h, colidx, p, len, isnull); - return; - default: - goto type_mismatch; - } - } - case DATEOID: - { - int64_t seconds = 0; - - if (kind != CHC_DATE && kind != CHC_DATE32) - goto type_mismatch; - if (!isnull) - seconds = ((int64_t) DatumGetDateADT(val) - + CH_TO_PG_DATE_OFFSET) * SECS_PER_DAY; - ch_binary_append_date_seconds(h, colidx, seconds, isnull); - return; - } - case TIMESTAMPOID: - case TIMESTAMPTZOID: - { - switch (kind) - { - case CHC_DATETIME: - { - int64_t seconds = isnull - ? 0 - : (int64_t) timestamptz_to_time_t(DatumGetTimestamp(val)); - - ch_binary_append_datetime_seconds(h, colidx, seconds, isnull); - } break; - case CHC_DATETIME64: - { - int64_t raw = 0; - - if (!isnull) - { - uint32_t prec = ch_binary_column_datetime64_precision(h, colidx); - Timestamp t = DatumGetTimestamp(val); - int64 power = pow10i[prec]; - int64 secs = t / USECS_PER_SEC; - int64 us_rem = t % USECS_PER_SEC; - - /* - * floor-divide; C trunc-to-zero leaves - * negative remainder - */ - if (us_rem < 0) - { - secs -= 1; - us_rem += USECS_PER_SEC; - } - secs += CH_TO_PG_DATE_OFFSET * SECS_PER_DAY; - raw = secs * power + us_rem * power / USECS_PER_SEC; - } - ch_binary_append_datetime64_raw(h, colidx, raw, isnull); - } break; - default: - goto type_mismatch; - } - return; - } - case ANYARRAYOID: - { - ch_binary_array_t *arr; - chc_kind item_kind; - Oid child_valtype; - - if (kind != CHC_ARRAY) - goto type_mismatch; - - arr = (ch_binary_array_t *) DatumGetPointer(val); - ch_binary_array_begin(h, colidx); - - /* - * While array_begin is active ch_binary_column_kind targets - * the inner element kind. For nested arrays the children are - * themselves ch_binary_array_t* so recurse with ANYARRAYOID; - * at the leaf level use the scalar item_type. - */ - item_kind = ch_binary_column_kind(h, colidx); - child_valtype = (arr->ndim > 1) ? ANYARRAYOID : arr->item_type; - for (size_t i = 0; i < arr->len; i++) - append_one(h, 0, item_kind, arr->datums[i], child_valtype, arr->nulls[i]); - - ch_binary_array_end(h); - return; - } - case UUIDOID: - { - uint8_t bytes[16]; - - if (kind != CHC_UUID) - goto type_mismatch; - if (!isnull) - memcpy(bytes, DatumGetUUIDP(val)->data, 16); - else - memset(bytes, 0, 16); - ch_binary_append_uuid(h, colidx, bytes, isnull); - return; - } - case INETOID: - { - const uint8_t *addr = NULL; - size_t addrlen = 0; - - if (kind != CHC_IPV4 && kind != CHC_IPV6) - goto type_mismatch; - if (!isnull) - { - inet *ipa = DatumGetInetPP(val); - int fam = ip_family(ipa); - int expected = (kind == CHC_IPV4) ? PGSQL_AF_INET : PGSQL_AF_INET6; - - if (fam != expected) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: inet family mismatch for column %zu", - colidx))); - addr = ip_addr(ipa); - addrlen = ip_addrsize(ipa); - } - else - addrlen = (kind == CHC_IPV4) ? 4 : 16; - ch_binary_append_inet(h, colidx, addr, addrlen, isnull); - return; - } - case JSONOID: - case JSONBOID: - { - char *s = NULL; - size_t len = 0; - - if (kind != CHC_JSON && kind != CHC_OBJECT) - goto type_mismatch; - if (!isnull) - { - s = DatumGetCString(DirectFunctionCall1(valtype == JSONBOID - ? jsonb_out - : json_out, - val)); - len = strlen(s); - } - ch_binary_append_bytes(h, colidx, s, len, isnull); - if (s) - pfree(s); - return; - } - default: - goto type_mismatch; - } +append_one( + ch_binary_insert_handle* h, + size_t colidx, + chc_kind kind, + Datum val, + Oid valtype, + bool isnull +) { + switch (valtype) { + case INT2OID: + case INT4OID: + case INT8OID: { + int64_t v = 0; + + /* Support mixing integer types. */ + if (!(kind == CHC_BOOL || (kind >= CHC_INT8 && kind <= CHC_INT64) || + (kind >= CHC_UINT8 && kind <= CHC_UINT64))) { + goto type_mismatch; + } + if (!isnull) { + if (valtype == INT2OID) { + v = (int64_t)DatumGetInt16(val); + } else if (valtype == INT4OID) { + v = (int64_t)DatumGetInt32(val); + } else { + v = DatumGetInt64(val); + } + } + ch_binary_append_int(h, colidx, v, isnull); + return; + } + case BOOLOID: + if (kind != CHC_BOOL && kind != CHC_UINT8) { + goto type_mismatch; + } + ch_binary_append_bool(h, colidx, (bool)val, isnull); + return; + case FLOAT4OID: + if (kind != CHC_FLOAT32) { + goto type_mismatch; + } + ch_binary_append_float(h, colidx, DatumGetFloat4(val), isnull); + return; + case FLOAT8OID: + if (kind != CHC_FLOAT64) { + goto type_mismatch; + } + ch_binary_append_double(h, colidx, DatumGetFloat8(val), isnull); + return; + case NUMERICOID: { + char* s = NULL; + + if (kind != CHC_DECIMAL32 && kind != CHC_DECIMAL64 && kind != CHC_DECIMAL128 && + kind != CHC_DECIMAL256) { + goto type_mismatch; + } + if (!isnull) { + s = DatumGetCString(DirectFunctionCall1(numeric_out, val)); + } + ch_binary_append_decimal(h, colidx, s, isnull); + if (s) { + pfree(s); + } + return; + } + case TEXTOID: { + const char* p = NULL; + size_t len = 0; + text* string = NULL; + + if (!isnull) { + string = PG_DETOAST_DATUM(val); + p = VARDATA(string); + len = VARSIZE_ANY_EXHDR(string); + } + switch (kind) { + case CHC_FIXED_STRING: + case CHC_STRING: + case CHC_ENUM8: + case CHC_ENUM16: + ch_binary_append_bytes(h, colidx, p, len, isnull); + return; + default: + goto type_mismatch; + } + } + case DATEOID: { + int64_t seconds = 0; + + if (kind != CHC_DATE && kind != CHC_DATE32) { + goto type_mismatch; + } + if (!isnull) { + seconds = + ((int64_t)DatumGetDateADT(val) + CH_TO_PG_DATE_OFFSET) * SECS_PER_DAY; + } + ch_binary_append_date_seconds(h, colidx, seconds, isnull); + return; + } + case TIMESTAMPOID: + case TIMESTAMPTZOID: { + switch (kind) { + case CHC_DATETIME: { + int64_t seconds = + isnull ? 0 : (int64_t)timestamptz_to_time_t(DatumGetTimestamp(val)); + + ch_binary_append_datetime_seconds(h, colidx, seconds, isnull); + } break; + case CHC_DATETIME64: { + int64_t raw = 0; + + if (!isnull) { + uint32_t prec = ch_binary_column_datetime64_precision(h, colidx); + Timestamp t = DatumGetTimestamp(val); + int64 power = pow10i[prec]; + int64 secs = t / USECS_PER_SEC; + int64 us_rem = t % USECS_PER_SEC; + + /* + * floor-divide; C trunc-to-zero leaves + * negative remainder + */ + if (us_rem < 0) { + secs -= 1; + us_rem += USECS_PER_SEC; + } + secs += CH_TO_PG_DATE_OFFSET * SECS_PER_DAY; + raw = secs * power + us_rem * power / USECS_PER_SEC; + } + ch_binary_append_datetime64_raw(h, colidx, raw, isnull); + } break; + default: + goto type_mismatch; + } + return; + } + case ANYARRAYOID: { + ch_binary_array_t* arr; + chc_kind item_kind; + Oid child_valtype; + + if (kind != CHC_ARRAY) { + goto type_mismatch; + } + + arr = (ch_binary_array_t*)DatumGetPointer(val); + ch_binary_array_begin(h, colidx); + + /* + * While array_begin is active ch_binary_column_kind targets + * the inner element kind. For nested arrays the children are + * themselves ch_binary_array_t* so recurse with ANYARRAYOID; + * at the leaf level use the scalar item_type. + */ + item_kind = ch_binary_column_kind(h, colidx); + child_valtype = (arr->ndim > 1) ? ANYARRAYOID : arr->item_type; + for (size_t i = 0; i < arr->len; i++) { + append_one(h, 0, item_kind, arr->datums[i], child_valtype, arr->nulls[i]); + } + + ch_binary_array_end(h); + return; + } + case UUIDOID: { + uint8_t bytes[16]; + + if (kind != CHC_UUID) { + goto type_mismatch; + } + if (!isnull) { + memcpy(bytes, DatumGetUUIDP(val)->data, 16); + } else { + memset(bytes, 0, 16); + } + ch_binary_append_uuid(h, colidx, bytes, isnull); + return; + } + case INETOID: { + const uint8_t* addr = NULL; + size_t addrlen = 0; + + if (kind != CHC_IPV4 && kind != CHC_IPV6) { + goto type_mismatch; + } + if (!isnull) { + inet* ipa = DatumGetInetPP(val); + int fam = ip_family(ipa); + int expected = (kind == CHC_IPV4) ? PGSQL_AF_INET : PGSQL_AF_INET6; + + if (fam != expected) { + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg( + "pg_clickhouse: inet family mismatch for column %zu", colidx + )) + ); + } + addr = ip_addr(ipa); + addrlen = ip_addrsize(ipa); + } else { + addrlen = (kind == CHC_IPV4) ? 4 : 16; + } + ch_binary_append_inet(h, colidx, addr, addrlen, isnull); + return; + } + case JSONOID: + case JSONBOID: { + char* s = NULL; + size_t len = 0; + + if (kind != CHC_JSON && kind != CHC_OBJECT) { + goto type_mismatch; + } + if (!isnull) { + s = DatumGetCString( + DirectFunctionCall1(valtype == JSONBOID ? jsonb_out : json_out, val) + ); + len = strlen(s); + } + ch_binary_append_bytes(h, colidx, s, len, isnull); + if (s) { + pfree(s); + } + return; + } + default: + goto type_mismatch; + } type_mismatch: - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: unexpected PG/CH type pair for column %zu", colidx))); + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: unexpected PG/CH type pair for column %zu", colidx)) + ); } void -ch_binary_column_append_data(ch_binary_insert_state * state, size_t colidx) -{ - Datum val = state->values[colidx]; - Oid valtype = TupleDescAttr(state->outdesc, colidx)->atttypid; - bool isnull = state->nulls[colidx]; - chc_kind kind = ch_binary_column_kind(state->insert_block, colidx); - - append_one(state->insert_block, colidx, kind, val, valtype, isnull); +ch_binary_column_append_data(ch_binary_insert_state* state, size_t colidx) { + Datum val = state->values[colidx]; + Oid valtype = TupleDescAttr(state->outdesc, colidx)->atttypid; + bool isnull = state->nulls[colidx]; + chc_kind kind = ch_binary_column_kind(state->insert_block, colidx); + + append_one(state->insert_block, colidx, kind, val, valtype, isnull); } void -ch_binary_insert_columns(ch_binary_insert_state * state) -{ - ch_binary_flush_block(state->insert_block); +ch_binary_insert_columns(ch_binary_insert_state* state) { + ch_binary_flush_block(state->insert_block); } void -ch_binary_insert_state_free(void *c) -{ - ch_binary_insert_state *state = (ch_binary_insert_state *) c; - - if (state->insert_block == NULL) - return; - - /* - * Reset-callback context: cannot ereport. Finalize runs from the FDW - * happy path before we get here; if it did not (mid-query abort), the - * release call flags the connection broken so the next query rebuilds. - */ - ch_binary_release_insert(state->insert_block); - state->insert_block = NULL; +ch_binary_insert_state_free(void* c) { + ch_binary_insert_state* state = (ch_binary_insert_state*)c; + + if (state->insert_block == NULL) { + return; + } + + /* + * Reset-callback context: cannot ereport. Finalize runs from the FDW + * happy path before we get here; if it did not (mid-query abort), the + * release call flags the connection broken so the next query rebuilds. + */ + ch_binary_release_insert(state->insert_block); + state->insert_block = NULL; } diff --git a/src/binary/insert.c b/src/binary/insert.c index 932cf85..0ecc60c 100644 --- a/src/binary/insert.c +++ b/src/binary/insert.c @@ -22,79 +22,75 @@ #include "binary_internal.h" /* dynamic buffer for raw bytes; palloc-backed, freed by context delete */ -typedef struct dynbuf -{ - uint8_t *data; - size_t len; - size_t cap; -} dynbuf; +typedef struct dynbuf { + uint8_t* data; + size_t len; + size_t cap; +} dynbuf; static void -dynbuf_reserve(dynbuf * b, size_t need) -{ - if (need <= b->cap) - return; - size_t ncap = b->cap ? b->cap : 64; - - while (ncap < need) - ncap *= 2; - b->data = b->data - ? repalloc_huge(b->data, ncap) - : MemoryContextAllocExtended(CurrentMemoryContext, ncap, MCXT_ALLOC_HUGE); - b->cap = ncap; +dynbuf_reserve(dynbuf* b, size_t need) { + if (need <= b->cap) { + return; + } + size_t ncap = b->cap ? b->cap : 64; + + while (ncap < need) { + ncap *= 2; + } + b->data = + b->data + ? repalloc_huge(b->data, ncap) + : MemoryContextAllocExtended(CurrentMemoryContext, ncap, MCXT_ALLOC_HUGE); + b->cap = ncap; } static void -dynbuf_append(dynbuf * b, const void *src, size_t n) -{ - dynbuf_reserve(b, b->len + n); - if (src && n) - memcpy(b->data + b->len, src, n); - b->len += n; +dynbuf_append(dynbuf* b, const void* src, size_t n) { + dynbuf_reserve(b, b->len + n); + if (src && n) { + memcpy(b->data + b->len, src, n); + } + b->len += n; } static void -dynbuf_append_zero(dynbuf * b, size_t n) -{ - dynbuf_reserve(b, b->len + n); - memset(b->data + b->len, 0, n); - b->len += n; +dynbuf_append_zero(dynbuf* b, size_t n) { + dynbuf_reserve(b, b->len + n); + memset(b->data + b->len, 0, n); + b->len += n; } static void -dynbuf_reset(dynbuf * b) -{ - b->len = 0; +dynbuf_reset(dynbuf* b) { + b->len = 0; } /* dynamic array of u64 */ -typedef struct u64buf -{ - uint64_t *data; - size_t len; - size_t cap; -} u64buf; +typedef struct u64buf { + uint64_t* data; + size_t len; + size_t cap; +} u64buf; static void -u64buf_push(u64buf * b, uint64_t v) -{ - if (b->len + 1 > b->cap) - { - size_t ncap = b->cap ? b->cap * 2 : 16; - size_t bytes = ncap * sizeof(uint64_t); - - b->data = b->data - ? repalloc_huge(b->data, bytes) - : MemoryContextAllocExtended(CurrentMemoryContext, bytes, MCXT_ALLOC_HUGE); - b->cap = ncap; - } - b->data[b->len++] = v; +u64buf_push(u64buf* b, uint64_t v) { + if (b->len + 1 > b->cap) { + size_t ncap = b->cap ? b->cap * 2 : 16; + size_t bytes = ncap * sizeof(uint64_t); + + b->data = b->data ? repalloc_huge(b->data, bytes) + : MemoryContextAllocExtended( + CurrentMemoryContext, bytes, MCXT_ALLOC_HUGE + ); + b->cap = ncap; + } + b->data[b->len++] = v; } static void -u64buf_reset(u64buf * b) -{ - b->len = 0; +u64buf_reset(u64buf* b) { + b->len = 0; } /* @@ -105,233 +101,231 @@ u64buf_reset(u64buf * b) * - arr_offs: cumulative outer ends for Array(*) columns. * - nulls: one byte per top-level row for Nullable(*). */ -typedef enum -{ - IC_FIXED, - IC_STRING, - IC_LC_STRING, - IC_ARRAY_FIXED, - IC_ARRAY_STRING, - IC_ARRAY_NESTED_FIXED, - IC_ARRAY_NESTED_STRING, - IC_JSON_STRING, -} ic_layout; +typedef enum { + IC_FIXED, + IC_STRING, + IC_LC_STRING, + IC_ARRAY_FIXED, + IC_ARRAY_STRING, + IC_ARRAY_NESTED_FIXED, + IC_ARRAY_NESTED_STRING, + IC_JSON_STRING, +} ic_layout; static inline bool -ic_layout_array_fixed(ic_layout l) -{ - return l == IC_ARRAY_FIXED || l == IC_ARRAY_NESTED_FIXED; +ic_layout_array_fixed(ic_layout l) { + return l == IC_ARRAY_FIXED || l == IC_ARRAY_NESTED_FIXED; } static inline bool -ic_layout_array_string(ic_layout l) -{ - return l == IC_ARRAY_STRING || l == IC_ARRAY_NESTED_STRING; +ic_layout_array_string(ic_layout l) { + return l == IC_ARRAY_STRING || l == IC_ARRAY_NESTED_STRING; } -typedef struct ic_col -{ - const chc_type *t; /* full column type (incl. Nullable wrapper) */ - const chc_type *inner_t; /* possibly unwrapped Nullable */ - const chc_type *elem_t; /* Array element type, with Nullable - * unwrapped */ - ic_layout layout; - bool is_nullable; - int array_depth; /* current open begin/end nesting */ - int ndim; /* >=1 for array layouts; 1 for top-level - * Array */ - bool array_inner_is_string; - size_t elem_size; /* fixed elem size (top-level FIXED or array - * element FIXED) */ - uint32_t dt64_precision; - dynbuf body; /* row-aligned for FIXED, byte-flat for - * STRING/LC */ - u64buf body_offs; /* offsets per row for STRING/LC inner */ - dynbuf nulls; /* per top-level row */ - u64buf arr_offs; /* cumulative ends for outermost array layer - * (or only layer when ndim==1) */ - u64buf *arr_offs_inner; /* for ndim>1: ndim-1 inner-layer offsets, - * [0]=just inside outer, [ndim-2]=adjacent to - * leaves */ - size_t n_rows; /* committed top-level rows */ - - /* - * Cached column info exposed to callers. info.type borrows from the - * initial_block's chc_type tree, unwrapping Nullable + outer LC. - */ - ch_binary_column_info info; -} ic_col; - -struct ch_binary_insert_handle -{ - MemoryContext cxt; - chc_client *client; - struct ch_binary_state *state; /* parent connection; used to flag broken - * state on error */ - chc_block *initial_block; /* schema source (server's empty Data) */ - size_t ncols; - ic_col *cols; - bool array_active; - size_t array_col_idx; - bool started; - bool finalized; /* finalize_insert has run (success or raised) */ +typedef struct ic_col { + const chc_type* t; /* full column type (incl. Nullable wrapper) */ + const chc_type* inner_t; /* possibly unwrapped Nullable */ + const chc_type* elem_t; /* Array element type, with Nullable + * unwrapped */ + ic_layout layout; + bool is_nullable; + int array_depth; /* current open begin/end nesting */ + int ndim; /* >=1 for array layouts; 1 for top-level + * Array */ + bool array_inner_is_string; + size_t elem_size; /* fixed elem size (top-level FIXED or array + * element FIXED) */ + uint32_t dt64_precision; + dynbuf body; /* row-aligned for FIXED, byte-flat for + * STRING/LC */ + u64buf body_offs; /* offsets per row for STRING/LC inner */ + dynbuf nulls; /* per top-level row */ + u64buf arr_offs; /* cumulative ends for outermost array layer + * (or only layer when ndim==1) */ + u64buf* arr_offs_inner; /* for ndim>1: ndim-1 inner-layer offsets, + * [0]=just inside outer, [ndim-2]=adjacent to + * leaves */ + size_t n_rows; /* committed top-level rows */ + + /* + * Cached column info exposed to callers. info.type borrows from the + * initial_block's chc_type tree, unwrapping Nullable + outer LC. + */ + ch_binary_column_info info; +} ic_col; + +struct ch_binary_insert_handle { + MemoryContext cxt; + chc_client* client; + struct ch_binary_state* state; /* parent connection; used to flag broken + * state on error */ + chc_block* initial_block; /* schema source (server's empty Data) */ + size_t ncols; + ic_col* cols; + bool array_active; + size_t array_col_idx; + bool started; + bool finalized; /* finalize_insert has run (success or raised) */ }; static void -classify_column(ic_col * ic, const chc_type * t) -{ - ic->t = t; - if (chc_type_kind(t) == CHC_NULLABLE) - { - ic->is_nullable = true; - t = chc_type_child(t, 0); - } - ic->inner_t = t; - - chc_kind k = chc_type_kind(t); - - if (k == CHC_LOW_CARDINALITY) - { - const chc_type *inner = chc_type_child(t, 0); - bool inner_nullable = chc_type_kind(inner) == CHC_NULLABLE; - const chc_type *base = inner_nullable ? chc_type_child(inner, 0) : inner; - - if (chc_type_kind(base) != CHC_STRING) - { - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported LowCardinality variant: %s", - chc_type_name(base, NULL)))); - } - - /* - * Nullable lives inside LowCardinality, not as an outer wrapper, so - * record it here: resolve_col tracks the per-row null bits - * build_lc_dict reads to map nulls onto dict slot 0. - */ - ic->is_nullable = inner_nullable; - ic->layout = IC_LC_STRING; - ic->elem_t = base; - return; - } - if (k == CHC_ARRAY) - { - /* - * Walk through nested Array layers to the leaf element type so - * Array(Array(...)) maps to a single ic_col with ndim levels. - */ - const chc_type *base = t; - int ndim = 0; - - while (chc_type_kind(base) == CHC_ARRAY) - { - ndim++; - base = chc_type_child(base, 0); - } - bool elem_nullable = chc_type_kind(base) == CHC_NULLABLE; - - /* Array(Nullable(T)) not supported yet. */ - if (elem_nullable) - { - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: %s not currently supported", - chc_type_name(base, NULL)))); - } - chc_kind ek = chc_type_kind(base); - - ic->elem_t = base; - ic->ndim = ndim; - if (ek == CHC_STRING || ek == CHC_FIXED_STRING) - { - ic->layout = ndim == 1 ? IC_ARRAY_STRING : IC_ARRAY_NESTED_STRING; - ic->array_inner_is_string = true; - } - else if (chc_type_elem_size(base) > 0) - { - ic->layout = ndim == 1 ? IC_ARRAY_FIXED : IC_ARRAY_NESTED_FIXED; - ic->elem_size = chc_type_elem_size(base); - } - else - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported Array element type: %s", chc_type_name(base, NULL)))); - if (ndim > 1) - ic->arr_offs_inner = palloc0((size_t) (ndim - 1) * sizeof(u64buf)); - return; - } - if (k == CHC_STRING) - { - ic->layout = IC_STRING; - return; - } - if (k == CHC_JSON) - { - ic->layout = IC_JSON_STRING; - return; - } - size_t es = chc_type_elem_size(t); - - if (es > 0) - { - ic->layout = IC_FIXED; - ic->elem_size = es; - if (k == CHC_DATETIME64) - ic->dt64_precision = (uint32_t) chc_type_datetime64_scale(t); - return; - } - if (k == CHC_FIXED_STRING) - { - ic->layout = IC_FIXED; - ic->elem_size = (size_t) chc_type_fixed_size(t); - return; - } - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: could not prepare insert - unsupported column type: %s", - chc_type_name(t, NULL)))); +classify_column(ic_col* ic, const chc_type* t) { + ic->t = t; + if (chc_type_kind(t) == CHC_NULLABLE) { + ic->is_nullable = true; + t = chc_type_child(t, 0); + } + ic->inner_t = t; + + chc_kind k = chc_type_kind(t); + + if (k == CHC_LOW_CARDINALITY) { + const chc_type* inner = chc_type_child(t, 0); + bool inner_nullable = chc_type_kind(inner) == CHC_NULLABLE; + const chc_type* base = inner_nullable ? chc_type_child(inner, 0) : inner; + + if (chc_type_kind(base) != CHC_STRING) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: unsupported LowCardinality variant: %s", + chc_type_name(base, NULL) + )) + ); + } + + /* + * Nullable lives inside LowCardinality, not as an outer wrapper, so + * record it here: resolve_col tracks the per-row null bits + * build_lc_dict reads to map nulls onto dict slot 0. + */ + ic->is_nullable = inner_nullable; + ic->layout = IC_LC_STRING; + ic->elem_t = base; + return; + } + if (k == CHC_ARRAY) { + /* + * Walk through nested Array layers to the leaf element type so + * Array(Array(...)) maps to a single ic_col with ndim levels. + */ + const chc_type* base = t; + int ndim = 0; + + while (chc_type_kind(base) == CHC_ARRAY) { + ndim++; + base = chc_type_child(base, 0); + } + bool elem_nullable = chc_type_kind(base) == CHC_NULLABLE; + + /* Array(Nullable(T)) not supported yet. */ + if (elem_nullable) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: %s not currently supported", + chc_type_name(base, NULL) + )) + ); + } + chc_kind ek = chc_type_kind(base); + + ic->elem_t = base; + ic->ndim = ndim; + if (ek == CHC_STRING || ek == CHC_FIXED_STRING) { + ic->layout = ndim == 1 ? IC_ARRAY_STRING : IC_ARRAY_NESTED_STRING; + ic->array_inner_is_string = true; + } else if (chc_type_elem_size(base) > 0) { + ic->layout = ndim == 1 ? IC_ARRAY_FIXED : IC_ARRAY_NESTED_FIXED; + ic->elem_size = chc_type_elem_size(base); + } else { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: unsupported Array element type: %s", + chc_type_name(base, NULL) + )) + ); + } + if (ndim > 1) { + ic->arr_offs_inner = palloc0((size_t)(ndim - 1) * sizeof(u64buf)); + } + return; + } + if (k == CHC_STRING) { + ic->layout = IC_STRING; + return; + } + if (k == CHC_JSON) { + ic->layout = IC_JSON_STRING; + return; + } + size_t es = chc_type_elem_size(t); + + if (es > 0) { + ic->layout = IC_FIXED; + ic->elem_size = es; + if (k == CHC_DATETIME64) { + ic->dt64_precision = (uint32_t)chc_type_datetime64_scale(t); + } + return; + } + if (k == CHC_FIXED_STRING) { + ic->layout = IC_FIXED; + ic->elem_size = (size_t)chc_type_fixed_size(t); + return; + } + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: could not prepare insert - unsupported column type: %s", + chc_type_name(t, NULL) + )) + ); } static void -recv_initial_block(struct ch_binary_state *s, ch_binary_insert_handle * h) -{ - for (;;) - { - chc_packet pkt = {}; - chc_err err = {}; - int rc = chc_client_recv_packet(s->client, &pkt, &err); - - if (rc != CHC_OK) - { - s->broken = true; - raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); - } - if (pkt.kind == CHC_PKT_EXCEPTION) - { - const char *msg = "server exception"; - - if (pkt.exception && pkt.exception->display_text) - msg = pkt.exception->display_text; - else if (pkt.exception && pkt.exception->name) - msg = pkt.exception->name; - char *msg_copy = pstrdup(msg); - - s->broken = true; - chc_packet_clear(s->client, &pkt); - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: could not prepare insert - %s", msg_copy))); - } - if (pkt.kind == CHC_PKT_DATA && pkt.block && - chc_block_n_columns(pkt.block) > 0) - { - h->initial_block = pkt.block; - pkt.block = NULL; - chc_packet_clear(s->client, &pkt); - return; - } - chc_packet_clear(s->client, &pkt); - } +recv_initial_block(struct ch_binary_state* s, ch_binary_insert_handle* h) { + for (;;) { + chc_packet pkt = {}; + chc_err err = {}; + int rc = chc_client_recv_packet(s->client, &pkt, &err); + + if (rc != CHC_OK) { + s->broken = true; + raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); + } + if (pkt.kind == CHC_PKT_EXCEPTION) { + const char* msg = "server exception"; + + if (pkt.exception && pkt.exception->display_text) { + msg = pkt.exception->display_text; + } else if (pkt.exception && pkt.exception->name) { + msg = pkt.exception->name; + } + char* msg_copy = pstrdup(msg); + + s->broken = true; + chc_packet_clear(s->client, &pkt); + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: could not prepare insert - %s", msg_copy)) + ); + } + if (pkt.kind == CHC_PKT_DATA && pkt.block && + chc_block_n_columns(pkt.block) > 0) { + h->initial_block = pkt.block; + pkt.block = NULL; + chc_packet_clear(s->client, &pkt); + return; + } + chc_packet_clear(s->client, &pkt); + } } /* @@ -339,159 +333,156 @@ recv_initial_block(struct ch_binary_state *s, ch_binary_insert_handle * h) * Data; send empty Data + drain so the connection stays usable. */ static void -drain_aborted_insert(struct ch_binary_state *s) -{ - chc_err ce = {}; - - (void) chc_client_send_data(s->client, NULL, &ce); - for (;;) - { - chc_packet drain = {}; - - ce = (chc_err) - { - 0 - }; - int drc = chc_client_recv_packet(s->client, &drain, &ce); - bool eos = (drc == CHC_OK && - (drain.kind == CHC_PKT_END_OF_STREAM || - drain.kind == CHC_PKT_EXCEPTION)); - - chc_packet_clear(s->client, &drain); - if (drc != CHC_OK || eos) - break; - } +drain_aborted_insert(struct ch_binary_state* s) { + chc_err ce = {}; + + (void)chc_client_send_data(s->client, NULL, &ce); + for (;;) { + chc_packet drain = {}; + + ce = (chc_err){ 0 }; + int drc = chc_client_recv_packet(s->client, &drain, &ce); + bool eos = + (drc == CHC_OK && + (drain.kind == CHC_PKT_END_OF_STREAM || drain.kind == CHC_PKT_EXCEPTION)); + + chc_packet_clear(s->client, &drain); + if (drc != CHC_OK || eos) { + break; + } + } } -ch_binary_insert_handle * -ch_binary_begin_insert(ch_binary_connection_t * conn, const ch_query * query, - ch_binary_column_info * *out_cols, size_t * out_n) -{ - struct ch_binary_state *s = conn_state(conn); - - /* - * Parent h's cxt to the connection's cxt, not CurrentMemoryContext. The - * caller registers a reset callback on a sibling context that drains the - * insert via end_insert(h); if h lived under that sibling, MemoryContext - * tree teardown would free h before the callback fired. - */ - MemoryContext cxt = AllocSetContextCreate(s->cxt, - "pg_clickhouse binary insert", - ALLOCSET_DEFAULT_SIZES); - MemoryContext old = MemoryContextSwitchTo(cxt); - ch_binary_insert_handle *h; - volatile bool need_drain = false; - - PG_TRY(); - { - h = palloc0(sizeof(*h)); - h->cxt = cxt; - h->client = s->client; - h->state = s; - - /* Append " VALUES" so server enters insert mode. */ - size_t sql_len = strlen(query->sql); - char *sql = palloc(sql_len + 8); - - memcpy(sql, query->sql, sql_len); - memcpy(sql + sql_len, " VALUES", 7); - sql[sql_len + 7] = '\0'; - - /* - * On servers that support it (24.10+), tell server to serialize any - * JSON columns using STRING wire format. INSERT path doesn't need - * this, server reads the per-column version prefix the builder - * writes, but we set it on the same packet for symmetry with the - * SELECT path and so any RETURNING-style projection on top still - * decodes. - */ - chc_query_setting json_setting = { - .name = "output_format_native_write_json_as_string", - .value = "1", - .important = true, - }; - chc_query_opts insert_opts = {}; - const chc_query_opts *opts_ptr = NULL; - - if (server_supports_json_as_string(s->client)) - { - insert_opts.settings = &json_setting; - insert_opts.n_settings = 1; - opts_ptr = &insert_opts; - } - - chc_err err = {}; - int rc = chc_client_send_query_ex(s->client, sql, sql_len + 7, opts_ptr, &err); - - if (rc != CHC_OK) - { - s->broken = true; - raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); - } - - recv_initial_block(s, h); - - /* - * Server is now waiting our Data; failures past this point need an - * empty-Data + drain so the connection stays usable. - */ - need_drain = true; - h->started = true; - - size_t nc = chc_block_n_columns(h->initial_block); - - h->ncols = nc; - h->cols = palloc0(nc * sizeof(ic_col)); - for (size_t i = 0; i < nc; i++) - { - ic_col *c = &h->cols[i]; - const chc_type *ct = chc_block_column_type(h->initial_block, i); - size_t nlen; - const char *nm; - - classify_column(c, ct); - nm = chc_block_column_name(h->initial_block, i, &nlen); - c->info.name = pnstrdup(nm ? nm : "", nlen); - c->info.is_nullable = c->is_nullable; - - /* - * inner_t already unwrapped Nullable; unwrap LowCardinality and - * perhaps its inner Nullable to expose innermost type. - */ - const chc_type *vt = c->inner_t; - - if (chc_type_kind(vt) == CHC_LOW_CARDINALITY) - { - vt = chc_type_child(vt, 0); - if (chc_type_kind(vt) == CHC_NULLABLE) - vt = chc_type_child(vt, 0); - } - c->info.type = vt; - } - - *out_cols = NULL; - if (nc) - { - ch_binary_column_info *arr = palloc(nc * sizeof(*arr)); - - for (size_t i = 0; i < nc; i++) - arr[i] = h->cols[i].info; - *out_cols = arr; - } - *out_n = nc; - } - PG_CATCH(); - { - if (need_drain) - drain_aborted_insert(s); - MemoryContextSwitchTo(old); - MemoryContextDelete(cxt); - PG_RE_THROW(); - } - PG_END_TRY(); - - MemoryContextSwitchTo(old); - return h; +ch_binary_insert_handle* +ch_binary_begin_insert( + ch_binary_connection_t* conn, + const ch_query* query, + ch_binary_column_info** out_cols, + size_t* out_n +) { + struct ch_binary_state* s = conn_state(conn); + + /* + * Parent h's cxt to the connection's cxt, not CurrentMemoryContext. The + * caller registers a reset callback on a sibling context that drains the + * insert via end_insert(h); if h lived under that sibling, MemoryContext + * tree teardown would free h before the callback fired. + */ + MemoryContext cxt = AllocSetContextCreate( + s->cxt, "pg_clickhouse binary insert", ALLOCSET_DEFAULT_SIZES + ); + MemoryContext old = MemoryContextSwitchTo(cxt); + ch_binary_insert_handle* h; + volatile bool need_drain = false; + + PG_TRY(); + { + h = palloc0(sizeof(*h)); + h->cxt = cxt; + h->client = s->client; + h->state = s; + + /* Append " VALUES" so server enters insert mode. */ + size_t sql_len = strlen(query->sql); + char* sql = palloc(sql_len + 8); + + memcpy(sql, query->sql, sql_len); + memcpy(sql + sql_len, " VALUES", 7); + sql[sql_len + 7] = '\0'; + + /* + * On servers that support it (24.10+), tell server to serialize any + * JSON columns using STRING wire format. INSERT path doesn't need + * this, server reads the per-column version prefix the builder + * writes, but we set it on the same packet for symmetry with the + * SELECT path and so any RETURNING-style projection on top still + * decodes. + */ + chc_query_setting json_setting = { + .name = "output_format_native_write_json_as_string", + .value = "1", + .important = true, + }; + chc_query_opts insert_opts = {}; + const chc_query_opts* opts_ptr = NULL; + + if (server_supports_json_as_string(s->client)) { + insert_opts.settings = &json_setting; + insert_opts.n_settings = 1; + opts_ptr = &insert_opts; + } + + chc_err err = {}; + int rc = chc_client_send_query_ex(s->client, sql, sql_len + 7, opts_ptr, &err); + + if (rc != CHC_OK) { + s->broken = true; + raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); + } + + recv_initial_block(s, h); + + /* + * Server is now waiting our Data; failures past this point need an + * empty-Data + drain so the connection stays usable. + */ + need_drain = true; + h->started = true; + + size_t nc = chc_block_n_columns(h->initial_block); + + h->ncols = nc; + h->cols = palloc0(nc * sizeof(ic_col)); + for (size_t i = 0; i < nc; i++) { + ic_col* c = &h->cols[i]; + const chc_type* ct = chc_block_column_type(h->initial_block, i); + size_t nlen; + const char* nm; + + classify_column(c, ct); + nm = chc_block_column_name(h->initial_block, i, &nlen); + c->info.name = pnstrdup(nm ? nm : "", nlen); + c->info.is_nullable = c->is_nullable; + + /* + * inner_t already unwrapped Nullable; unwrap LowCardinality and + * perhaps its inner Nullable to expose innermost type. + */ + const chc_type* vt = c->inner_t; + + if (chc_type_kind(vt) == CHC_LOW_CARDINALITY) { + vt = chc_type_child(vt, 0); + if (chc_type_kind(vt) == CHC_NULLABLE) { + vt = chc_type_child(vt, 0); + } + } + c->info.type = vt; + } + + *out_cols = NULL; + if (nc) { + ch_binary_column_info* arr = palloc(nc * sizeof(*arr)); + + for (size_t i = 0; i < nc; i++) { + arr[i] = h->cols[i].info; + } + *out_cols = arr; + } + *out_n = nc; + } + PG_CATCH(); + { + if (need_drain) { + drain_aborted_insert(s); + } + MemoryContextSwitchTo(old); + MemoryContextDelete(cxt); + PG_RE_THROW(); + } + PG_END_TRY(); + + MemoryContextSwitchTo(old); + return h; } /* @@ -500,48 +491,48 @@ ch_binary_begin_insert(ch_binary_connection_t * conn, const ch_query * query, * the target. For Nullable wrappers, records the null bit. ereports on * NULL-into-NOT-NULL. */ -static ic_col * -resolve_col(ch_binary_insert_handle * h, size_t col_idx, bool isnull) -{ - ic_col *c = h->array_active ? &h->cols[h->array_col_idx] : &h->cols[col_idx]; - - if (isnull && !h->array_active && !c->is_nullable) - { - size_t tnlen; - const char *tname = chc_type_name(c->t, &tnlen); - - ereport(ERROR, - (errcode(ERRCODE_NOT_NULL_VIOLATION), - errmsg("pg_clickhouse: cannot append NULL to NOT NULL %.*s column", - (int) tnlen, tname ? tname : "?"))); - } - if (!h->array_active && c->is_nullable) - { - uint8_t b = isnull ? 1 : 0; - - dynbuf_append(&c->nulls, &b, 1); - } - return c; +static ic_col* +resolve_col(ch_binary_insert_handle* h, size_t col_idx, bool isnull) { + ic_col* c = h->array_active ? &h->cols[h->array_col_idx] : &h->cols[col_idx]; + + if (isnull && !h->array_active && !c->is_nullable) { + size_t tnlen; + const char* tname = chc_type_name(c->t, &tnlen); + + ereport( + ERROR, + (errcode(ERRCODE_NOT_NULL_VIOLATION), + errmsg( + "pg_clickhouse: cannot append NULL to NOT NULL %.*s column", + (int)tnlen, + tname ? tname : "?" + )) + ); + } + if (!h->array_active && c->is_nullable) { + uint8_t b = isnull ? 1 : 0; + + dynbuf_append(&c->nulls, &b, 1); + } + return c; } static void -append_fixed_bytes(ic_col * c, const void *p, size_t n) -{ - dynbuf_append(&c->body, p, n); +append_fixed_bytes(ic_col* c, const void* p, size_t n) { + dynbuf_append(&c->body, p, n); } static void -append_fixed_zero(ic_col * c, size_t n) -{ - dynbuf_append_zero(&c->body, n); +append_fixed_zero(ic_col* c, size_t n) { + dynbuf_append_zero(&c->body, n); } static void -append_string_row(ic_col * c, const void *p, size_t n) -{ - if (n) - dynbuf_append(&c->body, p, n); - u64buf_push(&c->body_offs, c->body.len); +append_string_row(ic_col* c, const void* p, size_t n) { + if (n) { + dynbuf_append(&c->body, p, n); + } + u64buf_push(&c->body_offs, c->body.len); } /* @@ -550,338 +541,351 @@ append_string_row(ic_col * c, const void *p, size_t n) * Decimal32/64/128/256), with `scale` fractional digits folded into the value. */ static void -decimal_text_to_bytes(const char *s, uint32_t scale, size_t width, uint8_t * out) -{ - const char *input = s; - bool neg = false; - - if (!s) - ereport(ERROR, - (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), - errmsg("pg_clickhouse: decimal parse failure"))); - if (*s == '-') - { - neg = true; - s++; - } - else if (*s == '+') - s++; - - /* find offsets, going to iterate digits, skipping non-digits */ - const char *dot = strchr(s, '.'); - size_t ilen = dot ? (size_t) (dot - s) : strlen(s); - const char *frac = dot ? dot + 1 : ""; - size_t flen = strlen(frac); - size_t ndig = ilen + scale; - - uint32_t mag[8] = {}; - size_t nwords = width / 4; - - /* accumulate digits (padded/truncated to scale) into mag */ - for (size_t i = 0; i < ndig; i++) - { - char c = i < ilen ? s[i] - : i - ilen < flen ? frac[i - ilen] - : '0'; - - /* reject NaN / Infinity from numeric_out */ - if (c < '0' || c > '9') - ereport(ERROR, - (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), - errmsg("pg_clickhouse: cannot encode \"%s\" as ClickHouse Decimal", - input))); - uint64_t carry = (uint64_t) (c - '0'); - - for (size_t b = 0; b < nwords; b++) - { - uint64_t v = (uint64_t) mag[b] * 10 + carry; - - mag[b] = (uint32_t) v; - carry = v >> 32; - } - } - /* two's-complement negation */ - if (neg) - { - for (size_t b = 0; b < nwords; b++) - mag[b] = ~mag[b]; - uint64_t carry = 1; - - for (size_t b = 0; b < nwords && carry; b++) - { - uint64_t v = (uint64_t) mag[b] + carry; - - mag[b] = (uint32_t) v; - carry = v >> 32; - } - } - - memcpy(out, mag, width); +decimal_text_to_bytes(const char* s, uint32_t scale, size_t width, uint8_t* out) { + const char* input = s; + bool neg = false; + + if (!s) { + ereport( + ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("pg_clickhouse: decimal parse failure")) + ); + } + if (*s == '-') { + neg = true; + s++; + } else if (*s == '+') { + s++; + } + + /* find offsets, going to iterate digits, skipping non-digits */ + const char* dot = strchr(s, '.'); + size_t ilen = dot ? (size_t)(dot - s) : strlen(s); + const char* frac = dot ? dot + 1 : ""; + size_t flen = strlen(frac); + size_t ndig = ilen + scale; + + uint32_t mag[8] = {}; + size_t nwords = width / 4; + + /* accumulate digits (padded/truncated to scale) into mag */ + for (size_t i = 0; i < ndig; i++) { + char c = i < ilen ? s[i] : i - ilen < flen ? frac[i - ilen] : '0'; + + /* reject NaN / Infinity from numeric_out */ + if (c < '0' || c > '9') { + ereport( + ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg( + "pg_clickhouse: cannot encode \"%s\" as ClickHouse Decimal", input + )) + ); + } + uint64_t carry = (uint64_t)(c - '0'); + + for (size_t b = 0; b < nwords; b++) { + uint64_t v = (uint64_t)mag[b] * 10 + carry; + + mag[b] = (uint32_t)v; + carry = v >> 32; + } + } + /* two's-complement negation */ + if (neg) { + for (size_t b = 0; b < nwords; b++) { + mag[b] = ~mag[b]; + } + uint64_t carry = 1; + + for (size_t b = 0; b < nwords && carry; b++) { + uint64_t v = (uint64_t)mag[b] + carry; + + mag[b] = (uint32_t)v; + carry = v >> 32; + } + } + + memcpy(out, mag, width); } static void -append_int_kind(ic_col * c, int64_t val) -{ - chc_kind k = ic_layout_array_fixed(c->layout) - ? chc_type_kind(c->elem_t) - : chc_type_kind(c->inner_t); - - switch (k) - { - case CHC_INT8: - case CHC_UINT8: - case CHC_BOOL: - { - int8_t v = (int8_t) val; - - append_fixed_bytes(c, &v, 1); - return; - } - case CHC_INT16: - case CHC_UINT16: - { - int16_t v = (int16_t) val; - - append_fixed_bytes(c, &v, 2); - return; - } - case CHC_INT32: - case CHC_UINT32: - { - int32_t v = (int32_t) val; - - append_fixed_bytes(c, &v, 4); - return; - } - case CHC_INT64: - case CHC_UINT64: - { - append_fixed_bytes(c, &val, 8); - return; - } - default: - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: int value into non-integer column"))); - } +append_int_kind(ic_col* c, int64_t val) { + chc_kind k = ic_layout_array_fixed(c->layout) ? chc_type_kind(c->elem_t) + : chc_type_kind(c->inner_t); + + switch (k) { + case CHC_INT8: + case CHC_UINT8: + case CHC_BOOL: { + int8_t v = (int8_t)val; + + append_fixed_bytes(c, &v, 1); + return; + } + case CHC_INT16: + case CHC_UINT16: { + int16_t v = (int16_t)val; + + append_fixed_bytes(c, &v, 2); + return; + } + case CHC_INT32: + case CHC_UINT32: { + int32_t v = (int32_t)val; + + append_fixed_bytes(c, &v, 4); + return; + } + case CHC_INT64: + case CHC_UINT64: { + append_fixed_bytes(c, &val, 8); + return; + } + default: + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: int value into non-integer column")) + ); + } } void -ch_binary_append_int(ch_binary_insert_handle * h, size_t col, int64_t val, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - - if (isnull) - append_fixed_zero(c, c->elem_size); - else - append_int_kind(c, val); - MemoryContextSwitchTo(old); +ch_binary_append_int(ch_binary_insert_handle* h, size_t col, int64_t val, bool isnull) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + + if (isnull) { + append_fixed_zero(c, c->elem_size); + } else { + append_int_kind(c, val); + } + MemoryContextSwitchTo(old); } void -ch_binary_append_uint(ch_binary_insert_handle * h, size_t col, uint64_t val, bool isnull) -{ - ch_binary_append_int(h, col, (int64_t) val, isnull); +ch_binary_append_uint( + ch_binary_insert_handle* h, + size_t col, + uint64_t val, + bool isnull +) { + ch_binary_append_int(h, col, (int64_t)val, isnull); } void -ch_binary_append_bool(ch_binary_insert_handle * h, size_t col, bool val, bool isnull) -{ - ch_binary_append_int(h, col, val, isnull); +ch_binary_append_bool(ch_binary_insert_handle* h, size_t col, bool val, bool isnull) { + ch_binary_append_int(h, col, val, isnull); } void -ch_binary_append_double(ch_binary_insert_handle * h, size_t col, double val, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - - if (isnull) - append_fixed_zero(c, 8); - else - append_fixed_bytes(c, &val, 8); - MemoryContextSwitchTo(old); +ch_binary_append_double( + ch_binary_insert_handle* h, + size_t col, + double val, + bool isnull +) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + + if (isnull) { + append_fixed_zero(c, 8); + } else { + append_fixed_bytes(c, &val, 8); + } + MemoryContextSwitchTo(old); } void -ch_binary_append_float(ch_binary_insert_handle * h, size_t col, float val, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - - if (isnull) - append_fixed_zero(c, 4); - else - append_fixed_bytes(c, &val, 4); - MemoryContextSwitchTo(old); +ch_binary_append_float(ch_binary_insert_handle* h, size_t col, float val, bool isnull) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + + if (isnull) { + append_fixed_zero(c, 4); + } else { + append_fixed_bytes(c, &val, 4); + } + MemoryContextSwitchTo(old); } void -ch_binary_append_bytes(ch_binary_insert_handle * h, size_t col, const void *p, - size_t n, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - chc_kind k = ic_layout_array_string(c->layout) - ? chc_type_kind(c->elem_t) - : chc_type_kind(c->inner_t); - - if (c->layout == IC_LC_STRING) - { - /* Accumulate via body+body_offs; dict materialized at flush. */ - if (isnull) - append_string_row(c, NULL, 0); - else - append_string_row(c, p, n); - MemoryContextSwitchTo(old); - return; - } - - if (k == CHC_FIXED_STRING) - { - size_t w = (size_t) chc_type_fixed_size(ic_layout_array_string(c->layout) ? c->elem_t : c->inner_t); - - if (isnull) - append_fixed_zero(c, w); - else - { - size_t take = n < w ? n : w; - - if (take) - dynbuf_append(&c->body, p, take); - if (take < w) - dynbuf_append_zero(&c->body, w - take); - } - MemoryContextSwitchTo(old); - return; - } - if (k == CHC_ENUM8 || k == CHC_ENUM16) - { - size_t ew = (k == CHC_ENUM8) ? 1 : 2; - - if (isnull) - { - append_fixed_zero(c, ew); - MemoryContextSwitchTo(old); - return; - } - const chc_type *et = ic_layout_array_string(c->layout) ? c->elem_t : c->inner_t; - size_t nenum = chc_type_enum_count(et); - int64_t val = 0; - bool found = false; - - for (size_t i = 0; i < nenum; i++) - { - const char *en; - size_t el; - int64_t ev; - - chc_type_enum_at(et, i, &en, &el, &ev); - if (el == n && memcmp(en, p, n) == 0) - { - val = ev; - found = true; - break; - } - } - if (!found) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("pg_clickhouse: enum value '%.*s' not found", - (int) n, (const char *) p))); - if (k == CHC_ENUM8) - { - int8_t v = (int8_t) val; - - append_fixed_bytes(c, &v, 1); - } - else - { - int16_t v = (int16_t) val; - - append_fixed_bytes(c, &v, 2); - } - MemoryContextSwitchTo(old); - return; - } - if (k == CHC_STRING || k == CHC_JSON || k == CHC_OBJECT) - { - if (isnull) - append_string_row(c, NULL, 0); - else - append_string_row(c, p, n); - MemoryContextSwitchTo(old); - return; - } - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: bytes into non-text column"))); +ch_binary_append_bytes( + ch_binary_insert_handle* h, + size_t col, + const void* p, + size_t n, + bool isnull +) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + chc_kind k = ic_layout_array_string(c->layout) ? chc_type_kind(c->elem_t) + : chc_type_kind(c->inner_t); + + if (c->layout == IC_LC_STRING) { + /* Accumulate via body+body_offs; dict materialized at flush. */ + if (isnull) { + append_string_row(c, NULL, 0); + } else { + append_string_row(c, p, n); + } + MemoryContextSwitchTo(old); + return; + } + + if (k == CHC_FIXED_STRING) { + size_t w = (size_t)chc_type_fixed_size( + ic_layout_array_string(c->layout) ? c->elem_t : c->inner_t + ); + + if (isnull) { + append_fixed_zero(c, w); + } else { + size_t take = n < w ? n : w; + + if (take) { + dynbuf_append(&c->body, p, take); + } + if (take < w) { + dynbuf_append_zero(&c->body, w - take); + } + } + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_ENUM8 || k == CHC_ENUM16) { + size_t ew = (k == CHC_ENUM8) ? 1 : 2; + + if (isnull) { + append_fixed_zero(c, ew); + MemoryContextSwitchTo(old); + return; + } + const chc_type* et = ic_layout_array_string(c->layout) ? c->elem_t : c->inner_t; + size_t nenum = chc_type_enum_count(et); + int64_t val = 0; + bool found = false; + + for (size_t i = 0; i < nenum; i++) { + const char* en; + size_t el; + int64_t ev; + + chc_type_enum_at(et, i, &en, &el, &ev); + if (el == n && memcmp(en, p, n) == 0) { + val = ev; + found = true; + break; + } + } + if (!found) { + ereport( + ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg( + "pg_clickhouse: enum value '%.*s' not found", + (int)n, + (const char*)p + )) + ); + } + if (k == CHC_ENUM8) { + int8_t v = (int8_t)val; + + append_fixed_bytes(c, &v, 1); + } else { + int16_t v = (int16_t)val; + + append_fixed_bytes(c, &v, 2); + } + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_STRING || k == CHC_JSON || k == CHC_OBJECT) { + if (isnull) { + append_string_row(c, NULL, 0); + } else { + append_string_row(c, p, n); + } + MemoryContextSwitchTo(old); + return; + } + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: bytes into non-text column")) + ); } void -ch_binary_append_decimal(ch_binary_insert_handle * h, size_t col, - const char *digits, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; - chc_kind k = chc_type_kind(t); - size_t w; - - switch (k) - { - case CHC_DECIMAL32: - w = 4; - break; - case CHC_DECIMAL64: - w = 8; - break; - case CHC_DECIMAL128: - w = 16; - break; - case CHC_DECIMAL256: - w = 32; - break; - default: - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: decimal into non-decimal column"))); - } - uint8_t raw[32] = {}; - - if (!isnull && digits) - { - uint32_t scale = (uint32_t) chc_type_decimal_scale(t); - - decimal_text_to_bytes(digits, scale, w, raw); - } - append_fixed_bytes(c, raw, w); - MemoryContextSwitchTo(old); +ch_binary_append_decimal( + ch_binary_insert_handle* h, + size_t col, + const char* digits, + bool isnull +) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + const chc_type* t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + size_t w; + + switch (k) { + case CHC_DECIMAL32: + w = 4; + break; + case CHC_DECIMAL64: + w = 8; + break; + case CHC_DECIMAL128: + w = 16; + break; + case CHC_DECIMAL256: + w = 32; + break; + default: + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: decimal into non-decimal column")) + ); + } + uint8_t raw[32] = {}; + + if (!isnull && digits) { + uint32_t scale = (uint32_t)chc_type_decimal_scale(t); + + decimal_text_to_bytes(digits, scale, w, raw); + } + append_fixed_bytes(c, raw, w); + MemoryContextSwitchTo(old); } void -ch_binary_append_uuid(ch_binary_insert_handle * h, size_t col, - const uint8_t bytes[16], bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - uint8_t wire[16] = {}; - - if (!isnull) - { - uint64_t a, - b; - - memcpy(&a, bytes, 8); - memcpy(&b, bytes + 8, 8); - a = pg_ntoh64(a); - b = pg_ntoh64(b); - memcpy(wire, &a, 8); - memcpy(wire + 8, &b, 8); - } - append_fixed_bytes(c, wire, 16); - MemoryContextSwitchTo(old); +ch_binary_append_uuid( + ch_binary_insert_handle* h, + size_t col, + const uint8_t bytes[16], + bool isnull +) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + uint8_t wire[16] = {}; + + if (!isnull) { + uint64_t a, b; + + memcpy(&a, bytes, 8); + memcpy(&b, bytes + 8, 8); + a = pg_ntoh64(a); + b = pg_ntoh64(b); + memcpy(wire, &a, 8); + memcpy(wire + 8, &b, 8); + } + append_fixed_bytes(c, wire, 16); + MemoryContextSwitchTo(old); } /* @@ -890,496 +894,549 @@ ch_binary_append_uuid(ch_binary_insert_handle * h, size_t col, * For IPv6 CH wire matches PG byte order. */ void -ch_binary_append_inet(ch_binary_insert_handle * h, size_t col, - const uint8_t * addr_be, size_t addrlen, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; - chc_kind k = chc_type_kind(t); - - if (k == CHC_IPV4 && addrlen == 4) - { - uint32_t addr = 0; - - if (!isnull && addr_be) - { - uint32_t be; - - memcpy(&be, addr_be, 4); - addr = pg_ntoh32(be); - } - append_fixed_bytes(c, &addr, 4); - MemoryContextSwitchTo(old); - return; - } - if (k == CHC_IPV6 && addrlen == 16) - { - uint8_t raw[16] = {}; - - if (!isnull && addr_be) - memcpy(raw, addr_be, 16); - append_fixed_bytes(c, raw, 16); - MemoryContextSwitchTo(old); - return; - } - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: cannot insert inet into non-inet column"))); +ch_binary_append_inet( + ch_binary_insert_handle* h, + size_t col, + const uint8_t* addr_be, + size_t addrlen, + bool isnull +) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + const chc_type* t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + + if (k == CHC_IPV4 && addrlen == 4) { + uint32_t addr = 0; + + if (!isnull && addr_be) { + uint32_t be; + + memcpy(&be, addr_be, 4); + addr = pg_ntoh32(be); + } + append_fixed_bytes(c, &addr, 4); + MemoryContextSwitchTo(old); + return; + } + if (k == CHC_IPV6 && addrlen == 16) { + uint8_t raw[16] = {}; + + if (!isnull && addr_be) { + memcpy(raw, addr_be, 16); + } + append_fixed_bytes(c, raw, 16); + MemoryContextSwitchTo(old); + return; + } + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: cannot insert inet into non-inet column")) + ); } void -ch_binary_append_date_seconds(ch_binary_insert_handle * h, size_t col, - int64_t seconds, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - const chc_type *t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; - chc_kind k = chc_type_kind(t); - - if (k == CHC_DATE) - { - uint16_t days = isnull ? 0 : (uint16_t) (seconds / 86400); - - append_fixed_bytes(c, &days, 2); - } - else if (k == CHC_DATE32) - { - int32_t days = isnull ? 0 : (int32_t) (seconds / 86400); - - append_fixed_bytes(c, &days, 4); - } - else - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: date into non-date column"))); - MemoryContextSwitchTo(old); +ch_binary_append_date_seconds( + ch_binary_insert_handle* h, + size_t col, + int64_t seconds, + bool isnull +) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + const chc_type* t = ic_layout_array_fixed(c->layout) ? c->elem_t : c->inner_t; + chc_kind k = chc_type_kind(t); + + if (k == CHC_DATE) { + uint16_t days = isnull ? 0 : (uint16_t)(seconds / 86400); + + append_fixed_bytes(c, &days, 2); + } else if (k == CHC_DATE32) { + int32_t days = isnull ? 0 : (int32_t)(seconds / 86400); + + append_fixed_bytes(c, &days, 4); + } else { + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("pg_clickhouse: date into non-date column")) + ); + } + MemoryContextSwitchTo(old); } void -ch_binary_append_datetime_seconds(ch_binary_insert_handle * h, size_t col, - int64_t seconds, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - uint32_t v = isnull ? 0 : (uint32_t) seconds; - - append_fixed_bytes(c, &v, 4); - MemoryContextSwitchTo(old); +ch_binary_append_datetime_seconds( + ch_binary_insert_handle* h, + size_t col, + int64_t seconds, + bool isnull +) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + uint32_t v = isnull ? 0 : (uint32_t)seconds; + + append_fixed_bytes(c, &v, 4); + MemoryContextSwitchTo(old); } void -ch_binary_append_datetime64_raw(ch_binary_insert_handle * h, size_t col, - int64_t raw, bool isnull) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - ic_col *c = resolve_col(h, col, isnull); - int64_t v = isnull ? 0 : raw; - - append_fixed_bytes(c, &v, 8); - MemoryContextSwitchTo(old); +ch_binary_append_datetime64_raw( + ch_binary_insert_handle* h, + size_t col, + int64_t raw, + bool isnull +) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + ic_col* c = resolve_col(h, col, isnull); + int64_t v = isnull ? 0 : raw; + + append_fixed_bytes(c, &v, 8); + MemoryContextSwitchTo(old); } void -ch_binary_array_begin(ch_binary_insert_handle * h, size_t col) -{ - /* - * Nested arrays recurse via append_one with col=0, so once an array is - * open the caller's col is meaningless, resolve from array_col_idx. - */ - size_t idx = h->array_active ? h->array_col_idx : col; - - if (idx >= h->ncols) - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: array_begin: col out of range"))); - ic_col *c = &h->cols[idx]; - - if (c->layout != IC_ARRAY_FIXED && c->layout != IC_ARRAY_STRING - && c->layout != IC_ARRAY_NESTED_FIXED && c->layout != IC_ARRAY_NESTED_STRING) - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: array_begin: column is not Array"))); - if (c->array_depth >= c->ndim) - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: array_begin: nesting exceeds column ndim"))); - if (c->array_depth == 0) - { - if (c->is_nullable) - { - uint8_t b = 0; - MemoryContext old = MemoryContextSwitchTo(h->cxt); - - dynbuf_append(&c->nulls, &b, 1); - MemoryContextSwitchTo(old); - } - h->array_active = true; - h->array_col_idx = idx; - } - c->array_depth++; +ch_binary_array_begin(ch_binary_insert_handle* h, size_t col) { + /* + * Nested arrays recurse via append_one with col=0, so once an array is + * open the caller's col is meaningless, resolve from array_col_idx. + */ + size_t idx = h->array_active ? h->array_col_idx : col; + + if (idx >= h->ncols) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: col out of range")) + ); + } + ic_col* c = &h->cols[idx]; + + if (c->layout != IC_ARRAY_FIXED && c->layout != IC_ARRAY_STRING && + c->layout != IC_ARRAY_NESTED_FIXED && c->layout != IC_ARRAY_NESTED_STRING) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: column is not Array")) + ); + } + if (c->array_depth >= c->ndim) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: array_begin: nesting exceeds column ndim")) + ); + } + if (c->array_depth == 0) { + if (c->is_nullable) { + uint8_t b = 0; + MemoryContext old = MemoryContextSwitchTo(h->cxt); + + dynbuf_append(&c->nulls, &b, 1); + MemoryContextSwitchTo(old); + } + h->array_active = true; + h->array_col_idx = idx; + } + c->array_depth++; } void -ch_binary_array_end(ch_binary_insert_handle * h) -{ - if (!h->array_active) - return; - ic_col *c = &h->cols[h->array_col_idx]; - - if (c->array_depth == 0) - return; - - uint64_t end; - - /* - * At innermost depth, count is leaf-element count; at outer levels the - * count is total children written so far at the level below. - */ - if (c->array_depth == c->ndim) - { - if (c->layout == IC_ARRAY_FIXED || c->layout == IC_ARRAY_NESTED_FIXED) - end = c->elem_size ? (uint64_t) (c->body.len / c->elem_size) : 0; - else - end = c->body_offs.len; - } - else - end = c->arr_offs_inner[c->array_depth - 1].len; - - MemoryContext old = MemoryContextSwitchTo(h->cxt); - - if (c->array_depth == 1) - { - u64buf_push(&c->arr_offs, end); - c->n_rows++; - h->array_active = false; - } - else - u64buf_push(&c->arr_offs_inner[c->array_depth - 2], end); - MemoryContextSwitchTo(old); - c->array_depth--; +ch_binary_array_end(ch_binary_insert_handle* h) { + if (!h->array_active) { + return; + } + ic_col* c = &h->cols[h->array_col_idx]; + + if (c->array_depth == 0) { + return; + } + + uint64_t end; + + /* + * At innermost depth, count is leaf-element count; at outer levels the + * count is total children written so far at the level below. + */ + if (c->array_depth == c->ndim) { + if (c->layout == IC_ARRAY_FIXED || c->layout == IC_ARRAY_NESTED_FIXED) { + end = c->elem_size ? (uint64_t)(c->body.len / c->elem_size) : 0; + } else { + end = c->body_offs.len; + } + } else { + end = c->arr_offs_inner[c->array_depth - 1].len; + } + + MemoryContext old = MemoryContextSwitchTo(h->cxt); + + if (c->array_depth == 1) { + u64buf_push(&c->arr_offs, end); + c->n_rows++; + h->array_active = false; + } else { + u64buf_push(&c->arr_offs_inner[c->array_depth - 2], end); + } + MemoryContextSwitchTo(old); + c->array_depth--; } bool -ch_binary_array_active(const ch_binary_insert_handle * h) -{ - return h && h->array_active; +ch_binary_array_active(const ch_binary_insert_handle* h) { + return h && h->array_active; } chc_kind -ch_binary_column_kind(const ch_binary_insert_handle * h, size_t col) -{ - if (col >= h->ncols) - return CHC_VOID; - const ic_col *c = &h->cols[col]; - - if (h->array_active) - { - c = &h->cols[h->array_col_idx]; - - /* - * While nested, surface CHC_ARRAY until the innermost layer is open; - * at that point return the leaf kind so encode targets scalars. - */ - if (c->array_depth < c->ndim) - return CHC_ARRAY; - return chc_type_kind(c->elem_t); - } - if (c->layout == IC_ARRAY_FIXED || c->layout == IC_ARRAY_STRING - || c->layout == IC_ARRAY_NESTED_FIXED || c->layout == IC_ARRAY_NESTED_STRING) - return CHC_ARRAY; - if (c->layout == IC_LC_STRING) - return CHC_STRING; /* PG side targets TEXT */ - return chc_type_kind(c->inner_t); +ch_binary_column_kind(const ch_binary_insert_handle* h, size_t col) { + if (col >= h->ncols) { + return CHC_VOID; + } + const ic_col* c = &h->cols[col]; + + if (h->array_active) { + c = &h->cols[h->array_col_idx]; + + /* + * While nested, surface CHC_ARRAY until the innermost layer is open; + * at that point return the leaf kind so encode targets scalars. + */ + if (c->array_depth < c->ndim) { + return CHC_ARRAY; + } + return chc_type_kind(c->elem_t); + } + if (c->layout == IC_ARRAY_FIXED || c->layout == IC_ARRAY_STRING || + c->layout == IC_ARRAY_NESTED_FIXED || c->layout == IC_ARRAY_NESTED_STRING) { + return CHC_ARRAY; + } + if (c->layout == IC_LC_STRING) { + return CHC_STRING; /* PG side targets TEXT */ + } + return chc_type_kind(c->inner_t); } uint32_t -ch_binary_column_datetime64_precision(const ch_binary_insert_handle * h, size_t col) -{ - if (col >= h->ncols) - return 0; - return h->cols[col].dt64_precision; +ch_binary_column_datetime64_precision(const ch_binary_insert_handle* h, size_t col) { + if (col >= h->ncols) { + return 0; + } + return h->cols[col].dt64_precision; } /* Dedup map for LowCardinality dict */ -typedef struct lcd_key -{ - const uint8_t *bytes; - size_t len; -} lcd_key; - -typedef struct lcd_entry -{ - uint32 status; - lcd_key key; - uint32 idx; -} lcd_entry; - -#define SH_PREFIX lcd -#define SH_ELEMENT_TYPE lcd_entry -#define SH_KEY_TYPE lcd_key -#define SH_KEY key -#define SH_HASH_KEY(tb, key) hash_bytes((key).bytes, (int) (key).len) -#define SH_EQUAL(tb, a, b) \ - ((a).len == (b).len && memcmp((a).bytes, (b).bytes, (a).len) == 0) -#define SH_SCOPE static inline +typedef struct lcd_key { + const uint8_t* bytes; + size_t len; +} lcd_key; + +typedef struct lcd_entry { + uint32 status; + lcd_key key; + uint32 idx; +} lcd_entry; + +#define SH_PREFIX lcd +#define SH_ELEMENT_TYPE lcd_entry +#define SH_KEY_TYPE lcd_key +#define SH_KEY key +#define SH_HASH_KEY(tb, key) hash_bytes((key).bytes, (int)(key).len) +#define SH_EQUAL(tb, a, b) \ + ((a).len == (b).len && memcmp((a).bytes, (b).bytes, (a).len) == 0) +#define SH_SCOPE static inline #define SH_DECLARE #define SH_DEFINE #include "lib/simplehash.h" /* Build LC dict (collect unique strings in insertion order) */ static void -build_lc_dict(ic_col * c, bool nullable, - uint64_t * *out_dict_offs, uint8_t * *out_dict_data, - size_t * out_dict_n, void **out_keys, int *out_key_size, - size_t * out_n_rows) -{ - size_t n_rows = c->body_offs.len; - uint64_t *dict_offs = NULL; - uint8_t *dict_data = NULL; - uint32_t *keys = n_rows ? palloc(n_rows * sizeof(uint32_t)) : NULL; - size_t dict_n = 0; - size_t dict_cap = 0; - size_t data_len = 0; - lcd_hash *ht = n_rows - ? lcd_create(CurrentMemoryContext, - (uint32) Min(n_rows, (size_t) PG_UINT32_MAX), NULL) - : NULL; - - if (nullable) - { - /* dict[0] = "" sentinel. */ - dict_cap = 8; - dict_offs = palloc(dict_cap * sizeof(uint64_t)); - dict_offs[0] = 0; - dict_n = 1; - } - - for (size_t i = 0; i < n_rows; i++) - { - uint64_t start = i == 0 ? 0 : c->body_offs.data[i - 1]; - uint64_t end = c->body_offs.data[i]; - size_t len = (size_t) (end - start); - const uint8_t *bytes = c->body.data + start; - lcd_key k = {bytes, len}; - lcd_entry *entry; - bool found; - - /* If nullable and this row was null (signalled by null bit), key 0. */ - if (nullable && c->nulls.data[i]) - { - keys[i] = 0; - continue; - } - - entry = lcd_insert(ht, k, &found); - if (found) - { - keys[i] = entry->idx; - continue; - } - - if (dict_n == dict_cap) - { - dict_cap = dict_cap ? dict_cap * 2 : 64; - dict_offs = dict_offs - ? repalloc(dict_offs, dict_cap * sizeof(uint64_t)) - : palloc(dict_cap * sizeof(uint64_t)); - } - data_len += len; - dict_offs[dict_n] = data_len; - entry->idx = (uint32) dict_n; - keys[i] = (uint32) dict_n; - dict_n++; - } - - if (data_len) - { - lcd_iterator it; - lcd_entry *e; - - dict_data = MemoryContextAllocExtended(CurrentMemoryContext, data_len, - MCXT_ALLOC_HUGE); - lcd_start_iterate(ht, &it); - while ((e = lcd_iterate(ht, &it)) != NULL) - { - uint64_t s = e->idx == 0 ? 0 : dict_offs[e->idx - 1]; - - memcpy(dict_data + s, e->key.bytes, e->key.len); - } - } - if (ht) - lcd_destroy(ht); - *out_dict_offs = dict_offs; - *out_dict_data = dict_data; - *out_dict_n = dict_n; - *out_keys = keys; - *out_key_size = 4; - *out_n_rows = n_rows; +build_lc_dict( + ic_col* c, + bool nullable, + uint64_t** out_dict_offs, + uint8_t** out_dict_data, + size_t* out_dict_n, + void** out_keys, + int* out_key_size, + size_t* out_n_rows +) { + size_t n_rows = c->body_offs.len; + uint64_t* dict_offs = NULL; + uint8_t* dict_data = NULL; + uint32_t* keys = n_rows ? palloc(n_rows * sizeof(uint32_t)) : NULL; + size_t dict_n = 0; + size_t dict_cap = 0; + size_t data_len = 0; + lcd_hash* ht = + n_rows + ? lcd_create( + CurrentMemoryContext, (uint32)Min(n_rows, (size_t)PG_UINT32_MAX), NULL + ) + : NULL; + + if (nullable) { + /* dict[0] = "" sentinel. */ + dict_cap = 8; + dict_offs = palloc(dict_cap * sizeof(uint64_t)); + dict_offs[0] = 0; + dict_n = 1; + } + + for (size_t i = 0; i < n_rows; i++) { + uint64_t start = i == 0 ? 0 : c->body_offs.data[i - 1]; + uint64_t end = c->body_offs.data[i]; + size_t len = (size_t)(end - start); + const uint8_t* bytes = c->body.data + start; + lcd_key k = { bytes, len }; + lcd_entry* entry; + bool found; + + /* If nullable and this row was null (signalled by null bit), key 0. */ + if (nullable && c->nulls.data[i]) { + keys[i] = 0; + continue; + } + + entry = lcd_insert(ht, k, &found); + if (found) { + keys[i] = entry->idx; + continue; + } + + if (dict_n == dict_cap) { + dict_cap = dict_cap ? dict_cap * 2 : 64; + dict_offs = dict_offs ? repalloc(dict_offs, dict_cap * sizeof(uint64_t)) + : palloc(dict_cap * sizeof(uint64_t)); + } + data_len += len; + dict_offs[dict_n] = data_len; + entry->idx = (uint32)dict_n; + keys[i] = (uint32)dict_n; + dict_n++; + } + + if (data_len) { + lcd_iterator it; + lcd_entry* e; + + dict_data = + MemoryContextAllocExtended(CurrentMemoryContext, data_len, MCXT_ALLOC_HUGE); + lcd_start_iterate(ht, &it); + while ((e = lcd_iterate(ht, &it)) != NULL) { + uint64_t s = e->idx == 0 ? 0 : dict_offs[e->idx - 1]; + + memcpy(dict_data + s, e->key.bytes, e->key.len); + } + } + if (ht) { + lcd_destroy(ht); + } + *out_dict_offs = dict_offs; + *out_dict_data = dict_data; + *out_dict_n = dict_n; + *out_keys = keys; + *out_key_size = 4; + *out_n_rows = n_rows; } void -ch_binary_flush_block(ch_binary_insert_handle * h) -{ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - chc_err err = {}; - chc_block_builder *bb = NULL; - int rc = chc_block_builder_init(&bb, &pg_chc_alloc, &err); - - if (rc != CHC_OK) - raise_chc(&err, ERRCODE_FDW_ERROR, "could not append data to column - "); - - for (size_t i = 0; i < h->ncols; i++) - { - ic_col *c = &h->cols[i]; - const char *name = c->info.name ? c->info.name : ""; - size_t nlen = strlen(name); - - switch (c->layout) - { - case IC_FIXED: - { - size_t n_rows = c->elem_size ? c->body.len / c->elem_size : 0; - - if (c->is_nullable) - rc = chc_block_builder_append_nullable_fixed(bb, name, nlen, c->t, - c->nulls.data, c->body.data, - n_rows, &err); - else - rc = chc_block_builder_append_fixed(bb, name, nlen, c->t, - c->body.data, n_rows, &err); - break; - } - case IC_STRING: - { - size_t n_rows = c->body_offs.len; - - if (c->is_nullable) - rc = chc_block_builder_append_nullable_string(bb, name, nlen, c->t, - c->nulls.data, - c->body_offs.data, - c->body.data, - n_rows, &err); - else - rc = chc_block_builder_append_string(bb, name, nlen, - c->body_offs.data, - c->body.data, - n_rows, &err); - break; - } - case IC_JSON_STRING: - { - /* - * JSON columns share IC_STRING's per-row accumulator; the - * library emits the 8-byte SerializationObject::STRING - * prefix and writes rows as String-Binary. - */ - size_t n_rows = c->body_offs.len; - - rc = chc_block_builder_append_json_string(bb, name, nlen, c->t, - c->body_offs.data, - c->body.data, - n_rows, &err); - break; - } - case IC_LC_STRING: - { - bool nullable_inner = chc_type_kind(chc_type_child(c->inner_t, 0)) == CHC_NULLABLE; - size_t dict_n, - n_rows; - int key_size; - uint64_t *lc_offs; - uint8_t *lc_data; - void *lc_keys; - - build_lc_dict(c, nullable_inner, &lc_offs, &lc_data, - &dict_n, &lc_keys, &key_size, &n_rows); - rc = chc_block_builder_append_low_cardinality_string(bb, name, nlen, c->t, - key_size, lc_keys, - lc_offs, lc_data, - dict_n, n_rows, &err); - break; - } - case IC_ARRAY_FIXED: - { - size_t n_rows = c->arr_offs.len; - - rc = chc_block_builder_append_array_fixed(bb, name, nlen, c->t, - c->arr_offs.data, - c->body.data, - n_rows, &err); - break; - } - case IC_ARRAY_STRING: - { - size_t n_rows = c->arr_offs.len; - - rc = chc_block_builder_append_array_string(bb, name, nlen, c->t, - c->arr_offs.data, - c->body_offs.data, - c->body.data, - n_rows, &err); - break; - } - case IC_ARRAY_NESTED_FIXED: - case IC_ARRAY_NESTED_STRING: - { - size_t n_rows = c->arr_offs.len; - const uint64_t **lvl_offsets = palloc((size_t) c->ndim * sizeof(*lvl_offsets)); - size_t *lvl_lens = palloc((size_t) c->ndim * sizeof(*lvl_lens)); - - lvl_offsets[0] = c->arr_offs.data; - lvl_lens[0] = c->arr_offs.len; - for (int lvl = 1; lvl < c->ndim; lvl++) - { - lvl_offsets[lvl] = c->arr_offs_inner[lvl - 1].data; - lvl_lens[lvl] = c->arr_offs_inner[lvl - 1].len; - } - if (c->layout == IC_ARRAY_NESTED_FIXED) - rc = chc_block_builder_append_array_nested_fixed( - bb, name, nlen, c->t, c->ndim, - lvl_offsets, lvl_lens, - c->body.data, n_rows, &err); - else - rc = chc_block_builder_append_array_nested_string( - bb, name, nlen, c->t, c->ndim, - lvl_offsets, lvl_lens, - c->body_offs.data, c->body.data, - n_rows, &err); - break; - } - } - if (rc != CHC_OK) - raise_chc(&err, ERRCODE_FDW_ERROR, - "could not append data to column - "); - } - - rc = chc_client_send_data(h->client, bb, &err); - chc_block_builder_destroy(bb); - if (rc != CHC_OK) - { - if (h->state) - h->state->broken = true; - raise_chc(&err, ERRCODE_FDW_ERROR, "could not insert columns - "); - } - - /* Reset per-column buffers for next batch. */ - for (size_t i = 0; i < h->ncols; i++) - { - ic_col *c = &h->cols[i]; - - dynbuf_reset(&c->body); - dynbuf_reset(&c->nulls); - u64buf_reset(&c->body_offs); - u64buf_reset(&c->arr_offs); - if (c->arr_offs_inner) - { - for (int lvl = 0; lvl + 1 < c->ndim; lvl++) - u64buf_reset(&c->arr_offs_inner[lvl]); - } - c->n_rows = 0; - } - MemoryContextSwitchTo(old); +ch_binary_flush_block(ch_binary_insert_handle* h) { + MemoryContext old = MemoryContextSwitchTo(h->cxt); + chc_err err = {}; + chc_block_builder* bb = NULL; + int rc = chc_block_builder_init(&bb, &pg_chc_alloc, &err); + + if (rc != CHC_OK) { + raise_chc(&err, ERRCODE_FDW_ERROR, "could not append data to column - "); + } + + for (size_t i = 0; i < h->ncols; i++) { + ic_col* c = &h->cols[i]; + const char* name = c->info.name ? c->info.name : ""; + size_t nlen = strlen(name); + + switch (c->layout) { + case IC_FIXED: { + size_t n_rows = c->elem_size ? c->body.len / c->elem_size : 0; + + if (c->is_nullable) { + rc = chc_block_builder_append_nullable_fixed( + bb, name, nlen, c->t, c->nulls.data, c->body.data, n_rows, &err + ); + } else { + rc = chc_block_builder_append_fixed( + bb, name, nlen, c->t, c->body.data, n_rows, &err + ); + } + break; + } + case IC_STRING: { + size_t n_rows = c->body_offs.len; + + if (c->is_nullable) { + rc = chc_block_builder_append_nullable_string( + bb, + name, + nlen, + c->t, + c->nulls.data, + c->body_offs.data, + c->body.data, + n_rows, + &err + ); + } else { + rc = chc_block_builder_append_string( + bb, name, nlen, c->body_offs.data, c->body.data, n_rows, &err + ); + } + break; + } + case IC_JSON_STRING: { + /* + * JSON columns share IC_STRING's per-row accumulator; the + * library emits the 8-byte SerializationObject::STRING + * prefix and writes rows as String-Binary. + */ + size_t n_rows = c->body_offs.len; + + rc = chc_block_builder_append_json_string( + bb, name, nlen, c->t, c->body_offs.data, c->body.data, n_rows, &err + ); + break; + } + case IC_LC_STRING: { + bool nullable_inner = + chc_type_kind(chc_type_child(c->inner_t, 0)) == CHC_NULLABLE; + size_t dict_n, n_rows; + int key_size; + uint64_t* lc_offs; + uint8_t* lc_data; + void* lc_keys; + + build_lc_dict( + c, + nullable_inner, + &lc_offs, + &lc_data, + &dict_n, + &lc_keys, + &key_size, + &n_rows + ); + rc = chc_block_builder_append_low_cardinality_string( + bb, + name, + nlen, + c->t, + key_size, + lc_keys, + lc_offs, + lc_data, + dict_n, + n_rows, + &err + ); + break; + } + case IC_ARRAY_FIXED: { + size_t n_rows = c->arr_offs.len; + + rc = chc_block_builder_append_array_fixed( + bb, name, nlen, c->t, c->arr_offs.data, c->body.data, n_rows, &err + ); + break; + } + case IC_ARRAY_STRING: { + size_t n_rows = c->arr_offs.len; + + rc = chc_block_builder_append_array_string( + bb, + name, + nlen, + c->t, + c->arr_offs.data, + c->body_offs.data, + c->body.data, + n_rows, + &err + ); + break; + } + case IC_ARRAY_NESTED_FIXED: + case IC_ARRAY_NESTED_STRING: { + size_t n_rows = c->arr_offs.len; + const uint64_t** lvl_offsets = + palloc((size_t)c->ndim * sizeof(*lvl_offsets)); + size_t* lvl_lens = palloc((size_t)c->ndim * sizeof(*lvl_lens)); + + lvl_offsets[0] = c->arr_offs.data; + lvl_lens[0] = c->arr_offs.len; + for (int lvl = 1; lvl < c->ndim; lvl++) { + lvl_offsets[lvl] = c->arr_offs_inner[lvl - 1].data; + lvl_lens[lvl] = c->arr_offs_inner[lvl - 1].len; + } + if (c->layout == IC_ARRAY_NESTED_FIXED) { + rc = chc_block_builder_append_array_nested_fixed( + bb, + name, + nlen, + c->t, + c->ndim, + lvl_offsets, + lvl_lens, + c->body.data, + n_rows, + &err + ); + } else { + rc = chc_block_builder_append_array_nested_string( + bb, + name, + nlen, + c->t, + c->ndim, + lvl_offsets, + lvl_lens, + c->body_offs.data, + c->body.data, + n_rows, + &err + ); + } + break; + } + } + if (rc != CHC_OK) { + raise_chc(&err, ERRCODE_FDW_ERROR, "could not append data to column - "); + } + } + + rc = chc_client_send_data(h->client, bb, &err); + chc_block_builder_destroy(bb); + if (rc != CHC_OK) { + if (h->state) { + h->state->broken = true; + } + raise_chc(&err, ERRCODE_FDW_ERROR, "could not insert columns - "); + } + + /* Reset per-column buffers for next batch. */ + for (size_t i = 0; i < h->ncols; i++) { + ic_col* c = &h->cols[i]; + + dynbuf_reset(&c->body); + dynbuf_reset(&c->nulls); + u64buf_reset(&c->body_offs); + u64buf_reset(&c->arr_offs); + if (c->arr_offs_inner) { + for (int lvl = 0; lvl + 1 < c->ndim; lvl++) { + u64buf_reset(&c->arr_offs_inner[lvl]); + } + } + c->n_rows = 0; + } + MemoryContextSwitchTo(old); } /* @@ -1388,87 +1445,83 @@ ch_binary_flush_block(ch_binary_insert_handle * h) * ch_binary_release_insert deletes it. */ void -ch_binary_finalize_insert(ch_binary_insert_handle * h) -{ - if (!h || h->finalized) - return; - - /* - * Set early so an ereport(ERROR) below still leaves h in the "do not - * touch the wire" state for the release callback. - */ - h->finalized = true; - - if (!(h->started && h->client)) - return; - - MemoryContext old = MemoryContextSwitchTo(h->cxt); - char *exc_msg = NULL; - bool broke = false; - chc_err err = {}; - int rc = chc_client_send_data(h->client, NULL, &err); - - if (rc != CHC_OK) - { - broke = true; - exc_msg = pstrdup(err.msg[0] ? err.msg : "send_data failed"); - } - else - { - /* Drain until EOS or exception. */ - for (;;) - { - chc_packet pkt = {}; - - err = (chc_err) - { - 0 - }; - rc = chc_client_recv_packet(h->client, &pkt, &err); - if (rc != CHC_OK) - { - broke = true; - exc_msg = pstrdup(err.msg[0] ? err.msg : "recv_packet failed"); - chc_packet_clear(h->client, &pkt); - break; - } - if (pkt.kind == CHC_PKT_EXCEPTION) - { - const char *msg = "server exception"; - - if (pkt.exception && pkt.exception->display_text) - msg = pkt.exception->display_text; - else if (pkt.exception && pkt.exception->name) - msg = pkt.exception->name; - exc_msg = pstrdup(msg); - broke = true; - chc_packet_clear(h->client, &pkt); - break; - } - chc_packet_clear(h->client, &pkt); - if (pkt.kind == CHC_PKT_END_OF_STREAM) - break; - } - } - - /* - * Server raised mid-INSERT and typically closes the socket; the next op - * would EPIPE. Mark broken so the cache rebuilds. - */ - if (broke && h->state) - h->state->broken = true; - - MemoryContextSwitchTo(old); - - if (exc_msg) - { - /* exc_msg lives in h->cxt; copy into parent before raising. */ - char *parent_msg = pstrdup(exc_msg); - - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: could not finish INSERT - %s", parent_msg))); - } +ch_binary_finalize_insert(ch_binary_insert_handle* h) { + if (!h || h->finalized) { + return; + } + + /* + * Set early so an ereport(ERROR) below still leaves h in the "do not + * touch the wire" state for the release callback. + */ + h->finalized = true; + + if (!(h->started && h->client)) { + return; + } + + MemoryContext old = MemoryContextSwitchTo(h->cxt); + char* exc_msg = NULL; + bool broke = false; + chc_err err = {}; + int rc = chc_client_send_data(h->client, NULL, &err); + + if (rc != CHC_OK) { + broke = true; + exc_msg = pstrdup(err.msg[0] ? err.msg : "send_data failed"); + } else { + /* Drain until EOS or exception. */ + for (;;) { + chc_packet pkt = {}; + + err = (chc_err){ 0 }; + rc = chc_client_recv_packet(h->client, &pkt, &err); + if (rc != CHC_OK) { + broke = true; + exc_msg = pstrdup(err.msg[0] ? err.msg : "recv_packet failed"); + chc_packet_clear(h->client, &pkt); + break; + } + if (pkt.kind == CHC_PKT_EXCEPTION) { + const char* msg = "server exception"; + + if (pkt.exception && pkt.exception->display_text) { + msg = pkt.exception->display_text; + } else if (pkt.exception && pkt.exception->name) { + msg = pkt.exception->name; + } + exc_msg = pstrdup(msg); + broke = true; + chc_packet_clear(h->client, &pkt); + break; + } + chc_packet_clear(h->client, &pkt); + if (pkt.kind == CHC_PKT_END_OF_STREAM) { + break; + } + } + } + + /* + * Server raised mid-INSERT and typically closes the socket; the next op + * would EPIPE. Mark broken so the cache rebuilds. + */ + if (broke && h->state) { + h->state->broken = true; + } + + MemoryContextSwitchTo(old); + + if (exc_msg) { + /* exc_msg lives in h->cxt; copy into parent before raising. */ + char* parent_msg = pstrdup(exc_msg); + + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: could not finish INSERT - %s", parent_msg)) + ); + } } /* @@ -1478,13 +1531,14 @@ ch_binary_finalize_insert(ch_binary_insert_handle * h) * next use. */ void -ch_binary_release_insert(ch_binary_insert_handle * h) -{ - if (!h) - return; +ch_binary_release_insert(ch_binary_insert_handle* h) { + if (!h) { + return; + } - if (!h->finalized && h->started && h->state) - h->state->broken = true; + if (!h->finalized && h->started && h->state) { + h->state->broken = true; + } - MemoryContextDelete(h->cxt); + MemoryContextDelete(h->cxt); } diff --git a/src/binary/select.c b/src/binary/select.c index 7838643..f427fb7 100644 --- a/src/binary/select.c +++ b/src/binary/select.c @@ -29,45 +29,44 @@ * Cancel by flushing the in-flight block + EndOfStream within a second. */ #define DRAIN_DEADLINE_US (5 * 1000 * 1000) -struct ch_binary_response_t -{ - MemoryContext cxt; - struct ch_binary_state *state; /* parent connection state */ - chc_client *client; /* recv_packet target */ - bool (*check_cancel) (void); - - char *error; /* NULL on success */ - bool success; - bool eos; /* END_OF_STREAM or exception seen */ - - size_t columns_count; - chc_block *staged; /* next block to hand out; owned */ - chc_block *prev; /* last block handed out; freed at next fetch */ +struct ch_binary_response_t { + MemoryContext cxt; + struct ch_binary_state* state; /* parent connection state */ + chc_client* client; /* recv_packet target */ + bool (*check_cancel)(void); + + char* error; /* NULL on success */ + bool success; + bool eos; /* END_OF_STREAM or exception seen */ + + size_t columns_count; + chc_block* staged; /* next block to hand out; owned */ + chc_block* prev; /* last block handed out; freed at next fetch */ }; static void -resp_set_error(ch_binary_response_t * resp, const char *msg) -{ - if (resp->error) - return; - resp->error = pstrdup(msg && *msg ? msg : "?"); +resp_set_error(ch_binary_response_t* resp, const char* msg) { + if (resp->error) { + return; + } + resp->error = pstrdup(msg && *msg ? msg : "?"); } static void -resp_set_exception(ch_binary_response_t * resp, const chc_exception * ex) -{ - if (resp->error) - return; - const char *msg = NULL; - - if (ex) - { - if (ex->display_text && ex->display_text[0]) - msg = ex->display_text; - else if (ex->name && ex->name[0]) - msg = ex->name; - } - resp->error = pstrdup(msg ? msg : "server exception"); +resp_set_exception(ch_binary_response_t* resp, const chc_exception* ex) { + if (resp->error) { + return; + } + const char* msg = NULL; + + if (ex) { + if (ex->display_text && ex->display_text[0]) { + msg = ex->display_text; + } else if (ex->name && ex->name[0]) { + msg = ex->name; + } + } + resp->error = pstrdup(msg ? msg : "server exception"); } /* @@ -78,104 +77,97 @@ resp_set_exception(ch_binary_response_t * resp, const chc_exception * ex) * (progress, log, profile, ...) are silently consumed. */ static void -pump_one(ch_binary_response_t * resp) -{ - MemoryContext old = MemoryContextSwitchTo(resp->cxt); - chc_packet pkt = {}; - chc_err err = {}; - int rc = chc_client_recv_packet(resp->client, &pkt, &err); - - if (rc != CHC_OK) - { - resp_set_error(resp, err.msg); - resp->state->broken = true; - resp->eos = true; - MemoryContextSwitchTo(old); - return; - } - - if (resp->check_cancel && resp->check_cancel()) - resp_set_error(resp, "query was canceled"); - - switch (pkt.kind) - { - case CHC_PKT_DATA: - if (pkt.block && chc_block_n_columns(pkt.block) > 0) - { - if (resp->columns_count == 0) - resp->columns_count = chc_block_n_columns(pkt.block); - if (chc_block_n_rows(pkt.block) > 0 && resp->staged == NULL) - { - size_t ncols = chc_block_n_columns(pkt.block); - chc_err verr = {}; - int vrc = CHC_OK; - - for (size_t i = 0; i < ncols; i++) - { - vrc = chc_column_validate(chc_block_column(pkt.block, i), &verr); - if (vrc != CHC_OK) - break; - } - if (vrc != CHC_OK) - { - resp_set_error(resp, verr.msg); - resp->state->broken = true; - resp->eos = true; - } - else - { - resp->staged = pkt.block; - pkt.block = NULL; - } - } - } - chc_packet_clear(resp->client, &pkt); - break; - - case CHC_PKT_EXCEPTION: - resp_set_exception(resp, pkt.exception); - chc_packet_clear(resp->client, &pkt); - - /* - * Older servers (and some protocol states) close the socket after - * raising an exception, so reusing this connection for a - * follow-up query risks EPIPE. Match the legacy C++ driver (which - * always called Client::ResetConnection) and treat the connection - * as broken. - */ - resp->state->broken = true; - resp->eos = true; - break; - - case CHC_PKT_END_OF_STREAM: - chc_packet_clear(resp->client, &pkt); - resp->eos = true; - break; - - default: - chc_packet_clear(resp->client, &pkt); - break; - } - - MemoryContextSwitchTo(old); +pump_one(ch_binary_response_t* resp) { + MemoryContext old = MemoryContextSwitchTo(resp->cxt); + chc_packet pkt = {}; + chc_err err = {}; + int rc = chc_client_recv_packet(resp->client, &pkt, &err); + + if (rc != CHC_OK) { + resp_set_error(resp, err.msg); + resp->state->broken = true; + resp->eos = true; + MemoryContextSwitchTo(old); + return; + } + + if (resp->check_cancel && resp->check_cancel()) { + resp_set_error(resp, "query was canceled"); + } + + switch (pkt.kind) { + case CHC_PKT_DATA: + if (pkt.block && chc_block_n_columns(pkt.block) > 0) { + if (resp->columns_count == 0) { + resp->columns_count = chc_block_n_columns(pkt.block); + } + if (chc_block_n_rows(pkt.block) > 0 && resp->staged == NULL) { + size_t ncols = chc_block_n_columns(pkt.block); + chc_err verr = {}; + int vrc = CHC_OK; + + for (size_t i = 0; i < ncols; i++) { + vrc = chc_column_validate(chc_block_column(pkt.block, i), &verr); + if (vrc != CHC_OK) { + break; + } + } + if (vrc != CHC_OK) { + resp_set_error(resp, verr.msg); + resp->state->broken = true; + resp->eos = true; + } else { + resp->staged = pkt.block; + pkt.block = NULL; + } + } + } + chc_packet_clear(resp->client, &pkt); + break; + + case CHC_PKT_EXCEPTION: + resp_set_exception(resp, pkt.exception); + chc_packet_clear(resp->client, &pkt); + + /* + * Older servers (and some protocol states) close the socket after + * raising an exception, so reusing this connection for a + * follow-up query risks EPIPE. Match the legacy C++ driver (which + * always called Client::ResetConnection) and treat the connection + * as broken. + */ + resp->state->broken = true; + resp->eos = true; + break; + + case CHC_PKT_END_OF_STREAM: + chc_packet_clear(resp->client, &pkt); + resp->eos = true; + break; + + default: + chc_packet_clear(resp->client, &pkt); + break; + } + + MemoryContextSwitchTo(old); } static int64_t -drain_now_us(void) -{ - struct timespec ts; +drain_now_us(void) { + struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (int64_t) ts.tv_sec * 1000000 + ts.tv_nsec / 1000; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (int64_t)ts.tv_sec * 1000000 + ts.tv_nsec / 1000; } static void -drain_set_deadline(struct ch_binary_state *s, int64_t deadline_us) -{ - if (s->tls) - chc_openssl_io_set_deadline(&s->openssl_state, deadline_us); - else - chc_posix_io_set_deadline(&s->posix_state, deadline_us); +drain_set_deadline(struct ch_binary_state* s, int64_t deadline_us) { + if (s->tls) { + chc_openssl_io_set_deadline(&s->openssl_state, deadline_us); + } else { + chc_posix_io_set_deadline(&s->posix_state, deadline_us); + } } /* @@ -187,276 +179,261 @@ drain_set_deadline(struct ch_binary_state *s, int64_t deadline_us) * cache drops the connection. */ static void -drain_until_eos(ch_binary_response_t * resp) -{ - if (resp->eos) - return; - - MemoryContext old = MemoryContextSwitchTo(resp->cxt); - chc_err ce = {}; - - if (chc_client_send_cancel(resp->client, &ce) != CHC_OK) - { - resp->state->broken = true; - resp->eos = true; - MemoryContextSwitchTo(old); - return; - } - resp->state->check_cancel_fn = NULL; - resp->check_cancel = NULL; - - drain_set_deadline(resp->state, drain_now_us() + DRAIN_DEADLINE_US); - - for (;;) - { - chc_packet pkt = {}; - int rc; - - ce = (chc_err) - { - 0 - }; - rc = chc_client_recv_packet(resp->client, &pkt, &ce); - - if (rc != CHC_OK) - { - resp->state->broken = true; - resp->eos = true; - break; - } - bool stop = pkt.kind == CHC_PKT_END_OF_STREAM || - pkt.kind == CHC_PKT_EXCEPTION; - - chc_packet_clear(resp->client, &pkt); - if (stop) - { - resp->eos = true; - break; - } - } - - drain_set_deadline(resp->state, 0); - MemoryContextSwitchTo(old); +drain_until_eos(ch_binary_response_t* resp) { + if (resp->eos) { + return; + } + + MemoryContext old = MemoryContextSwitchTo(resp->cxt); + chc_err ce = {}; + + if (chc_client_send_cancel(resp->client, &ce) != CHC_OK) { + resp->state->broken = true; + resp->eos = true; + MemoryContextSwitchTo(old); + return; + } + resp->state->check_cancel_fn = NULL; + resp->check_cancel = NULL; + + drain_set_deadline(resp->state, drain_now_us() + DRAIN_DEADLINE_US); + + for (;;) { + chc_packet pkt = {}; + int rc; + + ce = (chc_err){ 0 }; + rc = chc_client_recv_packet(resp->client, &pkt, &ce); + + if (rc != CHC_OK) { + resp->state->broken = true; + resp->eos = true; + break; + } + bool stop = pkt.kind == CHC_PKT_END_OF_STREAM || pkt.kind == CHC_PKT_EXCEPTION; + + chc_packet_clear(resp->client, &pkt); + if (stop) { + resp->eos = true; + break; + } + } + + drain_set_deadline(resp->state, 0); + MemoryContextSwitchTo(old); } -ch_binary_response_t * -ch_binary_simple_query(ch_binary_connection_t * conn, const ch_query * query, - bool (*check_cancel) (void)) -{ - struct ch_binary_state *s = conn_state(conn); - MemoryContext cxt = AllocSetContextCreate(CurrentMemoryContext, - "pg_clickhouse binary response", - ALLOCSET_DEFAULT_SIZES); - MemoryContext old = MemoryContextSwitchTo(cxt); - ch_binary_response_t *resp = palloc0(sizeof(*resp)); - - resp->cxt = cxt; - resp->state = s; - resp->client = s->client; - resp->check_cancel = check_cancel; - s->check_cancel_fn = check_cancel; - - size_t n_user_settings = 0; - size_t n_params = (size_t) (query->num_params > 0 ? query->num_params : 0); - chc_query_setting *settings = NULL; - chc_query_param *params = NULL; - bool want_json_as_string = server_supports_json_as_string(s->client); - - { - const kv_list *kv = query->settings; - - if (kv) - n_user_settings = (size_t) kv->length; - } - size_t n_settings = n_user_settings + (want_json_as_string ? 1 : 0); - - if (n_settings) - { - settings = palloc0(n_settings * sizeof(*settings)); - size_t i = 0; - - for (kv_iter it = new_kv_iter(query->settings); !kv_iter_done(&it); kv_iter_next(&it), i++) - { - settings[i].name = it.name; - settings[i].value = it.value; - settings[i].important = true; - } - if (want_json_as_string) - { - settings[i].name = "output_format_native_write_json_as_string"; - settings[i].value = "1"; - settings[i].important = true; - } - } - if (n_params) - { - params = palloc0(n_params * sizeof(*params)); - for (size_t i = 0; i < n_params; i++) - { - char nm[32]; - - snprintf(nm, sizeof(nm), "p%zu", i + 1); - params[i].name = pstrdup(nm); - - /* - * Quote & escape the value the way clickhouse-cpp's - * WriteQuotedString did: wrap in single quotes, replace inner - * specials with backslash-escapes the server's - * Field::restoreFromDump understands. Without escaping inner - * quotes the server stops parsing at the first `'` inside the - * value, which breaks Array(String) parameters whose CH literal - * already contains quoted elements. - */ - const char *raw = query->param_values[i]; - - if (raw) - { - size_t rlen = strlen(raw); - size_t cap = rlen * 4 + 3; - char *dst = palloc(cap); - size_t o = 0; - - dst[o++] = '\''; - for (size_t j = 0; j < rlen; j++) - { - unsigned char ch = (unsigned char) raw[j]; - - switch (ch) - { - case '\0': - dst[o++] = '\\'; - dst[o++] = 'x'; - dst[o++] = '0'; - dst[o++] = '0'; - break; - case '\b': - dst[o++] = '\\'; - dst[o++] = 'x'; - dst[o++] = '0'; - dst[o++] = '8'; - break; - case '\t': - dst[o++] = '\\'; - dst[o++] = 't'; - break; - case '\n': - dst[o++] = '\\'; - dst[o++] = 'n'; - break; - case '\'': - dst[o++] = '\\'; - dst[o++] = 'x'; - dst[o++] = '2'; - dst[o++] = '7'; - break; - case '\\': - dst[o++] = '\\'; - dst[o++] = '\\'; - break; - default: - dst[o++] = (char) ch; - } - } - dst[o++] = '\''; - dst[o] = '\0'; - params[i].value = dst; - } - else - params[i].value = "'\\N'"; - } - } - - chc_query_opts opts = { - .settings = settings, - .n_settings = n_settings, - .params = params, - .n_params = n_params, - }; - chc_err err = {}; - int rc = chc_client_send_query_ex(s->client, query->sql, - strlen(query->sql), &opts, &err); - - if (rc != CHC_OK) - { - resp_set_error(resp, err.msg); - s->broken = true; - resp->eos = true; - goto done; - } - - /* - * Pump until schema is known so callers can call - * ch_binary_response_columns immediately. Empty result sets exit on eos - * with columns_count still set from the header Data block. - */ - while (resp->columns_count == 0 && !resp->eos && !resp->error) - pump_one(resp); +ch_binary_response_t* +ch_binary_simple_query( + ch_binary_connection_t* conn, + const ch_query* query, + bool (*check_cancel)(void) +) { + struct ch_binary_state* s = conn_state(conn); + MemoryContext cxt = AllocSetContextCreate( + CurrentMemoryContext, "pg_clickhouse binary response", ALLOCSET_DEFAULT_SIZES + ); + MemoryContext old = MemoryContextSwitchTo(cxt); + ch_binary_response_t* resp = palloc0(sizeof(*resp)); + + resp->cxt = cxt; + resp->state = s; + resp->client = s->client; + resp->check_cancel = check_cancel; + s->check_cancel_fn = check_cancel; + + size_t n_user_settings = 0; + size_t n_params = (size_t)(query->num_params > 0 ? query->num_params : 0); + chc_query_setting* settings = NULL; + chc_query_param* params = NULL; + bool want_json_as_string = server_supports_json_as_string(s->client); + + { + const kv_list* kv = query->settings; + + if (kv) { + n_user_settings = (size_t)kv->length; + } + } + size_t n_settings = n_user_settings + (want_json_as_string ? 1 : 0); + + if (n_settings) { + settings = palloc0(n_settings * sizeof(*settings)); + size_t i = 0; + + for (kv_iter it = new_kv_iter(query->settings); !kv_iter_done(&it); + kv_iter_next(&it), i++) { + settings[i].name = it.name; + settings[i].value = it.value; + settings[i].important = true; + } + if (want_json_as_string) { + settings[i].name = "output_format_native_write_json_as_string"; + settings[i].value = "1"; + settings[i].important = true; + } + } + if (n_params) { + params = palloc0(n_params * sizeof(*params)); + for (size_t i = 0; i < n_params; i++) { + char nm[32]; + + snprintf(nm, sizeof(nm), "p%zu", i + 1); + params[i].name = pstrdup(nm); + + /* + * Quote & escape the value the way clickhouse-cpp's + * WriteQuotedString did: wrap in single quotes, replace inner + * specials with backslash-escapes the server's + * Field::restoreFromDump understands. Without escaping inner + * quotes the server stops parsing at the first `'` inside the + * value, which breaks Array(String) parameters whose CH literal + * already contains quoted elements. + */ + const char* raw = query->param_values[i]; + + if (raw) { + size_t rlen = strlen(raw); + size_t cap = rlen * 4 + 3; + char* dst = palloc(cap); + size_t o = 0; + + dst[o++] = '\''; + for (size_t j = 0; j < rlen; j++) { + unsigned char ch = (unsigned char)raw[j]; + + switch (ch) { + case '\0': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '0'; + dst[o++] = '0'; + break; + case '\b': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '0'; + dst[o++] = '8'; + break; + case '\t': + dst[o++] = '\\'; + dst[o++] = 't'; + break; + case '\n': + dst[o++] = '\\'; + dst[o++] = 'n'; + break; + case '\'': + dst[o++] = '\\'; + dst[o++] = 'x'; + dst[o++] = '2'; + dst[o++] = '7'; + break; + case '\\': + dst[o++] = '\\'; + dst[o++] = '\\'; + break; + default: + dst[o++] = (char)ch; + } + } + dst[o++] = '\''; + dst[o] = '\0'; + params[i].value = dst; + } else { + params[i].value = "'\\N'"; + } + } + } + + chc_query_opts opts = { + .settings = settings, + .n_settings = n_settings, + .params = params, + .n_params = n_params, + }; + chc_err err = {}; + int rc = chc_client_send_query_ex( + s->client, query->sql, strlen(query->sql), &opts, &err + ); + + if (rc != CHC_OK) { + resp_set_error(resp, err.msg); + s->broken = true; + resp->eos = true; + goto done; + } + + /* + * Pump until schema is known so callers can call + * ch_binary_response_columns immediately. Empty result sets exit on eos + * with columns_count still set from the header Data block. + */ + while (resp->columns_count == 0 && !resp->eos && !resp->error) { + pump_one(resp); + } done: - resp->success = (resp->error == NULL); - MemoryContextSwitchTo(old); - return resp; + resp->success = (resp->error == NULL); + MemoryContextSwitchTo(old); + return resp; } void -ch_binary_response_free(ch_binary_response_t * resp) -{ - if (!resp) - return; - - /* Avoid raising from a MemoryContextResetCallback. */ - PG_TRY(); - { - drain_until_eos(resp); - } - PG_CATCH(); - { - FlushErrorState(); - resp->state->broken = true; - } - PG_END_TRY(); - - resp->state->check_cancel_fn = NULL; - MemoryContextDelete(resp->cxt); +ch_binary_response_free(ch_binary_response_t* resp) { + if (!resp) { + return; + } + + /* Avoid raising from a MemoryContextResetCallback. */ + PG_TRY(); + { drain_until_eos(resp); } + PG_CATCH(); + { + FlushErrorState(); + resp->state->broken = true; + } + PG_END_TRY(); + + resp->state->check_cancel_fn = NULL; + MemoryContextDelete(resp->cxt); } -const char * -ch_binary_response_error(const ch_binary_response_t * resp) -{ - return resp ? resp->error : NULL; +const char* +ch_binary_response_error(const ch_binary_response_t* resp) { + return resp ? resp->error : NULL; } bool -ch_binary_response_success(const ch_binary_response_t * resp) -{ - return resp && resp->success; +ch_binary_response_success(const ch_binary_response_t* resp) { + return resp && resp->success; } size_t -ch_binary_response_columns(const ch_binary_response_t * resp) -{ - return resp ? resp->columns_count : 0; +ch_binary_response_columns(const ch_binary_response_t* resp) { + return resp ? resp->columns_count : 0; } -const chc_block * -ch_binary_response_fetch_next_block(ch_binary_response_t * resp) -{ - if (!resp) - return NULL; +const chc_block* +ch_binary_response_fetch_next_block(ch_binary_response_t* resp) { + if (!resp) { + return NULL; + } - if (resp->prev) - { - chc_block_destroy(resp->prev, &pg_chc_alloc); - resp->prev = NULL; - } + if (resp->prev) { + chc_block_destroy(resp->prev, &pg_chc_alloc); + resp->prev = NULL; + } - while (resp->staged == NULL && !resp->eos && !resp->error) - pump_one(resp); + while (resp->staged == NULL && !resp->eos && !resp->error) { + pump_one(resp); + } - if (resp->staged == NULL) - return NULL; + if (resp->staged == NULL) { + return NULL; + } - resp->prev = resp->staged; - resp->staged = NULL; - return resp->prev; + resp->prev = resp->staged; + resp->staged = NULL; + return resp->prev; } diff --git a/src/connection.c b/src/connection.c index f0fd38b..4de12d3 100644 --- a/src/connection.c +++ b/src/connection.c @@ -16,9 +16,9 @@ #include "postgres.h" #include "access/htup_details.h" +#include "access/xact.h" #include "catalog/pg_user_mapping.h" #include "common/int.h" -#include "access/xact.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "pgstat.h" @@ -34,141 +34,151 @@ /* * Connection cache (initialized on first use) */ -static HTAB * ConnectionHash = NULL; -static void chfdw_inval_callback(Datum arg, int cacheid, uint32 hashvalue); +static HTAB* ConnectionHash = NULL; +static void +chfdw_inval_callback(Datum arg, int cacheid, uint32 hashvalue); static ch_connection -clickhouse_connect(ForeignServer * server, UserMapping * user) -{ - char *driver = "http"; +clickhouse_connect(ForeignServer* server, UserMapping* user) { + char* driver = "http"; - /* default settings */ - ch_connection_details details = {"127.0.0.1", 0, NULL, NULL, "default"}; + /* default settings */ + ch_connection_details details = { "127.0.0.1", 0, NULL, NULL, "default" }; - chfdw_extract_options(server->options, &driver, &details.host, - &details.port, &details.dbname, &details.username, - &details.password, &details.compression, &details.tls, - &details.min_tls_version); - chfdw_extract_options(user->options, &driver, &details.host, - &details.port, &details.dbname, &details.username, - &details.password, &details.compression, &details.tls, - &details.min_tls_version); + chfdw_extract_options( + server->options, + &driver, + &details.host, + &details.port, + &details.dbname, + &details.username, + &details.password, + &details.compression, + &details.tls, + &details.min_tls_version + ); + chfdw_extract_options( + user->options, + &driver, + &details.host, + &details.port, + &details.dbname, + &details.username, + &details.password, + &details.compression, + &details.tls, + &details.min_tls_version + ); - if (strcmp(driver, "http") == 0) - { - return chfdw_http_connect(&details); - } - else if (strcmp(driver, "binary") == 0) - { - return chfdw_binary_connect(&details); - } - else - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), - errmsg("invalid ClickHouse connection driver"))); + if (strcmp(driver, "http") == 0) { + return chfdw_http_connect(&details); + } else if (strcmp(driver, "binary") == 0) { + return chfdw_binary_connect(&details); + } else { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid ClickHouse connection driver")) + ); + } } ch_connection -chfdw_get_connection(UserMapping * user) -{ - bool found; - ConnCacheEntry *entry; - ConnCacheKey key; +chfdw_get_connection(UserMapping* user) { + bool found; + ConnCacheEntry* entry; + ConnCacheKey key; - /* First time through, initialize connection cache hashtable */ - if (ConnectionHash == NULL) - { - HASHCTL ctl; + /* First time through, initialize connection cache hashtable */ + if (ConnectionHash == NULL) { + HASHCTL ctl; - MemSet(&ctl, 0, sizeof(ctl)); - ctl.keysize = sizeof(ConnCacheKey); - ctl.entrysize = sizeof(ConnCacheEntry); - /* allocate ConnectionHash in the cache context */ - ctl.hcxt = CacheMemoryContext; - ConnectionHash = hash_create("pg_clickhouse connections", 8, - &ctl, - HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(ConnCacheKey); + ctl.entrysize = sizeof(ConnCacheEntry); + /* allocate ConnectionHash in the cache context */ + ctl.hcxt = CacheMemoryContext; + ConnectionHash = hash_create( + "pg_clickhouse connections", 8, &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT + ); - /* - * Register some callback functions that manage connection cleanup. - * This should be done just once in each backend. - */ - CacheRegisterSyscacheCallback(FOREIGNSERVEROID, - chfdw_inval_callback, (Datum) 0); - CacheRegisterSyscacheCallback(USERMAPPINGOID, - chfdw_inval_callback, (Datum) 0); - } + /* + * Register some callback functions that manage connection cleanup. + * This should be done just once in each backend. + */ + CacheRegisterSyscacheCallback(FOREIGNSERVEROID, chfdw_inval_callback, (Datum)0); + CacheRegisterSyscacheCallback(USERMAPPINGOID, chfdw_inval_callback, (Datum)0); + } - /* Create hash key for the entry. Assume no pad bytes in key struct */ - key.userid = user->umid; + /* Create hash key for the entry. Assume no pad bytes in key struct */ + key.userid = user->umid; - /* - * Find or create cached entry for requested connection. - */ - entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found); - if (!found) - { - /* - * We need only clear "conn" here; remaining fields will be filled - * later when "conn" is set. - */ - entry->gate.conn = NULL; - } + /* + * Find or create cached entry for requested connection. + */ + entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found); + if (!found) { + /* + * We need only clear "conn" here; remaining fields will be filled + * later when "conn" is set. + */ + entry->gate.conn = NULL; + } - /* - * If the connection needs to be remade due to invalidation, disconnect as - * soon as we're out of all transactions. - */ - if (entry->gate.conn != NULL && entry->invalidated) - { - elog(LOG, "closing connection to ClickHouse due to invalidation"); - entry->gate.methods->disconnect(entry->gate.conn); - entry->gate.conn = NULL; - } + /* + * If the connection needs to be remade due to invalidation, disconnect as + * soon as we're out of all transactions. + */ + if (entry->gate.conn != NULL && entry->invalidated) { + elog(LOG, "closing connection to ClickHouse due to invalidation"); + entry->gate.methods->disconnect(entry->gate.conn); + entry->gate.conn = NULL; + } - /* - * Drop connections that hit an unrecoverable protocol/IO error on the - * previous statement (server raised mid-INSERT and closed the socket, - * write hit EPIPE, etc). Without this, a subsequent statement would write - * to the dead socket and surface a useless "Broken pipe" instead of the - * real error from the next request. - */ - if (entry->gate.conn != NULL && - entry->gate.methods->is_broken != NULL && - entry->gate.methods->is_broken(entry->gate.conn)) - { - elog(LOG, "closing broken pg_clickhouse connection"); - entry->gate.methods->disconnect(entry->gate.conn); - entry->gate.conn = NULL; - } + /* + * Drop connections that hit an unrecoverable protocol/IO error on the + * previous statement (server raised mid-INSERT and closed the socket, + * write hit EPIPE, etc). Without this, a subsequent statement would write + * to the dead socket and surface a useless "Broken pipe" instead of the + * real error from the next request. + */ + if (entry->gate.conn != NULL && entry->gate.methods->is_broken != NULL && + entry->gate.methods->is_broken(entry->gate.conn)) { + elog(LOG, "closing broken pg_clickhouse connection"); + entry->gate.methods->disconnect(entry->gate.conn); + entry->gate.conn = NULL; + } - /* - * If cache entry doesn't have a connection, we have to establish a new - * connection. (If clickhouse_connect throws an error, the cache entry - * will remain in a valid empty state, ie conn == NULL.) - */ - if (entry->gate.conn == NULL) - { - ForeignServer *server = GetForeignServer(user->serverid); + /* + * If cache entry doesn't have a connection, we have to establish a new + * connection. (If clickhouse_connect throws an error, the cache entry + * will remain in a valid empty state, ie conn == NULL.) + */ + if (entry->gate.conn == NULL) { + ForeignServer* server = GetForeignServer(user->serverid); - /* Reset all transient state fields, to be sure all are clean */ - entry->invalidated = false; - entry->server_hashvalue = - GetSysCacheHashValue1(FOREIGNSERVEROID, - ObjectIdGetDatum(server->serverid)); - entry->mapping_hashvalue = - GetSysCacheHashValue1(USERMAPPINGOID, - ObjectIdGetDatum(user->umid)); + /* Reset all transient state fields, to be sure all are clean */ + entry->invalidated = false; + entry->server_hashvalue = + GetSysCacheHashValue1(FOREIGNSERVEROID, ObjectIdGetDatum(server->serverid)); + entry->mapping_hashvalue = + GetSysCacheHashValue1(USERMAPPINGOID, ObjectIdGetDatum(user->umid)); - /* Now try to make the connection */ - entry->gate = clickhouse_connect(server, user); + /* Now try to make the connection */ + entry->gate = clickhouse_connect(server, user); - elog(DEBUG3, - "new pg_clickhouse connection %p for server \"%s\" (user mapping oid %u, userid %u)", - entry->gate.conn, server->servername, user->umid, user->userid); - } + elog( + DEBUG3, + "new pg_clickhouse connection %p for server \"%s\" (user mapping oid %u, " + "userid %u)", + entry->gate.conn, + server->servername, + user->umid, + user->userid + ); + } - return entry->gate; + return entry->gate; } /* @@ -187,78 +197,64 @@ chfdw_get_connection(UserMapping * user) * individual option values, but it seems too much effort for the gain. */ static void -chfdw_inval_callback(Datum arg, int cacheid, uint32 hashvalue) -{ - HASH_SEQ_STATUS scan; - ConnCacheEntry *entry; +chfdw_inval_callback(Datum arg, int cacheid, uint32 hashvalue) { + HASH_SEQ_STATUS scan; + ConnCacheEntry* entry; - Assert(cacheid == FOREIGNSERVEROID || cacheid == USERMAPPINGOID); + Assert(cacheid == FOREIGNSERVEROID || cacheid == USERMAPPINGOID); - /* ConnectionHash must exist already, if we're registered */ - hash_seq_init(&scan, ConnectionHash); - while ((entry = (ConnCacheEntry *) hash_seq_search(&scan))) - { - /* Ignore empty entries */ - if (entry->gate.conn == NULL) - continue; + /* ConnectionHash must exist already, if we're registered */ + hash_seq_init(&scan, ConnectionHash); + while ((entry = (ConnCacheEntry*)hash_seq_search(&scan))) { + /* Ignore empty entries */ + if (entry->gate.conn == NULL) { + continue; + } - /* hashvalue == 0 means a cache reset, must clear all state */ - if (hashvalue == 0 || - (cacheid == FOREIGNSERVEROID && - entry->server_hashvalue == hashvalue) || - (cacheid == USERMAPPINGOID && - entry->mapping_hashvalue == hashvalue)) - { - entry->invalidated = true; - } - } + /* hashvalue == 0 means a cache reset, must clear all state */ + if (hashvalue == 0 || + (cacheid == FOREIGNSERVEROID && entry->server_hashvalue == hashvalue) || + (cacheid == USERMAPPINGOID && entry->mapping_hashvalue == hashvalue)) { + entry->invalidated = true; + } + } } -ch_connection_details * -connstring_parse(const char *connstring) -{ - ListCell *lc; - List *options = chfdw_parse_options(connstring, false, true); - ch_connection_details *details = palloc0(sizeof(ch_connection_details)); +ch_connection_details* +connstring_parse(const char* connstring) { + ListCell* lc; + List* options = chfdw_parse_options(connstring, false, true); + ch_connection_details* details = palloc0(sizeof(ch_connection_details)); - if (options == NIL) - return details; + if (options == NIL) { + return details; + } - foreach(lc, options) - { - DefElem *elem = (DefElem *) lfirst(lc); - char *pname = elem->defname; - char *pval = strVal(elem->arg); + foreach (lc, options) { + DefElem* elem = (DefElem*)lfirst(lc); + char* pname = elem->defname; + char* pval = strVal(elem->arg); - if (strcmp(pname, "host") == 0) - { - details->host = pval; - } - else if (strcmp(pname, "port") == 0) - { - details->port = pg_strtoint32(pval); - } - else if (strcmp(pname, "username") == 0) - { - details->username = pval; - } - else if (strcmp(pname, "password") == 0) - { - details->password = pval; - } - else if (strcmp(pname, "dbname") == 0) - { - details->dbname = pval; - } - else if (strcmp(pname, "") != 0) - { - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("pg_clickhouse: invalid connection option \"%s\"", pname))); - } - } + if (strcmp(pname, "host") == 0) { + details->host = pval; + } else if (strcmp(pname, "port") == 0) { + details->port = pg_strtoint32(pval); + } else if (strcmp(pname, "username") == 0) { + details->username = pval; + } else if (strcmp(pname, "password") == 0) { + details->password = pval; + } else if (strcmp(pname, "dbname") == 0) { + details->dbname = pval; + } else if (strcmp(pname, "") != 0) { + ereport( + ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("pg_clickhouse: invalid connection option \"%s\"", pname)) + ); + } + } - list_free(options); + list_free(options); - return details; + return details; } diff --git a/src/custom_types.c b/src/custom_types.c index 1813afb..0a6308d 100644 --- a/src/custom_types.c +++ b/src/custom_types.c @@ -1,5 +1,5 @@ #include "postgres.h" -#include "strings.h" + #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" @@ -10,6 +10,7 @@ #include "commands/defrem.h" #include "commands/extension.h" #include "parser/parse_func.h" +#include "strings.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" @@ -217,7 +218,8 @@ #define F_ARRAY_SORT_ANYARRAY 6388 #define F_ARRAY_SORT_ANYARRAY_BOOL 6389 #define F_ARRAY_SORT_ANYARRAY_BOOL_BOOL 6390 -/* before PG 18 reverse(text) was unique so macro was F_REVERSE; reverse(bytea) didn't exist */ +/* before PG 18 reverse(text) was unique so macro was F_REVERSE; reverse(bytea) didn't + * exist */ #define F_REVERSE_TEXT F_REVERSE #define F_REVERSE_BYTEA 6382 #endif @@ -225,61 +227,57 @@ #define STR_STARTS_WITH(str, sub) strncmp(str, sub, strlen(sub)) == 0 #define STR_EQUAL(a, b) strcmp(a, b) == 0 -static HTAB * custom_objects_cache = NULL; -static HTAB -* custom_columns_cache = NULL; +static HTAB* custom_objects_cache = NULL; +static HTAB* custom_columns_cache = NULL; -static HTAB * -create_custom_objects_cache(void) -{ - HASHCTL ctl; +static HTAB* +create_custom_objects_cache(void) { + HASHCTL ctl; - ctl.keysize = sizeof(Oid); - ctl.entrysize = sizeof(CustomObjectDef); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(CustomObjectDef); - return hash_create("pg_clickhouse custom functions", 20, &ctl, HASH_ELEM | HASH_BLOBS); + return hash_create( + "pg_clickhouse custom functions", 20, &ctl, HASH_ELEM | HASH_BLOBS + ); } static void -invalidate_custom_columns_cache(Datum arg, int cacheid, uint32 hashvalue) -{ - HASH_SEQ_STATUS status; - CustomColumnInfo *entry; - - hash_seq_init(&status, custom_columns_cache); - while ((entry = (CustomColumnInfo *) hash_seq_search(&status)) != NULL) - { - if (hash_search(custom_columns_cache, - (void *) &entry->relid, - HASH_REMOVE, - NULL) == NULL) - elog(ERROR, "hash table corrupted"); - } +invalidate_custom_columns_cache(Datum arg, int cacheid, uint32 hashvalue) { + HASH_SEQ_STATUS status; + CustomColumnInfo* entry; + + hash_seq_init(&status, custom_columns_cache); + while ((entry = (CustomColumnInfo*)hash_seq_search(&status)) != NULL) { + if (hash_search( + custom_columns_cache, (void*)&entry->relid, HASH_REMOVE, NULL + ) == NULL) { + elog(ERROR, "hash table corrupted"); + } + } } -static HTAB * -create_custom_columns_cache(void) -{ - HASHCTL ctl; +static HTAB* +create_custom_columns_cache(void) { + HASHCTL ctl; - ctl.keysize = sizeof(Oid) + sizeof(int); - ctl.entrysize = sizeof(CustomColumnInfo); + ctl.keysize = sizeof(Oid) + sizeof(int); + ctl.entrysize = sizeof(CustomColumnInfo); - CacheRegisterSyscacheCallback(ATTNUM, - invalidate_custom_columns_cache, - (Datum) 0); + CacheRegisterSyscacheCallback(ATTNUM, invalidate_custom_columns_cache, (Datum)0); - return hash_create("pg_clickhouse custom functions", 20, &ctl, HASH_ELEM | HASH_BLOBS); + return hash_create( + "pg_clickhouse custom functions", 20, &ctl, HASH_ELEM | HASH_BLOBS + ); } inline static void -init_custom_entry(CustomObjectDef * entry) -{ - entry->cf_type = CF_USUAL; - entry->custom_name[0] = '\0'; - entry->cf_context = NULL; - entry->rowfunc = InvalidOid; - entry->paren_count = 1; +init_custom_entry(CustomObjectDef* entry) { + entry->cf_type = CF_USUAL; + entry->custom_name[0] = '\0'; + entry->cf_context = NULL; + entry->rowfunc = InvalidOid; + entry->paren_count = 1; } /* @@ -287,95 +285,91 @@ init_custom_entry(CustomObjectDef * entry) * aggregate function. */ inline bool -chfdw_check_for_ordered_aggregate(Aggref * agg) -{ - switch (agg->aggfnoid) - { - case F_PERCENTILE_CONT_FLOAT8_FLOAT8: - case F_PERCENTILE_CONT_FLOAT8_INTERVAL: - /* Ordered aggregates that map to ClickHouse functions. */ - return true; - } - - /* Accept all ordered aggregates defined by pg_clickhouse. */ - Oid extoid = getExtensionOfObject(ProcedureRelationId, agg->aggfnoid); - char *extname = get_extension_name(extoid); - - return STR_EQUAL(extname, "pg_clickhouse"); +chfdw_check_for_ordered_aggregate(Aggref* agg) { + switch (agg->aggfnoid) { + case F_PERCENTILE_CONT_FLOAT8_FLOAT8: + case F_PERCENTILE_CONT_FLOAT8_INTERVAL: + /* Ordered aggregates that map to ClickHouse functions. */ + return true; + } + + /* Accept all ordered aggregates defined by pg_clickhouse. */ + Oid extoid = getExtensionOfObject(ProcedureRelationId, agg->aggfnoid); + char* extname = get_extension_name(extoid); + + return STR_EQUAL(extname, "pg_clickhouse"); } /* * Map sans-prefix pg_re2 function names to ClickHouse * case-sensitive names. Must be kept in lexicographic order. */ -static char *re2_func_map[][2] = { - {"countmatches", "countMatches"}, - {"countmatchescaseinsensitive", "countMatchesCaseInsensitive"}, - {"extractall", "extractAll"}, - {"extractallgroupshorizontal", "extractAllGroupsHorizontal"}, - {"extractallgroupsvertical", "extractAllGroupsVertical"}, - {"extractgroups", "extractGroups"}, - {"multimatchallindices", "multiMatchAllIndices"}, - {"multimatchany", "multiMatchAny"}, - {"multimatchanyindex", "multiMatchAnyIndex"}, - {"regexpextract", "regexpExtract"}, - {"regexpquotemeta", "regexpQuoteMeta"}, - {"replaceregexpall", "replaceRegexpAll"}, - {"replaceregexpone", "replaceRegexpOne"}, - {"splitbyregexp", "splitByRegexp"}, - {NULL, NULL}, +static char* re2_func_map[][2] = { + { "countmatches", "countMatches" }, + { "countmatchescaseinsensitive", "countMatchesCaseInsensitive" }, + { "extractall", "extractAll" }, + { "extractallgroupshorizontal", "extractAllGroupsHorizontal" }, + { "extractallgroupsvertical", "extractAllGroupsVertical" }, + { "extractgroups", "extractGroups" }, + { "multimatchallindices", "multiMatchAllIndices" }, + { "multimatchany", "multiMatchAny" }, + { "multimatchanyindex", "multiMatchAnyIndex" }, + { "regexpextract", "regexpExtract" }, + { "regexpquotemeta", "regexpQuoteMeta" }, + { "replaceregexpall", "replaceRegexpAll" }, + { "replaceregexpone", "replaceRegexpOne" }, + { "splitbyregexp", "splitByRegexp" }, + { NULL, NULL }, }; -inline static char * -re2_func_name(char *proname) -{ - Assert(strncmp(proname, "re2", 3) == 0); - char *stripped = proname + 3; - size_t i = 0; - - while (re2_func_map[i][0] != NULL) - { - if (STR_EQUAL(re2_func_map[i][0], stripped)) - return re2_func_map[i][1]; - i++; - } - return stripped; +inline static char* +re2_func_name(char* proname) { + Assert(strncmp(proname, "re2", 3) == 0); + char* stripped = proname + 3; + size_t i = 0; + + while (re2_func_map[i][0] != NULL) { + if (STR_EQUAL(re2_func_map[i][0], stripped)) { + return re2_func_map[i][1]; + } + i++; + } + return stripped; } /* * Map pg_clickhouse pushdown function names to ClickHouse case-sensitive * names. Must be kept in lexicographic order. */ -static char *ch_func_map[][2] = { - {"argmax", "argMax"}, - {"argmin", "argMin"}, - {"dictget", "dictGet"}, - {"quantileexact", "quantileExact"}, - {"touint128", "toUInt128"}, - {"touint16", "toUInt16"}, - {"touint32", "toUInt32"}, - {"touint64", "toUInt64"}, - {"touint8", "toUInt8"}, - {"uniqcombined", "uniqCombined"}, - {"uniqcombined64", "uniqCombined64"}, - {"uniqexact", "uniqExact"}, - {"uniqhll12", "uniqHLL12"}, - {"uniqtheta", "uniqTheta"}, - {NULL, NULL}, +static char* ch_func_map[][2] = { + { "argmax", "argMax" }, + { "argmin", "argMin" }, + { "dictget", "dictGet" }, + { "quantileexact", "quantileExact" }, + { "touint128", "toUInt128" }, + { "touint16", "toUInt16" }, + { "touint32", "toUInt32" }, + { "touint64", "toUInt64" }, + { "touint8", "toUInt8" }, + { "uniqcombined", "uniqCombined" }, + { "uniqcombined64", "uniqCombined64" }, + { "uniqexact", "uniqExact" }, + { "uniqhll12", "uniqHLL12" }, + { "uniqtheta", "uniqTheta" }, + { NULL, NULL }, }; -inline static char * -ch_func_name(char *proname) -{ - size_t i = 0; - - while (ch_func_map[i][0] != NULL) - { - if (STR_EQUAL(ch_func_map[i][0], proname)) - return ch_func_map[i][1]; - i++; - } - return proname; +inline static char* +ch_func_name(char* proname) { + size_t i = 0; + + while (ch_func_map[i][0] != NULL) { + if (STR_EQUAL(ch_func_map[i][0], proname)) { + return ch_func_map[i][1]; + } + i++; + } + return proname; } /* @@ -384,12 +378,11 @@ ch_func_name(char *proname) * ch_name = NULL: deparse emits PG name * ch_name = "\1": specialized cf_type handler */ -typedef struct -{ - custom_object_type cf_type; - const char *ch_name; - int paren_count; -} builtin_func_def; +typedef struct { + custom_object_type cf_type; + const char* ch_name; + int paren_count; +} builtin_func_def; /* * Classify a builtin function and, when we rewrite it, describe the @@ -399,548 +392,532 @@ typedef struct * untouched keeps the caller-supplied default (PG name, CF_USUAL, 1 paren). */ static bool -lookup_builtin_func(Oid funcid, builtin_func_def * def) -{ - switch (funcid) - { - case F_DATE_TRUNC_TEXT_TIMESTAMPTZ: - case F_DATE_TRUNC_TEXT_TIMESTAMP: - def->cf_type = CF_DATE_TRUNC; - def->ch_name = "\1"; - return true; - case F_DATE_PART_TEXT_TIMESTAMPTZ: - case F_DATE_PART_TEXT_TIMESTAMP: - case F_DATE_PART_TEXT_DATE: - case F_EXTRACT_TEXT_TIMESTAMP: - case F_EXTRACT_TEXT_TIMESTAMPTZ: - case F_EXTRACT_TEXT_DATE: - def->cf_type = CF_DATE_PART; - def->ch_name = "\1"; - return true; - case F_TIMEZONE_TEXT_TIMESTAMP: - case F_TIMEZONE_TEXT_TIMESTAMPTZ: - def->cf_type = CF_TIMEZONE; - def->ch_name = "toTimeZone"; - return true; - case F_ARRAY_POSITION_ANYCOMPATIBLEARRAY_ANYCOMPATIBLE: - def->ch_name = "indexOf"; - return true; - case F_BTRIM_TEXT_TEXT: - case F_BTRIM_TEXT: - def->ch_name = "trimBoth"; - return true; - case F_STRPOS: - def->ch_name = "positionUTF8"; - return true; - /* PG strpos counts code points, CH position counts bytes */ - case F_LOWER_TEXT: - def->ch_name = "lowerUTF8"; - return true; - /* PG lower(text) is locale-aware on code points */ - case F_UPPER_TEXT: - def->ch_name = "upperUTF8"; - return true; - /* PG upper(text) is locale-aware on code points */ - case F_SUBSTR_TEXT_INT4_INT4: - case F_SUBSTR_TEXT_INT4: - case F_SUBSTRING_TEXT_INT4_INT4: - case F_SUBSTRING_TEXT_INT4: - def->ch_name = "substringUTF8"; - return true; - /* PG substring(text, ...) counts code points */ - case F_SUBSTR_BYTEA_INT4_INT4: - case F_SUBSTR_BYTEA_INT4: - case F_SUBSTRING_BYTEA_INT4_INT4: - case F_SUBSTRING_BYTEA_INT4: - def->ch_name = "substring"; - return true; - /* bytea variant is byte-based; CH substring matches */ - case F_REGEXP_LIKE_TEXT_TEXT: - case F_REGEXP_LIKE_TEXT_TEXT_TEXT: - def->cf_type = CF_MATCH; - def->ch_name = "match"; - return true; - case F_REGEXP_MATCH_TEXT_TEXT: - case F_REGEXP_MATCH_TEXT_TEXT_TEXT: - def->cf_type = CF_REGEX_PG_MATCH; - def->ch_name = "\1"; - return true; - case F_REGEXP_SPLIT_TO_ARRAY_TEXT_TEXT: - case F_REGEXP_SPLIT_TO_ARRAY_TEXT_TEXT_TEXT: - def->cf_type = CF_SPLIT_BY_REGEX; - def->ch_name = "splitByRegexp"; - return true; - case F_REGEXP_REPLACE_TEXT_TEXT_TEXT: - case F_REGEXP_REPLACE_TEXT_TEXT_TEXT_TEXT: - def->cf_type = CF_REPLACE_REGEX; - def->ch_name = "\1"; - return true; - case F_PERCENTILE_CONT_FLOAT8_FLOAT8: - case F_PERCENTILE_CONT_FLOAT8_INTERVAL: - def->ch_name = "quantile"; - return true; - case F_ARRAY_AGG_ANYNONARRAY: - def->ch_name = "groupArray"; - return true; - case F_MD5_BYTEA: - case F_MD5_TEXT: - def->ch_name = "lower(hex(MD5"; - def->paren_count = 3; - return true; - /* Special hashing function returns lowercase hex. */ - case F_REVERSE_TEXT: - def->ch_name = "reverseUTF8"; - return true; - /* reverse code points, not bytes */ - case F_LENGTH_TEXT: - def->ch_name = "lengthUTF8"; - return true; - /* PG length(text) counts code points, CH length() counts */ - /* bytes */ - case F_OCTET_LENGTH_TEXT: - case F_OCTET_LENGTH_BPCHAR: - case F_OCTET_LENGTH_BYTEA: - def->ch_name = "length"; - return true; - case F_BIT_AND_INT2: - case F_BIT_AND_INT4: - case F_BIT_AND_INT8: - def->ch_name = "groupBitAnd"; - return true; - case F_BIT_OR_INT2: - case F_BIT_OR_INT4: - case F_BIT_OR_INT8: - def->ch_name = "groupBitOr"; - return true; - case F_BIT_XOR_INT2: - case F_BIT_XOR_INT4: - case F_BIT_XOR_INT8: - def->ch_name = "groupBitXor"; - return true; - case F_BIT_COUNT_BYTEA: - def->ch_name = "bitCount"; - return true; - case F_MOD_INT2_INT2: - case F_MOD_INT4_INT4: - case F_MOD_INT8_INT8: - case F_MOD_NUMERIC_NUMERIC: - def->ch_name = "modulo"; - return true; - case F_POW_FLOAT8_FLOAT8: - case F_POWER_FLOAT8_FLOAT8: - case F_POW_NUMERIC_NUMERIC: - case F_POWER_NUMERIC_NUMERIC: - def->ch_name = "pow"; - return true; - /* CH lacks "power", maps to pow */ - case F_TO_TIMESTAMP_FLOAT8: - def->ch_name = "fromUnixTimestamp(toInt64"; - def->paren_count = 2; - return true; - /* ClickHouse doesn't work with subsecond precision */ - /* timestamps. */ - case F_TO_CHAR_TIMESTAMP_TEXT: - case F_TO_CHAR_TIMESTAMPTZ_TEXT: - def->cf_type = CF_TO_CHAR; - def->ch_name = "formatDateTime"; - return true; - case F_JSONB_EXTRACT_PATH_TEXT: - case F_JSON_EXTRACT_PATH_TEXT: - def->cf_type = CF_JSON_EXTRACT_PATH_TEXT; - def->ch_name = "\1"; - return true; - case F_JSONB_EXTRACT_PATH: - case F_JSON_EXTRACT_PATH: - def->cf_type = CF_JSON_EXTRACT_PATH; - def->ch_name = "\1"; - return true; - case F_NOW: - def->ch_name = "now64"; - return true; - /* Postgres NOW() produces subsecond precision, to map to */ - /* now64() */ - case F_STATEMENT_TIMESTAMP: - case F_TRANSACTION_TIMESTAMP: - case F_CLOCK_TIMESTAMP: - def->cf_type = CF_CLOCK_TIMESTAMP; - def->ch_name = "\1"; - return true; - case F_CURRENT_SCHEMA: - def->cf_type = CF_CURRENT_SCHEMA; - def->ch_name = "\1"; - return true; - case F_CURRENT_DATABASE: - def->cf_type = CF_CURRENT_DATABASE; - def->ch_name = "\1"; - return true; - /* array functions: simple mappings */ - case F_ARRAY_CAT: - def->ch_name = "arrayConcat"; - return true; - case F_ARRAY_APPEND: - def->ch_name = "arrayPushBack"; - return true; - case F_ARRAY_REMOVE: - def->ch_name = "arrayRemove"; - return true; - case F_ARRAY_TO_STRING_ANYARRAY_TEXT: - def->ch_name = "arrayStringConcat"; - return true; - case F_CARDINALITY: - case F_ARRAY_LENGTH: - def->cf_type = CF_ARRAY_LENGTH; - def->ch_name = "length"; - return true; - case F_ARRAY_REVERSE: - def->ch_name = "arrayReverse"; - return true; - case F_ARRAY_SORT_ANYARRAY: - def->ch_name = "arraySort"; - return true; - case F_ARRAY_SHUFFLE: - def->ch_name = "arrayShuffle"; - return true; - case F_ARRAY_SAMPLE: - def->ch_name = "arrayRandomSample"; - return true; - case F_ARRAY_PREPEND: - def->cf_type = CF_ARRAY_PREPEND; - def->ch_name = "arrayPushFront"; - return true; - case F_STRING_TO_ARRAY_TEXT_TEXT: - def->cf_type = CF_STRING_TO_ARRAY; - def->ch_name = "splitByString"; - return true; - case F_SPLIT_PART: - def->cf_type = CF_STRING_TO_ARRAY_PART; - def->ch_name = "splitByString"; - return true; - case F_TRIM_ARRAY: - def->cf_type = CF_TRIM_ARRAY; - def->ch_name = "arrayResize"; - return true; - case F_ARRAY_FILL_ANYELEMENT__INT4: - def->cf_type = CF_ARRAY_FILL; - def->ch_name = "arrayWithConstant"; - return true; - case F_ARRAY_SORT_ANYARRAY_BOOL: - def->cf_type = CF_ARRAY_SORT_DESC; - def->ch_name = "\1"; - return true; - - /* 1:1 pass-through: PG and CH agree on name and semantics */ - case F_ARRAY_AGG_ANYARRAY: - case F_AVG_INT8: - case F_AVG_INT4: - case F_AVG_INT2: - case F_AVG_NUMERIC: - case F_AVG_FLOAT4: - case F_AVG_FLOAT8: - case F_AVG_INTERVAL: - case F_SUM_INT8: - case F_SUM_INT4: - case F_SUM_INT2: - case F_SUM_FLOAT4: - case F_SUM_FLOAT8: - case F_SUM_INTERVAL: - case F_SUM_NUMERIC: - case F_COUNT_ANY: - case F_COUNT_: - case F_MIN_INT8: - case F_MIN_INT4: - case F_MIN_INT2: - case F_MIN_FLOAT4: - case F_MIN_FLOAT8: - case F_MIN_DATE: - case F_MIN_TIMESTAMP: - case F_MIN_TIMESTAMPTZ: - case F_MIN_INTERVAL: - case F_MIN_TEXT: - case F_MIN_NUMERIC: - case F_MIN_BPCHAR: - case F_MAX_INT8: - case F_MAX_INT4: - case F_MAX_INT2: - case F_MAX_FLOAT4: - case F_MAX_FLOAT8: - case F_MAX_DATE: - case F_MAX_TIMESTAMP: - case F_MAX_TIMESTAMPTZ: - case F_MAX_INTERVAL: - case F_MAX_TEXT: - case F_MAX_NUMERIC: - case F_MAX_BPCHAR: - case F_BOOL_AND: - case F_BOOL_OR: - case F_EVERY: - case F_STRING_AGG_TEXT_TEXT: - /* window functions sharing PG and CH names */ - case F_ROW_NUMBER: - case F_RANK_: - case F_DENSE_RANK_: - case F_PERCENT_RANK_: - case F_CUME_DIST_: - case F_NTILE: - /* trig: PG and CH agree on all finite inputs. Skipping */ - /* asin/acos/atanh/acosh because PG errors on out-of-range */ - /* input where CH returns NaN; sin/cos/tan share the same */ - /* error-vs-NaN divergence only at infinity. */ - case F_SIN: - case F_COS: - case F_TAN: - case F_ATAN: - case F_ATAN2: - case F_SINH: - case F_COSH: - case F_TANH: - case F_ASINH: - case F_DEGREES: - case F_RADIANS: - case F_PI: - case F_REVERSE_BYTEA: - /* numeric scalar functions, names match ClickHouse */ - case F_ABS_INT2: - case F_ABS_INT4: - case F_ABS_INT8: - case F_ABS_FLOAT4: - case F_ABS_FLOAT8: - case F_ABS_NUMERIC: - case F_ROUND_FLOAT8: - case F_ROUND_NUMERIC: - case F_ROUND_NUMERIC_INT4: - case F_FACTORIAL: - /* string functions: CH ltrim/rtrim/concat_ws are aliases */ - case F_LTRIM_TEXT: - case F_RTRIM_TEXT: - case F_CONCAT_WS: - case F_LENGTH_BYTEA: - /* date(ts), date(tstz) deparse as CH date() (alias toDate) */ - case F_DATE_TIMESTAMP: - case F_DATE_TIMESTAMPTZ: - /* window functions: lead/lag share PG and CH names */ - case F_LEAD_ANYELEMENT: - case F_LEAD_ANYELEMENT_INT4: - case F_LEAD_ANYCOMPATIBLE_INT4_ANYCOMPATIBLE: - case F_LAG_ANYELEMENT: - case F_LAG_ANYELEMENT_INT4: - case F_LAG_ANYCOMPATIBLE_INT4_ANYCOMPATIBLE: - return true; - default: - return false; - } +lookup_builtin_func(Oid funcid, builtin_func_def* def) { + switch (funcid) { + case F_DATE_TRUNC_TEXT_TIMESTAMPTZ: + case F_DATE_TRUNC_TEXT_TIMESTAMP: + def->cf_type = CF_DATE_TRUNC; + def->ch_name = "\1"; + return true; + case F_DATE_PART_TEXT_TIMESTAMPTZ: + case F_DATE_PART_TEXT_TIMESTAMP: + case F_DATE_PART_TEXT_DATE: + case F_EXTRACT_TEXT_TIMESTAMP: + case F_EXTRACT_TEXT_TIMESTAMPTZ: + case F_EXTRACT_TEXT_DATE: + def->cf_type = CF_DATE_PART; + def->ch_name = "\1"; + return true; + case F_TIMEZONE_TEXT_TIMESTAMP: + case F_TIMEZONE_TEXT_TIMESTAMPTZ: + def->cf_type = CF_TIMEZONE; + def->ch_name = "toTimeZone"; + return true; + case F_ARRAY_POSITION_ANYCOMPATIBLEARRAY_ANYCOMPATIBLE: + def->ch_name = "indexOf"; + return true; + case F_BTRIM_TEXT_TEXT: + case F_BTRIM_TEXT: + def->ch_name = "trimBoth"; + return true; + case F_STRPOS: + def->ch_name = "positionUTF8"; + return true; + /* PG strpos counts code points, CH position counts bytes */ + case F_LOWER_TEXT: + def->ch_name = "lowerUTF8"; + return true; + /* PG lower(text) is locale-aware on code points */ + case F_UPPER_TEXT: + def->ch_name = "upperUTF8"; + return true; + /* PG upper(text) is locale-aware on code points */ + case F_SUBSTR_TEXT_INT4_INT4: + case F_SUBSTR_TEXT_INT4: + case F_SUBSTRING_TEXT_INT4_INT4: + case F_SUBSTRING_TEXT_INT4: + def->ch_name = "substringUTF8"; + return true; + /* PG substring(text, ...) counts code points */ + case F_SUBSTR_BYTEA_INT4_INT4: + case F_SUBSTR_BYTEA_INT4: + case F_SUBSTRING_BYTEA_INT4_INT4: + case F_SUBSTRING_BYTEA_INT4: + def->ch_name = "substring"; + return true; + /* bytea variant is byte-based; CH substring matches */ + case F_REGEXP_LIKE_TEXT_TEXT: + case F_REGEXP_LIKE_TEXT_TEXT_TEXT: + def->cf_type = CF_MATCH; + def->ch_name = "match"; + return true; + case F_REGEXP_MATCH_TEXT_TEXT: + case F_REGEXP_MATCH_TEXT_TEXT_TEXT: + def->cf_type = CF_REGEX_PG_MATCH; + def->ch_name = "\1"; + return true; + case F_REGEXP_SPLIT_TO_ARRAY_TEXT_TEXT: + case F_REGEXP_SPLIT_TO_ARRAY_TEXT_TEXT_TEXT: + def->cf_type = CF_SPLIT_BY_REGEX; + def->ch_name = "splitByRegexp"; + return true; + case F_REGEXP_REPLACE_TEXT_TEXT_TEXT: + case F_REGEXP_REPLACE_TEXT_TEXT_TEXT_TEXT: + def->cf_type = CF_REPLACE_REGEX; + def->ch_name = "\1"; + return true; + case F_PERCENTILE_CONT_FLOAT8_FLOAT8: + case F_PERCENTILE_CONT_FLOAT8_INTERVAL: + def->ch_name = "quantile"; + return true; + case F_ARRAY_AGG_ANYNONARRAY: + def->ch_name = "groupArray"; + return true; + case F_MD5_BYTEA: + case F_MD5_TEXT: + def->ch_name = "lower(hex(MD5"; + def->paren_count = 3; + return true; + /* Special hashing function returns lowercase hex. */ + case F_REVERSE_TEXT: + def->ch_name = "reverseUTF8"; + return true; + /* reverse code points, not bytes */ + case F_LENGTH_TEXT: + def->ch_name = "lengthUTF8"; + return true; + /* PG length(text) counts code points, CH length() counts */ + /* bytes */ + case F_OCTET_LENGTH_TEXT: + case F_OCTET_LENGTH_BPCHAR: + case F_OCTET_LENGTH_BYTEA: + def->ch_name = "length"; + return true; + case F_BIT_AND_INT2: + case F_BIT_AND_INT4: + case F_BIT_AND_INT8: + def->ch_name = "groupBitAnd"; + return true; + case F_BIT_OR_INT2: + case F_BIT_OR_INT4: + case F_BIT_OR_INT8: + def->ch_name = "groupBitOr"; + return true; + case F_BIT_XOR_INT2: + case F_BIT_XOR_INT4: + case F_BIT_XOR_INT8: + def->ch_name = "groupBitXor"; + return true; + case F_BIT_COUNT_BYTEA: + def->ch_name = "bitCount"; + return true; + case F_MOD_INT2_INT2: + case F_MOD_INT4_INT4: + case F_MOD_INT8_INT8: + case F_MOD_NUMERIC_NUMERIC: + def->ch_name = "modulo"; + return true; + case F_POW_FLOAT8_FLOAT8: + case F_POWER_FLOAT8_FLOAT8: + case F_POW_NUMERIC_NUMERIC: + case F_POWER_NUMERIC_NUMERIC: + def->ch_name = "pow"; + return true; + /* CH lacks "power", maps to pow */ + case F_TO_TIMESTAMP_FLOAT8: + def->ch_name = "fromUnixTimestamp(toInt64"; + def->paren_count = 2; + return true; + /* ClickHouse doesn't work with subsecond precision */ + /* timestamps. */ + case F_TO_CHAR_TIMESTAMP_TEXT: + case F_TO_CHAR_TIMESTAMPTZ_TEXT: + def->cf_type = CF_TO_CHAR; + def->ch_name = "formatDateTime"; + return true; + case F_JSONB_EXTRACT_PATH_TEXT: + case F_JSON_EXTRACT_PATH_TEXT: + def->cf_type = CF_JSON_EXTRACT_PATH_TEXT; + def->ch_name = "\1"; + return true; + case F_JSONB_EXTRACT_PATH: + case F_JSON_EXTRACT_PATH: + def->cf_type = CF_JSON_EXTRACT_PATH; + def->ch_name = "\1"; + return true; + case F_NOW: + def->ch_name = "now64"; + return true; + /* Postgres NOW() produces subsecond precision, to map to */ + /* now64() */ + case F_STATEMENT_TIMESTAMP: + case F_TRANSACTION_TIMESTAMP: + case F_CLOCK_TIMESTAMP: + def->cf_type = CF_CLOCK_TIMESTAMP; + def->ch_name = "\1"; + return true; + case F_CURRENT_SCHEMA: + def->cf_type = CF_CURRENT_SCHEMA; + def->ch_name = "\1"; + return true; + case F_CURRENT_DATABASE: + def->cf_type = CF_CURRENT_DATABASE; + def->ch_name = "\1"; + return true; + /* array functions: simple mappings */ + case F_ARRAY_CAT: + def->ch_name = "arrayConcat"; + return true; + case F_ARRAY_APPEND: + def->ch_name = "arrayPushBack"; + return true; + case F_ARRAY_REMOVE: + def->ch_name = "arrayRemove"; + return true; + case F_ARRAY_TO_STRING_ANYARRAY_TEXT: + def->ch_name = "arrayStringConcat"; + return true; + case F_CARDINALITY: + case F_ARRAY_LENGTH: + def->cf_type = CF_ARRAY_LENGTH; + def->ch_name = "length"; + return true; + case F_ARRAY_REVERSE: + def->ch_name = "arrayReverse"; + return true; + case F_ARRAY_SORT_ANYARRAY: + def->ch_name = "arraySort"; + return true; + case F_ARRAY_SHUFFLE: + def->ch_name = "arrayShuffle"; + return true; + case F_ARRAY_SAMPLE: + def->ch_name = "arrayRandomSample"; + return true; + case F_ARRAY_PREPEND: + def->cf_type = CF_ARRAY_PREPEND; + def->ch_name = "arrayPushFront"; + return true; + case F_STRING_TO_ARRAY_TEXT_TEXT: + def->cf_type = CF_STRING_TO_ARRAY; + def->ch_name = "splitByString"; + return true; + case F_SPLIT_PART: + def->cf_type = CF_STRING_TO_ARRAY_PART; + def->ch_name = "splitByString"; + return true; + case F_TRIM_ARRAY: + def->cf_type = CF_TRIM_ARRAY; + def->ch_name = "arrayResize"; + return true; + case F_ARRAY_FILL_ANYELEMENT__INT4: + def->cf_type = CF_ARRAY_FILL; + def->ch_name = "arrayWithConstant"; + return true; + case F_ARRAY_SORT_ANYARRAY_BOOL: + def->cf_type = CF_ARRAY_SORT_DESC; + def->ch_name = "\1"; + return true; + + /* 1:1 pass-through: PG and CH agree on name and semantics */ + case F_ARRAY_AGG_ANYARRAY: + case F_AVG_INT8: + case F_AVG_INT4: + case F_AVG_INT2: + case F_AVG_NUMERIC: + case F_AVG_FLOAT4: + case F_AVG_FLOAT8: + case F_AVG_INTERVAL: + case F_SUM_INT8: + case F_SUM_INT4: + case F_SUM_INT2: + case F_SUM_FLOAT4: + case F_SUM_FLOAT8: + case F_SUM_INTERVAL: + case F_SUM_NUMERIC: + case F_COUNT_ANY: + case F_COUNT_: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_BOOL_AND: + case F_BOOL_OR: + case F_EVERY: + case F_STRING_AGG_TEXT_TEXT: + /* window functions sharing PG and CH names */ + case F_ROW_NUMBER: + case F_RANK_: + case F_DENSE_RANK_: + case F_PERCENT_RANK_: + case F_CUME_DIST_: + case F_NTILE: + /* trig: PG and CH agree on all finite inputs. Skipping */ + /* asin/acos/atanh/acosh because PG errors on out-of-range */ + /* input where CH returns NaN; sin/cos/tan share the same */ + /* error-vs-NaN divergence only at infinity. */ + case F_SIN: + case F_COS: + case F_TAN: + case F_ATAN: + case F_ATAN2: + case F_SINH: + case F_COSH: + case F_TANH: + case F_ASINH: + case F_DEGREES: + case F_RADIANS: + case F_PI: + case F_REVERSE_BYTEA: + /* numeric scalar functions, names match ClickHouse */ + case F_ABS_INT2: + case F_ABS_INT4: + case F_ABS_INT8: + case F_ABS_FLOAT4: + case F_ABS_FLOAT8: + case F_ABS_NUMERIC: + case F_ROUND_FLOAT8: + case F_ROUND_NUMERIC: + case F_ROUND_NUMERIC_INT4: + case F_FACTORIAL: + /* string functions: CH ltrim/rtrim/concat_ws are aliases */ + case F_LTRIM_TEXT: + case F_RTRIM_TEXT: + case F_CONCAT_WS: + case F_LENGTH_BYTEA: + /* date(ts), date(tstz) deparse as CH date() (alias toDate) */ + case F_DATE_TIMESTAMP: + case F_DATE_TIMESTAMPTZ: + /* window functions: lead/lag share PG and CH names */ + case F_LEAD_ANYELEMENT: + case F_LEAD_ANYELEMENT_INT4: + case F_LEAD_ANYCOMPATIBLE_INT4_ANYCOMPATIBLE: + case F_LAG_ANYELEMENT: + case F_LAG_ANYELEMENT_INT4: + case F_LAG_ANYCOMPATIBLE_INT4_ANYCOMPATIBLE: + return true; + default: + return false; + } } -CustomObjectDef * -chfdw_check_for_custom_function(Oid funcid) -{ - CustomObjectDef *entry; - builtin_func_def def = {.paren_count = 1}; - bool is_builtin = chfdw_is_builtin(funcid); - - if (is_builtin && !lookup_builtin_func(funcid, &def)) - return NULL; - - if (!custom_objects_cache) - custom_objects_cache = create_custom_objects_cache(); - - entry = hash_search(custom_objects_cache, (void *) &funcid, HASH_FIND, NULL); - if (!entry) - { - Oid extoid; - char *extname; - - entry = hash_search(custom_objects_cache, (void *) &funcid, HASH_ENTER, NULL); - entry->cf_oid = funcid; - init_custom_entry(entry); - - if (is_builtin) - { - entry->cf_type = def.cf_type; - entry->paren_count = def.paren_count; - if (def.ch_name) - strlcpy(entry->custom_name, def.ch_name, NAMEDATALEN); - return entry; - } - - extoid = getExtensionOfObject(ProcedureRelationId, funcid); - extname = get_extension_name(extoid); - if (extname) - { - HeapTuple proctup; - Form_pg_proc procform; - char *proname; - - proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); - if (!HeapTupleIsValid(proctup)) - elog(ERROR, "cache lookup failed for function %u", funcid); - - procform = (Form_pg_proc) GETSTRUCT(proctup); - proname = NameStr(procform->proname); - - if (STR_EQUAL(extname, "intarray")) - { - if (STR_EQUAL(proname, "idx")) - { - entry->cf_type = CF_INTARRAY_IDX; - strcpy(entry->custom_name, "indexOf"); - } - } - else if (STR_EQUAL(extname, "re2")) - { - /* pg_re2: 1:1 pushdown to ClickHouse RE2 functions. */ - entry->cf_type = CF_CH_FUNCTION; - strlcpy(entry->custom_name, re2_func_name(proname), NAMEDATALEN); - } - else if (STR_EQUAL(extname, "fuzzystrmatch")) - { - if (STR_EQUAL(proname, "levenshtein") && - procform->pronargs == 2) - strcpy(entry->custom_name, "editDistanceUTF8"); - else if (!(STR_EQUAL(proname, "soundex"))) - { - ReleaseSysCache(proctup); - pfree(extname); - hash_search(custom_objects_cache, (void *) &funcid, HASH_REMOVE, NULL); - return NULL; - } - } - else if (STR_EQUAL(extname, "pg_clickhouse")) - { - /* pg_clickhouse custom functions. */ - entry->cf_type = CF_CH_FUNCTION; - strlcpy(entry->custom_name, ch_func_name(proname), NAMEDATALEN); - } - ReleaseSysCache(proctup); - pfree(extname); - } - } - - return entry; +CustomObjectDef* +chfdw_check_for_custom_function(Oid funcid) { + CustomObjectDef* entry; + builtin_func_def def = { .paren_count = 1 }; + bool is_builtin = chfdw_is_builtin(funcid); + + if (is_builtin && !lookup_builtin_func(funcid, &def)) { + return NULL; + } + + if (!custom_objects_cache) { + custom_objects_cache = create_custom_objects_cache(); + } + + entry = hash_search(custom_objects_cache, (void*)&funcid, HASH_FIND, NULL); + if (!entry) { + Oid extoid; + char* extname; + + entry = hash_search(custom_objects_cache, (void*)&funcid, HASH_ENTER, NULL); + entry->cf_oid = funcid; + init_custom_entry(entry); + + if (is_builtin) { + entry->cf_type = def.cf_type; + entry->paren_count = def.paren_count; + if (def.ch_name) { + strlcpy(entry->custom_name, def.ch_name, NAMEDATALEN); + } + return entry; + } + + extoid = getExtensionOfObject(ProcedureRelationId, funcid); + extname = get_extension_name(extoid); + if (extname) { + HeapTuple proctup; + Form_pg_proc procform; + char* proname; + + proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(proctup)) { + elog(ERROR, "cache lookup failed for function %u", funcid); + } + + procform = (Form_pg_proc)GETSTRUCT(proctup); + proname = NameStr(procform->proname); + + if (STR_EQUAL(extname, "intarray")) { + if (STR_EQUAL(proname, "idx")) { + entry->cf_type = CF_INTARRAY_IDX; + strcpy(entry->custom_name, "indexOf"); + } + } else if (STR_EQUAL(extname, "re2")) { + /* pg_re2: 1:1 pushdown to ClickHouse RE2 functions. */ + entry->cf_type = CF_CH_FUNCTION; + strlcpy(entry->custom_name, re2_func_name(proname), NAMEDATALEN); + } else if (STR_EQUAL(extname, "fuzzystrmatch")) { + if (STR_EQUAL(proname, "levenshtein") && procform->pronargs == 2) { + strcpy(entry->custom_name, "editDistanceUTF8"); + } else if (!(STR_EQUAL(proname, "soundex"))) { + ReleaseSysCache(proctup); + pfree(extname); + hash_search( + custom_objects_cache, (void*)&funcid, HASH_REMOVE, NULL + ); + return NULL; + } + } else if (STR_EQUAL(extname, "pg_clickhouse")) { + /* pg_clickhouse custom functions. */ + entry->cf_type = CF_CH_FUNCTION; + strlcpy(entry->custom_name, ch_func_name(proname), NAMEDATALEN); + } + ReleaseSysCache(proctup); + pfree(extname); + } + } + + return entry; } -CustomObjectDef * -chfdw_check_for_custom_type(Oid typeoid) -{ - CustomObjectDef *entry; +CustomObjectDef* +chfdw_check_for_custom_type(Oid typeoid) { + CustomObjectDef* entry; - if (!custom_objects_cache) - custom_objects_cache = create_custom_objects_cache(); + if (!custom_objects_cache) { + custom_objects_cache = create_custom_objects_cache(); + } - if (chfdw_is_builtin(typeoid)) - return NULL; + if (chfdw_is_builtin(typeoid)) { + return NULL; + } - entry = hash_search(custom_objects_cache, (void *) &typeoid, HASH_FIND, NULL); - if (!entry) - { - entry = hash_search(custom_objects_cache, (void *) &typeoid, HASH_ENTER, NULL); - init_custom_entry(entry); - } + entry = hash_search(custom_objects_cache, (void*)&typeoid, HASH_FIND, NULL); + if (!entry) { + entry = hash_search(custom_objects_cache, (void*)&typeoid, HASH_ENTER, NULL); + init_custom_entry(entry); + } - return entry; + return entry; } /* pg_operator_d.h only has oid_symbol for some operators */ -#define OID_TEXT_REGEX_NE_OP 642 -#define OID_TEXT_IREGEX_NE_OP 1229 -#define OID_JSONB_FETCHVAL_OP 3211 -#define OID_JSONB_FETCHVAL_TEXT_OP 3477 -#define OID_JSON_FETCHVAL_OP 3962 -#define OID_JSON_FETCHVAL_TEXT_OP 3963 +#define OID_TEXT_REGEX_NE_OP 642 +#define OID_TEXT_IREGEX_NE_OP 1229 +#define OID_JSONB_FETCHVAL_OP 3211 +#define OID_JSONB_FETCHVAL_TEXT_OP 3477 +#define OID_JSON_FETCHVAL_OP 3962 +#define OID_JSON_FETCHVAL_TEXT_OP 3963 /* * Map a builtin operator OID to its custom_object_type. Returns CF_USUAL * when the operator needs no special handling. */ static custom_object_type -classify_builtin_operator(Oid opoid) -{ - switch (opoid) - { - case OID_TEXT_REGEXEQ_OP: - return CF_REGEX_MATCH; - case OID_TEXT_REGEX_NE_OP: - return CF_REGEX_NO_MATCH; - case OID_TEXT_ICREGEXEQ_OP: - return CF_REGEX_ICASE_MATCH; - case OID_TEXT_IREGEX_NE_OP: - return CF_REGEX_ICASE_NO_MATCH; - case OID_JSONB_FETCHVAL_OP: - case OID_JSON_FETCHVAL_OP: - return CF_JSON_FETCHVAL; - case OID_JSONB_FETCHVAL_TEXT_OP: - case OID_JSON_FETCHVAL_TEXT_OP: - return CF_JSON_FETCHVAL_TEXT; - case OID_ARRAY_CONTAINS_OP: - return CF_ARRAY_CONTAINS; - case OID_ARRAY_CONTAINED_OP: - return CF_ARRAY_CONTAINED_BY; - case OID_ARRAY_OVERLAP_OP: - return CF_ARRAY_OVERLAP; - default: - return CF_USUAL; - } +classify_builtin_operator(Oid opoid) { + switch (opoid) { + case OID_TEXT_REGEXEQ_OP: + return CF_REGEX_MATCH; + case OID_TEXT_REGEX_NE_OP: + return CF_REGEX_NO_MATCH; + case OID_TEXT_ICREGEXEQ_OP: + return CF_REGEX_ICASE_MATCH; + case OID_TEXT_IREGEX_NE_OP: + return CF_REGEX_ICASE_NO_MATCH; + case OID_JSONB_FETCHVAL_OP: + case OID_JSON_FETCHVAL_OP: + return CF_JSON_FETCHVAL; + case OID_JSONB_FETCHVAL_TEXT_OP: + case OID_JSON_FETCHVAL_TEXT_OP: + return CF_JSON_FETCHVAL_TEXT; + case OID_ARRAY_CONTAINS_OP: + return CF_ARRAY_CONTAINS; + case OID_ARRAY_CONTAINED_OP: + return CF_ARRAY_CONTAINED_BY; + case OID_ARRAY_OVERLAP_OP: + return CF_ARRAY_OVERLAP; + default: + return CF_USUAL; + } } -CustomObjectDef * -chfdw_check_for_custom_operator(Oid opoid, Form_pg_operator form) -{ - HeapTuple tuple = NULL; - custom_object_type ctype; - - CustomObjectDef *entry; - - if (!custom_objects_cache) - custom_objects_cache = create_custom_objects_cache(); - - if (chfdw_is_builtin(opoid)) - { - ctype = classify_builtin_operator(opoid); - if (ctype == CF_USUAL && opoid != F_TIMESTAMPTZ_PL_INTERVAL) - { - return NULL; - } - } - - if (!form) - { - tuple = SearchSysCache1(OPEROID, ObjectIdGetDatum(opoid)); - if (!HeapTupleIsValid(tuple)) - elog(ERROR, "cache lookup failed for operator %u", opoid); - form = (Form_pg_operator) GETSTRUCT(tuple); - } - - entry = hash_search(custom_objects_cache, (void *) &opoid, HASH_FIND, NULL); - if (!entry) - { - entry = hash_search(custom_objects_cache, (void *) &opoid, HASH_ENTER, NULL); - init_custom_entry(entry); - - ctype = classify_builtin_operator(opoid); - if (opoid == F_TIMESTAMPTZ_PL_INTERVAL) - entry->cf_type = CF_TIMESTAMPTZ_PL_INTERVAL; - else if (ctype != CF_USUAL) - entry->cf_type = ctype; - else - { - Oid extoid = getExtensionOfObject(OperatorRelationId, opoid); - char *extname = get_extension_name(extoid); - - if (extname) - { - if (STR_EQUAL(extname, "hstore")) - { - if (form && strcmp(NameStr(form->oprname), "->") == 0) - entry->cf_type = CF_HSTORE_FETCHVAL; - } - pfree(extname); - } - } - } - - if (tuple) - ReleaseSysCache(tuple); - - return entry; +CustomObjectDef* +chfdw_check_for_custom_operator(Oid opoid, Form_pg_operator form) { + HeapTuple tuple = NULL; + custom_object_type ctype; + + CustomObjectDef* entry; + + if (!custom_objects_cache) { + custom_objects_cache = create_custom_objects_cache(); + } + + if (chfdw_is_builtin(opoid)) { + ctype = classify_builtin_operator(opoid); + if (ctype == CF_USUAL && opoid != F_TIMESTAMPTZ_PL_INTERVAL) { + return NULL; + } + } + + if (!form) { + tuple = SearchSysCache1(OPEROID, ObjectIdGetDatum(opoid)); + if (!HeapTupleIsValid(tuple)) { + elog(ERROR, "cache lookup failed for operator %u", opoid); + } + form = (Form_pg_operator)GETSTRUCT(tuple); + } + + entry = hash_search(custom_objects_cache, (void*)&opoid, HASH_FIND, NULL); + if (!entry) { + entry = hash_search(custom_objects_cache, (void*)&opoid, HASH_ENTER, NULL); + init_custom_entry(entry); + + ctype = classify_builtin_operator(opoid); + if (opoid == F_TIMESTAMPTZ_PL_INTERVAL) { + entry->cf_type = CF_TIMESTAMPTZ_PL_INTERVAL; + } else if (ctype != CF_USUAL) { + entry->cf_type = ctype; + } else { + Oid extoid = getExtensionOfObject(OperatorRelationId, opoid); + char* extname = get_extension_name(extoid); + + if (extname) { + if (STR_EQUAL(extname, "hstore")) { + if (form && strcmp(NameStr(form->oprname), "->") == 0) { + entry->cf_type = CF_HSTORE_FETCHVAL; + } + } + pfree(extname); + } + } + } + + if (tuple) { + ReleaseSysCache(tuple); + } + + return entry; } /* @@ -949,78 +926,85 @@ chfdw_check_for_custom_operator(Oid opoid, Form_pg_operator form) * New options might also require tweaking merge_fdw_options(). */ static void - populate_custom_column_entry(CustomColumnInfo * entry, Oid relid, - AttrNumber attnum, const char *attname, - CHRemoteTableEngine table_engine); +populate_custom_column_entry( + CustomColumnInfo* entry, + Oid relid, + AttrNumber attnum, + const char* attname, + CHRemoteTableEngine table_engine +); void -chfdw_apply_custom_table_options(CHFdwRelationInfo * fpinfo, Oid relid) -{ - ListCell *lc; - TupleDesc tupdesc; - int attnum; - Relation rel; - - foreach(lc, fpinfo->table->options) - { - DefElem *def = (DefElem *) lfirst(lc); - - if (STR_EQUAL(def->defname, "engine")) - { - static char *collapsing_text = "collapsingmergetree", - *aggregating_text = "aggregatingmergetree"; - - char *val = defGetString(def); - - if (strncasecmp(val, collapsing_text, strlen(collapsing_text)) == 0) - { - char *start = index(val, '('), - *end = rindex(val, ')'); - char sign[CH_ESCAPED_NAMEDATALEN]; - - if (end - start - 1 > (CH_ESCAPED_NAMEDATALEN - 1)) - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), - errmsg("pg_clickhouse: invalid engine parameter"))); - - fpinfo->ch_table_engine = CH_COLLAPSING_MERGE_TREE; - strncpy(sign, start + 1, end - start - 1); - sign[end - start - 1] = '\0'; - strlcpy(fpinfo->ch_table_sign_field, ch_quote_ident(sign), CH_ESCAPED_NAMEDATALEN); - } - else if (strncasecmp(val, aggregating_text, strlen(aggregating_text)) == 0) - { - fpinfo->ch_table_engine = CH_AGGREGATING_MERGE_TREE; - } - } - } - - if (custom_columns_cache == NULL) - custom_columns_cache = create_custom_columns_cache(); - - rel = table_open_compat(relid, NoLock); - tupdesc = RelationGetDescr(rel); - - for (attnum = 1; attnum <= tupdesc->natts; attnum++) - { - bool found; - CustomColumnInfo entry_key, - *entry; - Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1); - - entry_key.relid = relid; - entry_key.varattno = attnum; - - entry = hash_search(custom_columns_cache, - (void *) &entry_key.relid, HASH_ENTER, &found); - if (found) - continue; - - populate_custom_column_entry(entry, relid, attnum, - NameStr(attr->attname), - fpinfo->ch_table_engine); - } - table_close_compat(rel, NoLock); +chfdw_apply_custom_table_options(CHFdwRelationInfo* fpinfo, Oid relid) { + ListCell* lc; + TupleDesc tupdesc; + int attnum; + Relation rel; + + foreach (lc, fpinfo->table->options) { + DefElem* def = (DefElem*)lfirst(lc); + + if (STR_EQUAL(def->defname, "engine")) { + static char *collapsing_text = "collapsingmergetree", + *aggregating_text = "aggregatingmergetree"; + + char* val = defGetString(def); + + if (strncasecmp(val, collapsing_text, strlen(collapsing_text)) == 0) { + char *start = index(val, '('), *end = rindex(val, ')'); + char sign[CH_ESCAPED_NAMEDATALEN]; + + if (end - start - 1 > (CH_ESCAPED_NAMEDATALEN - 1)) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg("pg_clickhouse: invalid engine parameter")) + ); + } + + fpinfo->ch_table_engine = CH_COLLAPSING_MERGE_TREE; + strncpy(sign, start + 1, end - start - 1); + sign[end - start - 1] = '\0'; + strlcpy( + fpinfo->ch_table_sign_field, + ch_quote_ident(sign), + CH_ESCAPED_NAMEDATALEN + ); + } else if ( + strncasecmp(val, aggregating_text, strlen(aggregating_text)) == 0 + ) { + fpinfo->ch_table_engine = CH_AGGREGATING_MERGE_TREE; + } + } + } + + if (custom_columns_cache == NULL) { + custom_columns_cache = create_custom_columns_cache(); + } + + rel = table_open_compat(relid, NoLock); + tupdesc = RelationGetDescr(rel); + + for (attnum = 1; attnum <= tupdesc->natts; attnum++) { + bool found; + CustomColumnInfo entry_key, *entry; + Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1); + + entry_key.relid = relid; + entry_key.varattno = attnum; + + entry = hash_search( + custom_columns_cache, (void*)&entry_key.relid, HASH_ENTER, &found + ); + if (found) { + continue; + } + + populate_custom_column_entry( + entry, relid, attnum, NameStr(attr->attname), fpinfo->ch_table_engine + ); + } + table_close_compat(rel, NoLock); } /* @@ -1028,40 +1012,37 @@ chfdw_apply_custom_table_options(CHFdwRelationInfo * fpinfo, Oid relid) * Caller has already inserted the entry under (relid, attnum). */ static void -populate_custom_column_entry(CustomColumnInfo * entry, Oid relid, - AttrNumber attnum, const char *attname, - CHRemoteTableEngine table_engine) -{ - List *options; - ListCell *lc; - - entry->relid = relid; - entry->varattno = attnum; - entry->table_engine = table_engine; - entry->coltype = CF_USUAL; - entry->is_AggregateFunction = CF_AGGR_USUAL; - strlcpy(entry->colname, attname, NAMEDATALEN); - - /* column_name FDW option overrides attname */ - options = GetForeignColumnOptions(relid, attnum); - foreach(lc, options) - { - DefElem *def = (DefElem *) lfirst(lc); - - if (STR_EQUAL(def->defname, "column_name")) - { - strncpy(entry->colname, defGetString(def), NAMEDATALEN); - entry->colname[NAMEDATALEN - 1] = '\0'; - } - else if (STR_EQUAL(def->defname, "aggregatefunction")) - { - entry->is_AggregateFunction = CF_AGGR_FUNC; - } - else if (STR_EQUAL(def->defname, "simpleaggregatefunction")) - { - entry->is_AggregateFunction = CF_AGGR_SIMPLE; - } - } +populate_custom_column_entry( + CustomColumnInfo* entry, + Oid relid, + AttrNumber attnum, + const char* attname, + CHRemoteTableEngine table_engine +) { + List* options; + ListCell* lc; + + entry->relid = relid; + entry->varattno = attnum; + entry->table_engine = table_engine; + entry->coltype = CF_USUAL; + entry->is_AggregateFunction = CF_AGGR_USUAL; + strlcpy(entry->colname, attname, NAMEDATALEN); + + /* column_name FDW option overrides attname */ + options = GetForeignColumnOptions(relid, attnum); + foreach (lc, options) { + DefElem* def = (DefElem*)lfirst(lc); + + if (STR_EQUAL(def->defname, "column_name")) { + strncpy(entry->colname, defGetString(def), NAMEDATALEN); + entry->colname[NAMEDATALEN - 1] = '\0'; + } else if (STR_EQUAL(def->defname, "aggregatefunction")) { + entry->is_AggregateFunction = CF_AGGR_FUNC; + } else if (STR_EQUAL(def->defname, "simpleaggregatefunction")) { + entry->is_AggregateFunction = CF_AGGR_SIMPLE; + } + } } /* @@ -1070,52 +1051,48 @@ populate_custom_column_entry(CustomColumnInfo * entry, Oid relid, * chfdw_apply_custom_table_options does not run, leaving deparse without * column_name overrides. Populate lazily so any caller sees them. */ -CustomColumnInfo * -chfdw_get_custom_column_info(Oid relid, uint16 varattno) -{ - CustomColumnInfo entry_key, - *entry; - bool found; - - entry_key.relid = relid; - entry_key.varattno = varattno; - - if (custom_columns_cache == NULL) - custom_columns_cache = create_custom_columns_cache(); - - entry = hash_search(custom_columns_cache, - (void *) &entry_key.relid, HASH_ENTER, &found); - if (!found) - { - Relation rel; - TupleDesc tupdesc; - - rel = table_open_compat(relid, NoLock); - tupdesc = RelationGetDescr(rel); - - if (varattno > 0 && varattno <= tupdesc->natts) - { - Form_pg_attribute attr = TupleDescAttr(tupdesc, varattno - 1); - - if (!attr->attisdropped) - populate_custom_column_entry(entry, relid, varattno, - NameStr(attr->attname), - CH_DEFAULT); - else - { - hash_search(custom_columns_cache, - (void *) &entry_key.relid, HASH_REMOVE, NULL); - entry = NULL; - } - } - else - { - hash_search(custom_columns_cache, - (void *) &entry_key.relid, HASH_REMOVE, NULL); - entry = NULL; - } - table_close_compat(rel, NoLock); - } - - return entry; +CustomColumnInfo* +chfdw_get_custom_column_info(Oid relid, uint16 varattno) { + CustomColumnInfo entry_key, *entry; + bool found; + + entry_key.relid = relid; + entry_key.varattno = varattno; + + if (custom_columns_cache == NULL) { + custom_columns_cache = create_custom_columns_cache(); + } + + entry = + hash_search(custom_columns_cache, (void*)&entry_key.relid, HASH_ENTER, &found); + if (!found) { + Relation rel; + TupleDesc tupdesc; + + rel = table_open_compat(relid, NoLock); + tupdesc = RelationGetDescr(rel); + + if (varattno > 0 && varattno <= tupdesc->natts) { + Form_pg_attribute attr = TupleDescAttr(tupdesc, varattno - 1); + + if (!attr->attisdropped) { + populate_custom_column_entry( + entry, relid, varattno, NameStr(attr->attname), CH_DEFAULT + ); + } else { + hash_search( + custom_columns_cache, (void*)&entry_key.relid, HASH_REMOVE, NULL + ); + entry = NULL; + } + } else { + hash_search( + custom_columns_cache, (void*)&entry_key.relid, HASH_REMOVE, NULL + ); + entry = NULL; + } + table_close_compat(rel, NoLock); + } + + return entry; } diff --git a/src/deparse.c b/src/deparse.c index 569f61c..3501268 100644 --- a/src/deparse.c +++ b/src/deparse.c @@ -24,6 +24,7 @@ #include "catalog/pg_type.h" #include "commands/defrem.h" #include "fmgr.h" +#include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" @@ -42,7 +43,6 @@ #include "utils/rel.h" #include "utils/syscache.h" #include "utils/typcache.h" -#include "lib/stringinfo.h" #if PG_VERSION_NUM >= 120000 #include "access/table.h" @@ -79,7 +79,7 @@ #endif #ifndef MAXINT8LEN -#define MAXINT8LEN 25 +#define MAXINT8LEN 25 #endif /* Aggregate OIDs absent from fmgroids.h on all PG versions. */ @@ -94,129 +94,206 @@ static uint32 var_counter = 0; /* * Global context for foreign_expr_walker's search of an expression tree. */ -typedef struct foreign_glob_cxt -{ - PlannerInfo *root; /* global planner state */ - RelOptInfo *foreignrel; /* the foreign relation we are planning for */ - Relids relids; /* relids of base relations in the underlying - * scan */ -} foreign_glob_cxt; +typedef struct foreign_glob_cxt { + PlannerInfo* root; /* global planner state */ + RelOptInfo* foreignrel; /* the foreign relation we are planning for */ + Relids relids; /* relids of base relations in the underlying + * scan */ +} foreign_glob_cxt; /* * Context for deparseExpr */ -typedef struct deparse_expr_cxt -{ - PlannerInfo *root; /* global planner state */ - RelOptInfo *foreignrel; /* the foreign relation we are planning for */ - RelOptInfo *scanrel; /* the underlying scan relation. Same as - * foreignrel, when that represents a join or - * a base relation. */ - StringInfo buf; /* output buffer to append to */ - List **params_list; /* exprs that will become remote Params */ - CustomObjectDef *func; /* custom function deparse */ - CHFdwRelationInfo *fpinfo; /* fdw relation info */ - bool interval_op; - bool array_as_tuple; /* determines array output format */ - bool no_sort_parens; /* determines sort group clause format */ -} deparse_expr_cxt; - -#define REL_ALIAS_PREFIX "r" +typedef struct deparse_expr_cxt { + PlannerInfo* root; /* global planner state */ + RelOptInfo* foreignrel; /* the foreign relation we are planning for */ + RelOptInfo* scanrel; /* the underlying scan relation. Same as + * foreignrel, when that represents a join or + * a base relation. */ + StringInfo buf; /* output buffer to append to */ + List** params_list; /* exprs that will become remote Params */ + CustomObjectDef* func; /* custom function deparse */ + CHFdwRelationInfo* fpinfo; /* fdw relation info */ + bool interval_op; + bool array_as_tuple; /* determines array output format */ + bool no_sort_parens; /* determines sort group clause format */ +} deparse_expr_cxt; + +#define REL_ALIAS_PREFIX "r" /* Handy macro to add relation name qualification */ -#define ADD_REL_QUALIFIER(buf, varno) \ - appendStringInfo((buf), "%s%d.", REL_ALIAS_PREFIX, (varno)) -#define SUBQUERY_REL_ALIAS_PREFIX "s" -#define SUBQUERY_COL_ALIAS_PREFIX "c" - -#define CSTRING_TOLOWER(str) \ -do { \ - for (int i = 0; str[i]; i++) { \ - str[i] = tolower(str[i]); \ - } \ -} while (0) +#define ADD_REL_QUALIFIER(buf, varno) \ + appendStringInfo((buf), "%s%d.", REL_ALIAS_PREFIX, (varno)) +#define SUBQUERY_REL_ALIAS_PREFIX "s" +#define SUBQUERY_COL_ALIAS_PREFIX "c" + +#define CSTRING_TOLOWER(str) \ + do { \ + for (int i = 0; str[i]; i++) { \ + str[i] = tolower(str[i]); \ + } \ + } while (0) /* * Functions to determine whether an expression can be evaluated safely on * remote server. */ -static bool foreign_expr_walker(Node * node, - foreign_glob_cxt * glob_cxt); -static char *deparse_type_name(Oid type_oid, int32 typemod); +static bool +foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt); +static char* +deparse_type_name(Oid type_oid, int32 typemod); /* * Functions to construct string representation of a node tree. */ -static void deparseTargetList(StringInfo buf, - RangeTblEntry * rte, - Index rtindex, - Relation rel, - Bitmapset * attrs_used, - bool qualify_col, - List * *retrieved_attrs); -static void deparseExplicitTargetList(List * tlist, - List * *retrieved_attrs, - deparse_expr_cxt * context); -static void deparseSubqueryTargetList(deparse_expr_cxt * context); -static void deparseColumnRef(StringInfo buf, CustomObjectDef * cdef, - int varno, int varattno, RangeTblEntry * rte, bool qualify_col); -static void deparseRelation(StringInfo buf, Relation rel); -static void deparseExpr(Expr * expr, deparse_expr_cxt * context); -static void deparseVar(Var * node, deparse_expr_cxt * context); -static void deparseConst(Const * node, deparse_expr_cxt * context, int showtype); -static void deparseParam(Param * node, deparse_expr_cxt * context); -static void deparseSubscriptingRef(SubscriptingRef * node, deparse_expr_cxt * context); -static void deparseFuncExpr(FuncExpr * node, deparse_expr_cxt * context); -static void deparseSQLValueFunction(SQLValueFunction * node, deparse_expr_cxt * context); -static void deparseOpExpr(OpExpr * node, deparse_expr_cxt * context); -static void deparseOperatorName(StringInfo buf, Form_pg_operator opform); -static void deparseDistinctExpr(DistinctExpr * node, deparse_expr_cxt * context); -static void deparseScalarArrayOpExpr(ScalarArrayOpExpr * node, - deparse_expr_cxt * context); -static void deparseCaseExpr(CaseExpr * node, deparse_expr_cxt * context); -static void deparseCaseWhen(CaseWhen * node, deparse_expr_cxt * context); -static void deparseRelabelType(RelabelType * node, deparse_expr_cxt * context); -static void deparseBoolExpr(BoolExpr * node, deparse_expr_cxt * context); -static void deparseNullTest(NullTest * node, deparse_expr_cxt * context); -static void deparseArrayExpr(ArrayExpr * node, deparse_expr_cxt * context); -static void deparseArrayList(ArrayExpr * node, deparse_expr_cxt * context); -static void printRemoteParam(int paramindex, Oid paramtype, int32 paramtypmod, - deparse_expr_cxt * context); -static void printRemotePlaceholder(Oid paramtype, int32 paramtypmod, - deparse_expr_cxt * context); -static void deparseSelectSql(List * tlist, bool is_subquery, List * *retrieved_attrs, - deparse_expr_cxt * context); -static void appendOrderByClause(List * pathkeys, bool has_final_sort, - deparse_expr_cxt * context); -static void appendLimitClause(deparse_expr_cxt * context); -static void appendConditions(List * exprs, deparse_expr_cxt * context); -static void deparseFromExprForRel(StringInfo buf, PlannerInfo * root, - RelOptInfo * foreignrel, bool use_alias, - Index ignore_rel, List * *ignore_conds, - List * *params_list); -static void deparseFromExpr(List * quals, deparse_expr_cxt * context); -static void deparseRangeTblRef(StringInfo buf, PlannerInfo * root, - RelOptInfo * foreignrel, bool make_subquery, - Index ignore_rel, List * *ignore_conds, List * *params_list); -static void deparseAggref(Aggref * node, deparse_expr_cxt * context); -static void deparseWindowFunc(WindowFunc * node, deparse_expr_cxt * context); -static void appendGroupByClause(List * tlist, deparse_expr_cxt * context); -static CustomObjectDef * appendFunctionName(Oid funcid, deparse_expr_cxt * context); -static Node * deparseSortGroupClause(Index ref, List * tlist, bool force_colno, - deparse_expr_cxt * context); -static void deparseCoerceViaIO(CoerceViaIO * node, deparse_expr_cxt * context); -static void deparseCoalesceExpr(CoalesceExpr * node, deparse_expr_cxt * context); -static void deparseMinMaxExpr(MinMaxExpr * node, deparse_expr_cxt * context); -static void deparseRowExpr(RowExpr * node, deparse_expr_cxt * context); -static void deparseNullIfExpr(NullIfExpr * node, deparse_expr_cxt * context); -static void appendRegex(List * args, deparse_expr_cxt * context); +static void +deparseTargetList( + StringInfo buf, + RangeTblEntry* rte, + Index rtindex, + Relation rel, + Bitmapset* attrs_used, + bool qualify_col, + List** retrieved_attrs +); +static void +deparseExplicitTargetList( + List* tlist, + List** retrieved_attrs, + deparse_expr_cxt* context +); +static void +deparseSubqueryTargetList(deparse_expr_cxt* context); +static void +deparseColumnRef( + StringInfo buf, + CustomObjectDef* cdef, + int varno, + int varattno, + RangeTblEntry* rte, + bool qualify_col +); +static void +deparseRelation(StringInfo buf, Relation rel); +static void +deparseExpr(Expr* expr, deparse_expr_cxt* context); +static void +deparseVar(Var* node, deparse_expr_cxt* context); +static void +deparseConst(Const* node, deparse_expr_cxt* context, int showtype); +static void +deparseParam(Param* node, deparse_expr_cxt* context); +static void +deparseSubscriptingRef(SubscriptingRef* node, deparse_expr_cxt* context); +static void +deparseFuncExpr(FuncExpr* node, deparse_expr_cxt* context); +static void +deparseSQLValueFunction(SQLValueFunction* node, deparse_expr_cxt* context); +static void +deparseOpExpr(OpExpr* node, deparse_expr_cxt* context); +static void +deparseOperatorName(StringInfo buf, Form_pg_operator opform); +static void +deparseDistinctExpr(DistinctExpr* node, deparse_expr_cxt* context); +static void +deparseScalarArrayOpExpr(ScalarArrayOpExpr* node, deparse_expr_cxt* context); +static void +deparseCaseExpr(CaseExpr* node, deparse_expr_cxt* context); +static void +deparseCaseWhen(CaseWhen* node, deparse_expr_cxt* context); +static void +deparseRelabelType(RelabelType* node, deparse_expr_cxt* context); +static void +deparseBoolExpr(BoolExpr* node, deparse_expr_cxt* context); +static void +deparseNullTest(NullTest* node, deparse_expr_cxt* context); +static void +deparseArrayExpr(ArrayExpr* node, deparse_expr_cxt* context); +static void +deparseArrayList(ArrayExpr* node, deparse_expr_cxt* context); +static void +printRemoteParam( + int paramindex, + Oid paramtype, + int32 paramtypmod, + deparse_expr_cxt* context +); +static void +printRemotePlaceholder(Oid paramtype, int32 paramtypmod, deparse_expr_cxt* context); +static void +deparseSelectSql( + List* tlist, + bool is_subquery, + List** retrieved_attrs, + deparse_expr_cxt* context +); +static void +appendOrderByClause(List* pathkeys, bool has_final_sort, deparse_expr_cxt* context); +static void +appendLimitClause(deparse_expr_cxt* context); +static void +appendConditions(List* exprs, deparse_expr_cxt* context); +static void +deparseFromExprForRel( + StringInfo buf, + PlannerInfo* root, + RelOptInfo* foreignrel, + bool use_alias, + Index ignore_rel, + List** ignore_conds, + List** params_list +); +static void +deparseFromExpr(List* quals, deparse_expr_cxt* context); +static void +deparseRangeTblRef( + StringInfo buf, + PlannerInfo* root, + RelOptInfo* foreignrel, + bool make_subquery, + Index ignore_rel, + List** ignore_conds, + List** params_list +); +static void +deparseAggref(Aggref* node, deparse_expr_cxt* context); +static void +deparseWindowFunc(WindowFunc* node, deparse_expr_cxt* context); +static void +appendGroupByClause(List* tlist, deparse_expr_cxt* context); +static CustomObjectDef* +appendFunctionName(Oid funcid, deparse_expr_cxt* context); +static Node* +deparseSortGroupClause( + Index ref, + List* tlist, + bool force_colno, + deparse_expr_cxt* context +); +static void +deparseCoerceViaIO(CoerceViaIO* node, deparse_expr_cxt* context); +static void +deparseCoalesceExpr(CoalesceExpr* node, deparse_expr_cxt* context); +static void +deparseMinMaxExpr(MinMaxExpr* node, deparse_expr_cxt* context); +static void +deparseRowExpr(RowExpr* node, deparse_expr_cxt* context); +static void +deparseNullIfExpr(NullIfExpr* node, deparse_expr_cxt* context); +static void +appendRegex(List* args, deparse_expr_cxt* context); /* * Helper functions */ -static bool is_subquery_var(Var * node, RelOptInfo * foreignrel, - int *relno, int *colno); -static void get_relation_column_alias_ids(Var * node, RelOptInfo * foreignrel, - int *relno, int *colno); +static bool +is_subquery_var(Var* node, RelOptInfo* foreignrel, int* relno, int* colno); +static void +get_relation_column_alias_ids( + Var* node, + RelOptInfo* foreignrel, + int* relno, + int* colno +); /* * Examine each qual clause in input_conds, and classify them into two groups, @@ -225,84 +302,87 @@ static void get_relation_column_alias_ids(Var * node, RelOptInfo * foreignrel, * - local_conds contains expressions that can't be evaluated remotely */ void -chfdw_classify_conditions(PlannerInfo * root, - RelOptInfo * baserel, - List * input_conds, - List * *remote_conds, - List * *local_conds) -{ - ListCell *lc; - - *remote_conds = NIL; - *local_conds = NIL; - - foreach(lc, input_conds) - { - RestrictInfo *ri = lfirst_node(RestrictInfo, lc); - - if (chfdw_is_foreign_expr(root, baserel, ri->clause)) - *remote_conds = lappend(*remote_conds, ri); - else - *local_conds = lappend(*local_conds, ri); - } +chfdw_classify_conditions( + PlannerInfo* root, + RelOptInfo* baserel, + List* input_conds, + List** remote_conds, + List** local_conds +) { + ListCell* lc; + + *remote_conds = NIL; + *local_conds = NIL; + + foreach (lc, input_conds) { + RestrictInfo* ri = lfirst_node(RestrictInfo, lc); + + if (chfdw_is_foreign_expr(root, baserel, ri->clause)) { + *remote_conds = lappend(*remote_conds, ri); + } else { + *local_conds = lappend(*local_conds, ri); + } + } } /* * Returns true if given expr is safe to evaluate on the foreign server. */ bool -chfdw_is_foreign_expr(PlannerInfo * root, - RelOptInfo * baserel, - Expr * expr) -{ - foreign_glob_cxt glob_cxt; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) (baserel->fdw_private); - - /* - * Check that the expression consists of nodes that are safe to execute - * remotely. - */ - glob_cxt.root = root; - glob_cxt.foreignrel = baserel; - - /* - * For an upper relation, use relids from its underneath scan relation, - * because the upperrel's own relids currently aren't set to anything - * meaningful by the core code. For other relation, use their own relids. - */ - if (IS_UPPER_REL(baserel)) - glob_cxt.relids = fpinfo->outerrel->relids; - else - glob_cxt.relids = baserel->relids; - - if (!foreign_expr_walker((Node *) expr, &glob_cxt)) - return false; - - /* OK to evaluate on the remote server */ - return true; +chfdw_is_foreign_expr(PlannerInfo* root, RelOptInfo* baserel, Expr* expr) { + foreign_glob_cxt glob_cxt; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)(baserel->fdw_private); + + /* + * Check that the expression consists of nodes that are safe to execute + * remotely. + */ + glob_cxt.root = root; + glob_cxt.foreignrel = baserel; + + /* + * For an upper relation, use relids from its underneath scan relation, + * because the upperrel's own relids currently aren't set to anything + * meaningful by the core code. For other relation, use their own relids. + */ + if (IS_UPPER_REL(baserel)) { + glob_cxt.relids = fpinfo->outerrel->relids; + } else { + glob_cxt.relids = baserel->relids; + } + + if (!foreign_expr_walker((Node*)expr, &glob_cxt)) { + return false; + } + + /* OK to evaluate on the remote server */ + return true; } /* 1: '=', 2: '<>', 0 - false */ int -chfdw_is_equal_op(Oid opno) -{ - Form_pg_operator operform; - HeapTuple opertup; - int res = 0; - - opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(opno)); - if (!HeapTupleIsValid(opertup)) - elog(ERROR, "cache lookup failed for operator %u", opno); - - operform = (Form_pg_operator) GETSTRUCT(opertup); - - if (NameStr(operform->oprname)[0] == '=' && NameStr(operform->oprname)[1] == '\0') - res = 1; - else if (NameStr(operform->oprname)[0] == '<' && NameStr(operform->oprname)[1] == '>') - res = 2; - - ReleaseSysCache(opertup); - return res; +chfdw_is_equal_op(Oid opno) { + Form_pg_operator operform; + HeapTuple opertup; + int res = 0; + + opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(opno)); + if (!HeapTupleIsValid(opertup)) { + elog(ERROR, "cache lookup failed for operator %u", opno); + } + + operform = (Form_pg_operator)GETSTRUCT(opertup); + + if (NameStr(operform->oprname)[0] == '=' && NameStr(operform->oprname)[1] == '\0') { + res = 1; + } else if ( + NameStr(operform->oprname)[0] == '<' && NameStr(operform->oprname)[1] == '>' + ) { + res = 2; + } + + ReleaseSysCache(opertup); + return res; } /* @@ -317,435 +397,407 @@ chfdw_is_equal_op(Oid opno) * currently considered here. */ static bool -foreign_expr_walker(Node * node, - foreign_glob_cxt * glob_cxt) -{ - bool check_type = true; - CHFdwRelationInfo *fpinfo; - - /* Need do nothing for empty subexpressions */ - if (node == NULL) - return true; - - /* May need server info from baserel's fdw_private struct */ - fpinfo = (CHFdwRelationInfo *) (glob_cxt->foreignrel->fdw_private); - - switch (nodeTag(node)) - { - case T_Var: - { - Var *var = (Var *) node; - - /* - * If the Var is from the foreign table, we consider its - * collation (if any) safe to use. If it is from another - * table, we treat its collation the same way as we would a - * Param's collation, ie it's not safe for it to have a - * non-default collation. - */ - if (bms_is_member(var->varno, glob_cxt->relids) && - var->varlevelsup == 0) - { - /* Var belongs to foreign table */ - - /* - * System columns other than ctid and oid should not be - * sent to the remote, since we don't make any effort to - * ensure that local and remote values match (tableoid, in - * particular, almost certainly doesn't match). - */ - if (var->varattno < 0) - return false; - } - } - break; - case T_Const: - break; - case T_Param: - { - Param *p = (Param *) node; - - /* - * If it's a MULTIEXPR Param, punt. We can't tell from here - * whether the referenced sublink/subplan contains any remote - * Vars; if it does, handling that is too complicated to - * consider supporting at present. Fortunately, MULTIEXPR - * Params are not reduced to plain PARAM_EXEC until the end of - * planning, so we can easily detect this case. (Normal - * PARAM_EXEC Params are safe to ship because their values - * come from somewhere else in the plan tree; but a MULTIEXPR - * references a sub-select elsewhere in the same targetlist, - * so we'd be on the hook to evaluate it somehow if we wanted - * to handle such cases as direct foreign updates.) - */ - if (p->paramkind == PARAM_MULTIEXPR) - return false; - } - break; - case T_SubscriptingRef: - { - SubscriptingRef *ar = (SubscriptingRef *) node; - - /* Assignment should not be in restrictions. */ - if (ar->refassgnexpr != NULL) - return false; - - /* - * The jsonb subscript syntax column['key'] is not supported - * on ClickHouse JSON (it requires dot notation). Refuse to - * push down jsonb subscript expressions for now, so they are - * evaluated locally instead. - */ - if (ar->refcontainertype == JSONBOID) - return false; - - /* Recurse to remaining subexpressions. */ - if (!foreign_expr_walker((Node *) ar->refupperindexpr, - glob_cxt)) - return false; - - if (!foreign_expr_walker((Node *) ar->reflowerindexpr, - glob_cxt)) - return false; - - if (!foreign_expr_walker((Node *) ar->refexpr, - glob_cxt)) - return false; - } - break; - case T_FuncExpr: - { - CustomObjectDef *cdef = NULL; - FuncExpr *fe = (FuncExpr *) node; - - /* - * If function used by the expression is not shippable, it - * can't be sent to remote because it might have incompatible - * semantics on remote side. - */ - if (!chfdw_is_shippable(node, fe->funcid, ProcedureRelationId, fpinfo, &cdef)) - return false; - - /* - * jsonb?_extract_path_text / jsonb?_extract_path are always - * presented by the planner in variadic form: two args where - * the second is a text[] constant. Allow them through the - * variadic gate; only recurse on the column argument. - */ - if (cdef && - (cdef->cf_type == CF_JSON_EXTRACT_PATH_TEXT || - cdef->cf_type == CF_JSON_EXTRACT_PATH)) - { - if (list_length(fe->args) != 2 || - !IsA(lsecond(fe->args), Const) || - ((Const *) lsecond(fe->args))->constisnull) - return false; - - /* Only recurse on the column expression. */ - if (!foreign_expr_walker((Node *) linitial(fe->args), - glob_cxt)) - return false; - break; - } - - /* - * Recurse to input subexpressions. - */ - if (!foreign_expr_walker((Node *) fe->args, - glob_cxt)) - return false; - } - break; - case T_SQLValueFunction: - /* All handled by deparseSQLValueFunction(). */ - break; - case T_OpExpr: - case T_NullIfExpr: - case T_DistinctExpr: /* struct-equivalent to OpExpr */ - { - OpExpr *oe = (OpExpr *) node; - - /* - * Similarly, only shippable operators can be sent to remote. - * (If the operator is shippable, we assume its underlying - * function is too.) - */ - if (!chfdw_is_shippable(node, oe->opno, OperatorRelationId, fpinfo, NULL)) - return false; - - /* - * Recurse to input subexpressions. - */ - if (!foreign_expr_walker((Node *) oe->args, - glob_cxt)) - return false; - } - break; - case T_ScalarArrayOpExpr: - { - ScalarArrayOpExpr *oe = (ScalarArrayOpExpr *) node; - - if (!chfdw_is_equal_op(oe->opno)) - return false; - - /* - * Recurse to input subexpressions. - */ - if (!foreign_expr_walker((Node *) oe->args, - glob_cxt)) - return false; - } - break; - case T_RelabelType: - { - RelabelType *r = (RelabelType *) node; - - /* - * Recurse to input subexpression. - */ - if (!foreign_expr_walker((Node *) r->arg, - glob_cxt)) - { - return false; - } - } - break; - case T_BoolExpr: - { - BoolExpr *b = (BoolExpr *) node; - - /* - * Recurse to input subexpressions. - */ - if (!foreign_expr_walker((Node *) b->args, - glob_cxt)) - { - return false; - } - } - break; - case T_NullTest: - { - NullTest *nt = (NullTest *) node; - - /* - * Recurse to input subexpressions. - */ - if (!foreign_expr_walker((Node *) nt->arg, - glob_cxt)) - { - return false; - } - } - break; - case T_ArrayExpr: - { - ArrayExpr *a = (ArrayExpr *) node; - - /* - * Recurse to input subexpressions. - */ - if (!foreign_expr_walker((Node *) a->elements, - glob_cxt)) - { - return false; - } - } - break; - case T_List: - { - List *l = (List *) node; - ListCell *lc; - - /* - * Recurse to component subexpressions. - */ - foreach(lc, l) - { - if (!foreign_expr_walker((Node *) lfirst(lc), - glob_cxt)) - { - return false; - } - } - - /* Don't apply exprType() to the list. */ - check_type = false; - } - break; - case T_Aggref: - { - Aggref *agg = (Aggref *) node; - ListCell *lc; - - /* Not safe to pushdown when not in grouping context */ - if (!IS_UPPER_REL(glob_cxt->foreignrel)) - return false; - - /* Only non-split aggregates are pushable. */ - if (agg->aggsplit != AGGSPLIT_SIMPLE) - return false; - - /* As usual, it must be shippable. */ - if (!chfdw_is_shippable(node, agg->aggfnoid, ProcedureRelationId, fpinfo, NULL)) - return false; - - /* Features that ClickHouse doesn't support */ - if (AGGKIND_IS_ORDERED_SET(agg->aggkind) - && !chfdw_check_for_ordered_aggregate(agg)) - return false; - - if (agg->aggdistinct && agg->aggfilter) - return false; - - /* groupConcat has no ORDER BY; block ordered string_agg */ - if (agg->aggfnoid == F_STRING_AGG_TEXT_TEXT - && agg->aggorder != NIL) - return false; - - /* Block aggregates with no ClickHouse equivalent */ - switch (agg->aggfnoid) - { - case F_STRING_AGG_BYTEA_BYTEA: - case F_REGR_COUNT: - case F_REGR_SXX: - case F_REGR_SYY: - case F_REGR_SXY: - case F_REGR_AVGX: - case F_REGR_AVGY: - case F_REGR_R2: - case F_REGR_SLOPE: - case F_REGR_INTERCEPT: +foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt) { + bool check_type = true; + CHFdwRelationInfo* fpinfo; + + /* Need do nothing for empty subexpressions */ + if (node == NULL) { + return true; + } + + /* May need server info from baserel's fdw_private struct */ + fpinfo = (CHFdwRelationInfo*)(glob_cxt->foreignrel->fdw_private); + + switch (nodeTag(node)) { + case T_Var: { + Var* var = (Var*)node; + + /* + * If the Var is from the foreign table, we consider its + * collation (if any) safe to use. If it is from another + * table, we treat its collation the same way as we would a + * Param's collation, ie it's not safe for it to have a + * non-default collation. + */ + if (bms_is_member(var->varno, glob_cxt->relids) && var->varlevelsup == 0) { + /* Var belongs to foreign table */ + + /* + * System columns other than ctid and oid should not be + * sent to the remote, since we don't make any effort to + * ensure that local and remote values match (tableoid, in + * particular, almost certainly doesn't match). + */ + if (var->varattno < 0) { + return false; + } + } + } break; + case T_Const: + break; + case T_Param: { + Param* p = (Param*)node; + + /* + * If it's a MULTIEXPR Param, punt. We can't tell from here + * whether the referenced sublink/subplan contains any remote + * Vars; if it does, handling that is too complicated to + * consider supporting at present. Fortunately, MULTIEXPR + * Params are not reduced to plain PARAM_EXEC until the end of + * planning, so we can easily detect this case. (Normal + * PARAM_EXEC Params are safe to ship because their values + * come from somewhere else in the plan tree; but a MULTIEXPR + * references a sub-select elsewhere in the same targetlist, + * so we'd be on the hook to evaluate it somehow if we wanted + * to handle such cases as direct foreign updates.) + */ + if (p->paramkind == PARAM_MULTIEXPR) { + return false; + } + } break; + case T_SubscriptingRef: { + SubscriptingRef* ar = (SubscriptingRef*)node; + + /* Assignment should not be in restrictions. */ + if (ar->refassgnexpr != NULL) { + return false; + } + + /* + * The jsonb subscript syntax column['key'] is not supported + * on ClickHouse JSON (it requires dot notation). Refuse to + * push down jsonb subscript expressions for now, so they are + * evaluated locally instead. + */ + if (ar->refcontainertype == JSONBOID) { + return false; + } + + /* Recurse to remaining subexpressions. */ + if (!foreign_expr_walker((Node*)ar->refupperindexpr, glob_cxt)) { + return false; + } + + if (!foreign_expr_walker((Node*)ar->reflowerindexpr, glob_cxt)) { + return false; + } + + if (!foreign_expr_walker((Node*)ar->refexpr, glob_cxt)) { + return false; + } + } break; + case T_FuncExpr: { + CustomObjectDef* cdef = NULL; + FuncExpr* fe = (FuncExpr*)node; + + /* + * If function used by the expression is not shippable, it + * can't be sent to remote because it might have incompatible + * semantics on remote side. + */ + if (!chfdw_is_shippable(node, fe->funcid, ProcedureRelationId, fpinfo, &cdef)) { + return false; + } + + /* + * jsonb?_extract_path_text / jsonb?_extract_path are always + * presented by the planner in variadic form: two args where + * the second is a text[] constant. Allow them through the + * variadic gate; only recurse on the column argument. + */ + if (cdef && (cdef->cf_type == CF_JSON_EXTRACT_PATH_TEXT || + cdef->cf_type == CF_JSON_EXTRACT_PATH)) { + if (list_length(fe->args) != 2 || !IsA(lsecond(fe->args), Const) || + ((Const*)lsecond(fe->args))->constisnull) { + return false; + } + + /* Only recurse on the column expression. */ + if (!foreign_expr_walker((Node*)linitial(fe->args), glob_cxt)) { + return false; + } + break; + } + + /* + * Recurse to input subexpressions. + */ + if (!foreign_expr_walker((Node*)fe->args, glob_cxt)) { + return false; + } + } break; + case T_SQLValueFunction: + /* All handled by deparseSQLValueFunction(). */ + break; + case T_OpExpr: + case T_NullIfExpr: + case T_DistinctExpr: /* struct-equivalent to OpExpr */ + { + OpExpr* oe = (OpExpr*)node; + + /* + * Similarly, only shippable operators can be sent to remote. + * (If the operator is shippable, we assume its underlying + * function is too.) + */ + if (!chfdw_is_shippable(node, oe->opno, OperatorRelationId, fpinfo, NULL)) { + return false; + } + + /* + * Recurse to input subexpressions. + */ + if (!foreign_expr_walker((Node*)oe->args, glob_cxt)) { + return false; + } + } break; + case T_ScalarArrayOpExpr: { + ScalarArrayOpExpr* oe = (ScalarArrayOpExpr*)node; + + if (!chfdw_is_equal_op(oe->opno)) { + return false; + } + + /* + * Recurse to input subexpressions. + */ + if (!foreign_expr_walker((Node*)oe->args, glob_cxt)) { + return false; + } + } break; + case T_RelabelType: { + RelabelType* r = (RelabelType*)node; + + /* + * Recurse to input subexpression. + */ + if (!foreign_expr_walker((Node*)r->arg, glob_cxt)) { + return false; + } + } break; + case T_BoolExpr: { + BoolExpr* b = (BoolExpr*)node; + + /* + * Recurse to input subexpressions. + */ + if (!foreign_expr_walker((Node*)b->args, glob_cxt)) { + return false; + } + } break; + case T_NullTest: { + NullTest* nt = (NullTest*)node; + + /* + * Recurse to input subexpressions. + */ + if (!foreign_expr_walker((Node*)nt->arg, glob_cxt)) { + return false; + } + } break; + case T_ArrayExpr: { + ArrayExpr* a = (ArrayExpr*)node; + + /* + * Recurse to input subexpressions. + */ + if (!foreign_expr_walker((Node*)a->elements, glob_cxt)) { + return false; + } + } break; + case T_List: { + List* l = (List*)node; + ListCell* lc; + + /* + * Recurse to component subexpressions. + */ + foreach (lc, l) { + if (!foreign_expr_walker((Node*)lfirst(lc), glob_cxt)) { + return false; + } + } + + /* Don't apply exprType() to the list. */ + check_type = false; + } break; + case T_Aggref: { + Aggref* agg = (Aggref*)node; + ListCell* lc; + + /* Not safe to pushdown when not in grouping context */ + if (!IS_UPPER_REL(glob_cxt->foreignrel)) { + return false; + } + + /* Only non-split aggregates are pushable. */ + if (agg->aggsplit != AGGSPLIT_SIMPLE) { + return false; + } + + /* As usual, it must be shippable. */ + if (!chfdw_is_shippable( + node, agg->aggfnoid, ProcedureRelationId, fpinfo, NULL + )) { + return false; + } + + /* Features that ClickHouse doesn't support */ + if (AGGKIND_IS_ORDERED_SET(agg->aggkind) && + !chfdw_check_for_ordered_aggregate(agg)) { + return false; + } + + if (agg->aggdistinct && agg->aggfilter) { + return false; + } + + /* groupConcat has no ORDER BY; block ordered string_agg */ + if (agg->aggfnoid == F_STRING_AGG_TEXT_TEXT && agg->aggorder != NIL) { + return false; + } + + /* Block aggregates with no ClickHouse equivalent */ + switch (agg->aggfnoid) { + case F_STRING_AGG_BYTEA_BYTEA: + case F_REGR_COUNT: + case F_REGR_SXX: + case F_REGR_SYY: + case F_REGR_SXY: + case F_REGR_AVGX: + case F_REGR_AVGY: + case F_REGR_R2: + case F_REGR_SLOPE: + case F_REGR_INTERCEPT: #if PG_VERSION_NUM >= 160000 - case F_JSON_AGG_STRICT: - case F_JSONB_AGG_STRICT: + case F_JSON_AGG_STRICT: + case F_JSONB_AGG_STRICT: #endif - return false; - } - - /* - * Recurse to input args. aggdirectargs, aggorder and - * aggdistinct are all present in args, so no need to check - * their shippability explicitly. - */ - foreach(lc, agg->args) - { - Node *n = (Node *) lfirst(lc); - - /* If TargetEntry, extract the expression from it */ - if (IsA(n, TargetEntry)) - { - TargetEntry *tle = (TargetEntry *) n; - - n = (Node *) tle->expr; - } - - if (!foreign_expr_walker(n, glob_cxt)) - return false; - } - - /* Check aggregate filter */ - if (!foreign_expr_walker((Node *) agg->aggfilter, - glob_cxt)) - return false; - } - break; - case T_WindowFunc: - { - WindowFunc *wfunc = (WindowFunc *) node; - - /* Not safe to pushdown when not in upper relation context */ - if (!IS_UPPER_REL(glob_cxt->foreignrel)) - return false; - - /* The window function must be shippable */ - if (!chfdw_is_shippable(node, wfunc->winfnoid, ProcedureRelationId, fpinfo, NULL)) - return false; - - /* FILTER is not supported in ClickHouse window functions */ - if (wfunc->aggfilter) - return false; - - /* Recurse to input arguments */ - if (!foreign_expr_walker((Node *) wfunc->args, - glob_cxt)) - return false; - } - break; - case T_CaseExpr: - { - CaseExpr *caseexpr = (CaseExpr *) node; - ListCell *lc; - - if (!foreign_expr_walker((Node *) caseexpr->arg, glob_cxt)) - return true; - - foreach(lc, caseexpr->args) - { - CaseWhen *when = lfirst_node(CaseWhen, lc); - - if (!foreign_expr_walker((Node *) when->expr, glob_cxt)) - return false; - if (!foreign_expr_walker((Node *) when->result, glob_cxt)) - return false; - } - if (!foreign_expr_walker((Node *) caseexpr->defresult, glob_cxt)) - return false; - } - break; - case T_CoalesceExpr: - { - CoalesceExpr *ce = (CoalesceExpr *) node; - - if (!foreign_expr_walker((Node *) ce->args, glob_cxt)) - return false; - } - break; - case T_MinMaxExpr: - { - MinMaxExpr *me = (MinMaxExpr *) node; - - if (!foreign_expr_walker((Node *) me->args, glob_cxt)) - return false; - } - break; - case T_CoerceViaIO: - { - CoerceViaIO *me = (CoerceViaIO *) node; - - if (!foreign_expr_walker((Node *) me->arg, glob_cxt)) - return false; - } - break; - case T_RowExpr: - { - RowExpr *me = (RowExpr *) node; - - if (!foreign_expr_walker((Node *) me->args, glob_cxt)) - return false; - } - break; - case T_CaseTestExpr: - break; - default: - - /* - * If it's anything else, assume it's unsafe. This list can be - * expanded later, but don't forget to add deparse support below. - */ - return false; - } - - /* - * If result type of given expression is not shippable, it can't be sent - * to remote because it might have incompatible semantics on remote side. - */ - if (check_type && !chfdw_is_shippable(node, exprType(node), TypeRelationId, fpinfo, NULL)) - { - return false; - } - - return true; + return false; + } + + /* + * Recurse to input args. aggdirectargs, aggorder and + * aggdistinct are all present in args, so no need to check + * their shippability explicitly. + */ + foreach (lc, agg->args) { + Node* n = (Node*)lfirst(lc); + + /* If TargetEntry, extract the expression from it */ + if (IsA(n, TargetEntry)) { + TargetEntry* tle = (TargetEntry*)n; + + n = (Node*)tle->expr; + } + + if (!foreign_expr_walker(n, glob_cxt)) { + return false; + } + } + + /* Check aggregate filter */ + if (!foreign_expr_walker((Node*)agg->aggfilter, glob_cxt)) { + return false; + } + } break; + case T_WindowFunc: { + WindowFunc* wfunc = (WindowFunc*)node; + + /* Not safe to pushdown when not in upper relation context */ + if (!IS_UPPER_REL(glob_cxt->foreignrel)) { + return false; + } + + /* The window function must be shippable */ + if (!chfdw_is_shippable( + node, wfunc->winfnoid, ProcedureRelationId, fpinfo, NULL + )) { + return false; + } + + /* FILTER is not supported in ClickHouse window functions */ + if (wfunc->aggfilter) { + return false; + } + + /* Recurse to input arguments */ + if (!foreign_expr_walker((Node*)wfunc->args, glob_cxt)) { + return false; + } + } break; + case T_CaseExpr: { + CaseExpr* caseexpr = (CaseExpr*)node; + ListCell* lc; + + if (!foreign_expr_walker((Node*)caseexpr->arg, glob_cxt)) { + return true; + } + + foreach (lc, caseexpr->args) { + CaseWhen* when = lfirst_node(CaseWhen, lc); + + if (!foreign_expr_walker((Node*)when->expr, glob_cxt)) { + return false; + } + if (!foreign_expr_walker((Node*)when->result, glob_cxt)) { + return false; + } + } + if (!foreign_expr_walker((Node*)caseexpr->defresult, glob_cxt)) { + return false; + } + } break; + case T_CoalesceExpr: { + CoalesceExpr* ce = (CoalesceExpr*)node; + + if (!foreign_expr_walker((Node*)ce->args, glob_cxt)) { + return false; + } + } break; + case T_MinMaxExpr: { + MinMaxExpr* me = (MinMaxExpr*)node; + + if (!foreign_expr_walker((Node*)me->args, glob_cxt)) { + return false; + } + } break; + case T_CoerceViaIO: { + CoerceViaIO* me = (CoerceViaIO*)node; + + if (!foreign_expr_walker((Node*)me->arg, glob_cxt)) { + return false; + } + } break; + case T_RowExpr: { + RowExpr* me = (RowExpr*)node; + + if (!foreign_expr_walker((Node*)me->args, glob_cxt)) { + return false; + } + } break; + case T_CaseTestExpr: + break; + default: + + /* + * If it's anything else, assume it's unsafe. This list can be + * expanded later, but don't forget to add deparse support below. + */ + return false; + } + + /* + * If result type of given expression is not shippable, it can't be sent + * to remote because it might have incompatible semantics on remote side. + */ + if (check_type && + !chfdw_is_shippable(node, exprType(node), TypeRelationId, fpinfo, NULL)) { + return false; + } + + return true; } /* @@ -761,225 +813,219 @@ foreign_expr_walker(Node * node, * expression into). */ bool -is_foreign_param(PlannerInfo * root, - RelOptInfo * baserel, - Expr * expr) -{ - if (expr == NULL) - return false; - - switch (nodeTag(expr)) - { - case T_Var: - { - /* It would have to be sent unless it's a foreign Var */ - Var *var = (Var *) expr; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) (baserel->fdw_private); - Relids relids; - - if (IS_UPPER_REL(baserel)) - relids = fpinfo->outerrel->relids; - else - relids = baserel->relids; - - if (bms_is_member(var->varno, relids) && var->varlevelsup == 0) - return false; /* foreign Var, so not a param */ - else - return true; /* it'd have to be a param */ - break; - } - case T_Param: - /* Params always have to be sent to the foreign server */ - return true; - default: - break; - } - return false; +is_foreign_param(PlannerInfo* root, RelOptInfo* baserel, Expr* expr) { + if (expr == NULL) { + return false; + } + + switch (nodeTag(expr)) { + case T_Var: { + /* It would have to be sent unless it's a foreign Var */ + Var* var = (Var*)expr; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)(baserel->fdw_private); + Relids relids; + + if (IS_UPPER_REL(baserel)) { + relids = fpinfo->outerrel->relids; + } else { + relids = baserel->relids; + } + + if (bms_is_member(var->varno, relids) && var->varlevelsup == 0) { + return false; /* foreign Var, so not a param */ + } else { + return true; /* it'd have to be a param */ + } + break; + } + case T_Param: + /* Params always have to be sent to the foreign server */ + return true; + default: + break; + } + return false; } /* * Add typmod decoration to the basic type name. Copied from * src/backend/utils/adt/format_type.c in the Postgres source. */ -static char * -printTypmod(const char *typname, int32 typmod, Oid typmodout) -{ - char *res; - - /* Shouldn't be called if typmod is -1 */ - Assert(typmod >= 0); - - if (typmodout == InvalidOid) - { - /* Default behavior: just print the integer typmod with parens */ - res = psprintf("%s(%d)", typname, (int) typmod); - } - else - { - /* Use the type-specific typmodout procedure */ - char *tmstr; - - tmstr = DatumGetCString(OidFunctionCall1(typmodout, - Int32GetDatum(typmod))); - res = psprintf("%s%s", typname, tmstr); - } - - return res; +static char* +printTypmod(const char* typname, int32 typmod, Oid typmodout) { + char* res; + + /* Shouldn't be called if typmod is -1 */ + Assert(typmod >= 0); + + if (typmodout == InvalidOid) { + /* Default behavior: just print the integer typmod with parens */ + res = psprintf("%s(%d)", typname, (int)typmod); + } else { + /* Use the type-specific typmodout procedure */ + char* tmstr; + + tmstr = DatumGetCString(OidFunctionCall1(typmodout, Int32GetDatum(typmod))); + res = psprintf("%s%s", typname, tmstr); + } + + return res; } -static char * -ch_format_type_extended(Oid type_oid, int32 typemod, uint16 flags) -{ - HeapTuple tuple; - Form_pg_type typeform; - Oid array_base_type; - bool is_array; - char *buf; - bool with_typemod; - - tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_oid)); - if (!HeapTupleIsValid(tuple)) - elog(ERROR, "pg_clickhouse: cache lookup failed for type %u", type_oid); - - typeform = (Form_pg_type) GETSTRUCT(tuple); - - /* - * Check if it's a regular (variable length) array type. Fixed-length - * array types such as "name" shouldn't get deconstructed. As of Postgres - * 8.1, rather than checking typlen we check the toast property, and don't - * deconstruct "plain storage" array types --- this is because we don't - * want to show oidvector as oid[]. - */ - array_base_type = typeform->typelem; - - if (array_base_type != InvalidOid && typeform->typstorage != 'p') - { - /* Switch our attention to the array element type */ - ReleaseSysCache(tuple); - tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(array_base_type)); - if (!HeapTupleIsValid(tuple)) - elog(ERROR, "pg_clickhouse: cache lookup failed for type %u", type_oid); - - typeform = (Form_pg_type) GETSTRUCT(tuple); - type_oid = array_base_type; - is_array = true; - } - else - is_array = false; - - with_typemod = (flags & FORMAT_TYPE_TYPEMOD_GIVEN) != 0 && (typemod >= 0); - - /* - * See if we want to special-case the output for certain built-in types. - * Note that these special cases should all correspond to special - * productions in gram.y, to ensure that the type name will be taken as a - * system type, not a user type of the same name. - * - * If we do not provide a special-case output here, the type name will be - * handled the same way as a user type name --- in particular, it will be - * double-quoted if it matches any lexer keyword. This behavior is - * essential for some cases, such as types "bit" and "char". - */ - buf = NULL; /* flag for no special case */ - - switch (type_oid) - { - case BOOLOID: - buf = pstrdup("Boolean"); - break; - - case BPCHAROID: - if (with_typemod) - buf = printTypmod("FixedString", typemod, typeform->typmodout); - else if ((flags & FORMAT_TYPE_TYPEMOD_GIVEN) != 0) - { - /* - * bpchar with typmod -1 is not the same as CHARACTER, which - * means CHARACTER(1) per SQL spec. Report it as bpchar so - * that parser will not assign a bogus typmod. - */ - } - else - buf = pstrdup("String"); - break; - - case FLOAT4OID: - buf = pstrdup("Float32"); - break; - - case FLOAT8OID: - buf = pstrdup("Float64"); - break; - - case INT2OID: - buf = pstrdup("Int16"); - break; - - case INT4OID: - buf = pstrdup("Int32"); - break; - - case INT8OID: - buf = pstrdup("Int64"); - break; - - case NUMERICOID: - if (with_typemod) - buf = printTypmod("Decimal", typemod, typeform->typmodout); - else - buf = pstrdup("Decimal"); - break; - - case INTERVALOID: - if (with_typemod) - buf = printTypmod("UInt64", typemod, typeform->typmodout); - else - buf = pstrdup("UInt64"); - break; - - case TIMESTAMPTZOID: - case TIMESTAMPOID: - buf = pstrdup("DateTime"); - break; - case DATEOID: - buf = pstrdup("Date"); - break; - - case VARCHAROID: - if (with_typemod) - buf = printTypmod("FixedString", typemod, typeform->typmodout); - else - buf = pstrdup("String"); - break; - case TEXTOID: - buf = pstrdup("String"); - break; - } - - if (buf == NULL) - { - CustomObjectDef *cdef; - char *typname; - - cdef = chfdw_check_for_custom_type(type_oid); - if (cdef && cdef->custom_name[0] != '\0') - buf = pstrdup(cdef->custom_name); - else - { - typname = NameStr(typeform->typname); - buf = quote_qualified_identifier(NULL, typname); - - if (with_typemod) - buf = printTypmod(buf, typemod, typeform->typmodout); - } - } - - if (is_array) - buf = psprintf("Array(%s)", buf); - - ReleaseSysCache(tuple); - - return buf; +static char* +ch_format_type_extended(Oid type_oid, int32 typemod, uint16 flags) { + HeapTuple tuple; + Form_pg_type typeform; + Oid array_base_type; + bool is_array; + char* buf; + bool with_typemod; + + tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_oid)); + if (!HeapTupleIsValid(tuple)) { + elog(ERROR, "pg_clickhouse: cache lookup failed for type %u", type_oid); + } + + typeform = (Form_pg_type)GETSTRUCT(tuple); + + /* + * Check if it's a regular (variable length) array type. Fixed-length + * array types such as "name" shouldn't get deconstructed. As of Postgres + * 8.1, rather than checking typlen we check the toast property, and don't + * deconstruct "plain storage" array types --- this is because we don't + * want to show oidvector as oid[]. + */ + array_base_type = typeform->typelem; + + if (array_base_type != InvalidOid && typeform->typstorage != 'p') { + /* Switch our attention to the array element type */ + ReleaseSysCache(tuple); + tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(array_base_type)); + if (!HeapTupleIsValid(tuple)) { + elog(ERROR, "pg_clickhouse: cache lookup failed for type %u", type_oid); + } + + typeform = (Form_pg_type)GETSTRUCT(tuple); + type_oid = array_base_type; + is_array = true; + } else { + is_array = false; + } + + with_typemod = (flags & FORMAT_TYPE_TYPEMOD_GIVEN) != 0 && (typemod >= 0); + + /* + * See if we want to special-case the output for certain built-in types. + * Note that these special cases should all correspond to special + * productions in gram.y, to ensure that the type name will be taken as a + * system type, not a user type of the same name. + * + * If we do not provide a special-case output here, the type name will be + * handled the same way as a user type name --- in particular, it will be + * double-quoted if it matches any lexer keyword. This behavior is + * essential for some cases, such as types "bit" and "char". + */ + buf = NULL; /* flag for no special case */ + + switch (type_oid) { + case BOOLOID: + buf = pstrdup("Boolean"); + break; + + case BPCHAROID: + if (with_typemod) { + buf = printTypmod("FixedString", typemod, typeform->typmodout); + } else if ((flags & FORMAT_TYPE_TYPEMOD_GIVEN) != 0) { + /* + * bpchar with typmod -1 is not the same as CHARACTER, which + * means CHARACTER(1) per SQL spec. Report it as bpchar so + * that parser will not assign a bogus typmod. + */ + } else { + buf = pstrdup("String"); + } + break; + + case FLOAT4OID: + buf = pstrdup("Float32"); + break; + + case FLOAT8OID: + buf = pstrdup("Float64"); + break; + + case INT2OID: + buf = pstrdup("Int16"); + break; + + case INT4OID: + buf = pstrdup("Int32"); + break; + + case INT8OID: + buf = pstrdup("Int64"); + break; + + case NUMERICOID: + if (with_typemod) { + buf = printTypmod("Decimal", typemod, typeform->typmodout); + } else { + buf = pstrdup("Decimal"); + } + break; + + case INTERVALOID: + if (with_typemod) { + buf = printTypmod("UInt64", typemod, typeform->typmodout); + } else { + buf = pstrdup("UInt64"); + } + break; + + case TIMESTAMPTZOID: + case TIMESTAMPOID: + buf = pstrdup("DateTime"); + break; + case DATEOID: + buf = pstrdup("Date"); + break; + + case VARCHAROID: + if (with_typemod) { + buf = printTypmod("FixedString", typemod, typeform->typmodout); + } else { + buf = pstrdup("String"); + } + break; + case TEXTOID: + buf = pstrdup("String"); + break; + } + + if (buf == NULL) { + CustomObjectDef* cdef; + char* typname; + + cdef = chfdw_check_for_custom_type(type_oid); + if (cdef && cdef->custom_name[0] != '\0') { + buf = pstrdup(cdef->custom_name); + } else { + typname = NameStr(typeform->typname); + buf = quote_qualified_identifier(NULL, typname); + + if (with_typemod) { + buf = printTypmod(buf, typemod, typeform->typmodout); + } + } + } + + if (is_array) { + buf = psprintf("Array(%s)", buf); + } + + ReleaseSysCache(tuple); + + return buf; } /* @@ -993,15 +1039,15 @@ ch_format_type_extended(Oid type_oid, int32 typemod, uint16 flags) * type names that are not in pg_catalog. We assume here that built-in types * are all in pg_catalog and need not be qualified; otherwise, qualify. */ -static char * -deparse_type_name(Oid type_oid, int32 typemod) -{ - uint16 flags = FORMAT_TYPE_TYPEMOD_GIVEN; +static char* +deparse_type_name(Oid type_oid, int32 typemod) { + uint16 flags = FORMAT_TYPE_TYPEMOD_GIVEN; - if (!chfdw_is_builtin(type_oid)) - flags |= FORMAT_TYPE_FORCE_QUALIFY; + if (!chfdw_is_builtin(type_oid)) { + flags |= FORMAT_TYPE_FORCE_QUALIFY; + } - return ch_format_type_extended(type_oid, typemod, flags); + return ch_format_type_extended(type_oid, typemod, flags); } /* @@ -1012,37 +1058,37 @@ deparse_type_name(Oid type_oid, int32 typemod) * then the output targetlist can also contain expressions to be evaluated on * foreign server. */ -List * -chfdw_build_tlist_to_deparse(RelOptInfo * foreignrel) -{ - List *tlist = NIL; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) foreignrel->fdw_private; - ListCell *lc; - - /* - * For an upper relation, we have already built the target list while - * checking shippability, so just return that. - */ - if (IS_UPPER_REL(foreignrel)) - return fpinfo->grouped_tlist; - - /* - * We require columns specified in foreignrel->reltarget->exprs and those - * required for evaluating the local conditions. - */ - tlist = add_to_flat_tlist(tlist, - pull_var_clause((Node *) foreignrel->reltarget->exprs, - PVC_RECURSE_PLACEHOLDERS)); - foreach(lc, fpinfo->local_conds) - { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); - - tlist = add_to_flat_tlist(tlist, - pull_var_clause((Node *) rinfo->clause, - PVC_RECURSE_PLACEHOLDERS)); - } - - return tlist; +List* +chfdw_build_tlist_to_deparse(RelOptInfo* foreignrel) { + List* tlist = NIL; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)foreignrel->fdw_private; + ListCell* lc; + + /* + * For an upper relation, we have already built the target list while + * checking shippability, so just return that. + */ + if (IS_UPPER_REL(foreignrel)) { + return fpinfo->grouped_tlist; + } + + /* + * We require columns specified in foreignrel->reltarget->exprs and those + * required for evaluating the local conditions. + */ + tlist = add_to_flat_tlist( + tlist, + pull_var_clause((Node*)foreignrel->reltarget->exprs, PVC_RECURSE_PLACEHOLDERS) + ); + foreach (lc, fpinfo->local_conds) { + RestrictInfo* rinfo = lfirst_node(RestrictInfo, lc); + + tlist = add_to_flat_tlist( + tlist, pull_var_clause((Node*)rinfo->clause, PVC_RECURSE_PLACEHOLDERS) + ); + } + + return tlist; } /* @@ -1070,75 +1116,82 @@ chfdw_build_tlist_to_deparse(RelOptInfo * foreignrel) * List of columns selected is returned in retrieved_attrs. */ void -chfdw_deparse_select_stmt_for_rel(StringInfo buf, PlannerInfo * root, RelOptInfo * rel, - List * tlist, List * remote_conds, List * pathkeys, - bool has_final_sort, bool has_limit, bool is_subquery, - List * *retrieved_attrs, List * *params_list) -{ - deparse_expr_cxt context; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) rel->fdw_private; - List *quals; - - elog(DEBUG2, "> %s:%d", __FUNCTION__, __LINE__); - - /* - * We handle relations for foreign tables, joins between those and upper - * relations. - */ - Assert(IS_JOIN_REL(rel) || IS_SIMPLE_REL(rel) || IS_UPPER_REL(rel)); - - /* Fill portions of context common to upper, join and base relation */ - context.buf = buf; - context.root = root; - context.foreignrel = rel; - context.scanrel = IS_UPPER_REL(rel) ? fpinfo->outerrel : rel; - context.params_list = params_list; - context.func = NULL; - context.interval_op = false; - context.array_as_tuple = false; - context.no_sort_parens = false; - - /* Construct SELECT clause */ - deparseSelectSql(tlist, is_subquery, retrieved_attrs, &context); - - /* - * For upper relations, the WHERE clause is built from the remote - * conditions of the underlying scan relation; otherwise, we can use the - * supplied list of remote conditions directly. - */ - if (IS_UPPER_REL(rel)) - { - CHFdwRelationInfo *ofpinfo; - - ofpinfo = (CHFdwRelationInfo *) fpinfo->outerrel->fdw_private; - quals = ofpinfo->remote_conds; - } - else - quals = remote_conds; - - /* Construct FROM and WHERE clauses */ - deparseFromExpr(quals, &context); - - if (IS_UPPER_REL(rel)) - { - /* Append GROUP BY clause */ - appendGroupByClause(tlist, &context); - - /* Append HAVING clause */ - if (remote_conds) - { - appendStringInfoString(buf, " HAVING "); - appendConditions(remote_conds, &context); - } - } - - /* Add ORDER BY clause if we found any useful pathkeys */ - if (pathkeys) - appendOrderByClause(pathkeys, has_final_sort, &context); - - /* Add LIMIT clause if necessary */ - if (has_limit) - appendLimitClause(&context); +chfdw_deparse_select_stmt_for_rel( + StringInfo buf, + PlannerInfo* root, + RelOptInfo* rel, + List* tlist, + List* remote_conds, + List* pathkeys, + bool has_final_sort, + bool has_limit, + bool is_subquery, + List** retrieved_attrs, + List** params_list +) { + deparse_expr_cxt context; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)rel->fdw_private; + List* quals; + + elog(DEBUG2, "> %s:%d", __FUNCTION__, __LINE__); + + /* + * We handle relations for foreign tables, joins between those and upper + * relations. + */ + Assert(IS_JOIN_REL(rel) || IS_SIMPLE_REL(rel) || IS_UPPER_REL(rel)); + + /* Fill portions of context common to upper, join and base relation */ + context.buf = buf; + context.root = root; + context.foreignrel = rel; + context.scanrel = IS_UPPER_REL(rel) ? fpinfo->outerrel : rel; + context.params_list = params_list; + context.func = NULL; + context.interval_op = false; + context.array_as_tuple = false; + context.no_sort_parens = false; + + /* Construct SELECT clause */ + deparseSelectSql(tlist, is_subquery, retrieved_attrs, &context); + + /* + * For upper relations, the WHERE clause is built from the remote + * conditions of the underlying scan relation; otherwise, we can use the + * supplied list of remote conditions directly. + */ + if (IS_UPPER_REL(rel)) { + CHFdwRelationInfo* ofpinfo; + + ofpinfo = (CHFdwRelationInfo*)fpinfo->outerrel->fdw_private; + quals = ofpinfo->remote_conds; + } else { + quals = remote_conds; + } + + /* Construct FROM and WHERE clauses */ + deparseFromExpr(quals, &context); + + if (IS_UPPER_REL(rel)) { + /* Append GROUP BY clause */ + appendGroupByClause(tlist, &context); + + /* Append HAVING clause */ + if (remote_conds) { + appendStringInfoString(buf, " HAVING "); + appendConditions(remote_conds, &context); + } + } + + /* Add ORDER BY clause if we found any useful pathkeys */ + if (pathkeys) { + appendOrderByClause(pathkeys, has_final_sort, &context); + } + + /* Add LIMIT clause if necessary */ + if (has_limit) { + appendLimitClause(&context); + } } /* @@ -1155,59 +1208,58 @@ chfdw_deparse_select_stmt_for_rel(StringInfo buf, PlannerInfo * root, RelOptInfo * Read prologue of chfdw_deparse_select_stmt_for_rel() for details. */ static void -deparseSelectSql(List * tlist, bool is_subquery, List * *retrieved_attrs, - deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - RelOptInfo *foreignrel = context->foreignrel; - PlannerInfo *root = context->root; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) foreignrel->fdw_private; - - /* reset var counter */ - var_counter = 0; - elog(DEBUG2, "> %s:%d", __FUNCTION__, __LINE__); - - /* - * Construct SELECT list - */ - appendStringInfoString(buf, "SELECT "); - - if (is_subquery) - { - /* - * For a relation that is deparsed as a subquery, emit expressions - * specified in the relation's reltarget. Note that since this is for - * the subquery, no need to care about *retrieved_attrs. - */ - deparseSubqueryTargetList(context); - } - else if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel)) - { - /* - * For a join or upper relation the input tlist gives the list of - * columns required to be fetched from the foreign server. - */ - deparseExplicitTargetList(tlist, retrieved_attrs, context); - } - else - { - /* - * For a base relation fpinfo->attrs_used gives the list of columns - * required to be fetched from the foreign server. - */ - RangeTblEntry *rte = planner_rt_fetch(foreignrel->relid, root); - - /* - * Core code already has some lock on each rel being planned, so we - * can use NoLock here. - */ - Relation rel = table_open_compat(rte->relid, NoLock); - - deparseTargetList(buf, rte, foreignrel->relid, rel, - fpinfo->attrs_used, false, retrieved_attrs); - table_close_compat(rel, NoLock); - } - elog(DEBUG2, "< %s:%d", __FUNCTION__, __LINE__); +deparseSelectSql( + List* tlist, + bool is_subquery, + List** retrieved_attrs, + deparse_expr_cxt* context +) { + StringInfo buf = context->buf; + RelOptInfo* foreignrel = context->foreignrel; + PlannerInfo* root = context->root; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)foreignrel->fdw_private; + + /* reset var counter */ + var_counter = 0; + elog(DEBUG2, "> %s:%d", __FUNCTION__, __LINE__); + + /* + * Construct SELECT list + */ + appendStringInfoString(buf, "SELECT "); + + if (is_subquery) { + /* + * For a relation that is deparsed as a subquery, emit expressions + * specified in the relation's reltarget. Note that since this is for + * the subquery, no need to care about *retrieved_attrs. + */ + deparseSubqueryTargetList(context); + } else if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel)) { + /* + * For a join or upper relation the input tlist gives the list of + * columns required to be fetched from the foreign server. + */ + deparseExplicitTargetList(tlist, retrieved_attrs, context); + } else { + /* + * For a base relation fpinfo->attrs_used gives the list of columns + * required to be fetched from the foreign server. + */ + RangeTblEntry* rte = planner_rt_fetch(foreignrel->relid, root); + + /* + * Core code already has some lock on each rel being planned, so we + * can use NoLock here. + */ + Relation rel = table_open_compat(rte->relid, NoLock); + + deparseTargetList( + buf, rte, foreignrel->relid, rel, fpinfo->attrs_used, false, retrieved_attrs + ); + table_close_compat(rel, NoLock); + } + elog(DEBUG2, "< %s:%d", __FUNCTION__, __LINE__); } /* @@ -1218,27 +1270,33 @@ deparseSelectSql(List * tlist, bool is_subquery, List * *retrieved_attrs, * (These may or may not include RestrictInfo decoration.) */ static void -deparseFromExpr(List * quals, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - RelOptInfo *scanrel = context->scanrel; - - /* For upper relations, scanrel must be either a joinrel or a baserel */ - Assert(!IS_UPPER_REL(context->foreignrel) || - IS_JOIN_REL(scanrel) || IS_SIMPLE_REL(scanrel)); - - /* Construct FROM clause */ - appendStringInfoString(buf, " FROM "); - deparseFromExprForRel(buf, context->root, scanrel, - (bms_num_members(scanrel->relids) > 1), - (Index) 0, NULL, context->params_list); - - /* Construct WHERE clause */ - if (quals != NIL) - { - appendStringInfoString(buf, " WHERE "); - appendConditions(quals, context); - } +deparseFromExpr(List* quals, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + RelOptInfo* scanrel = context->scanrel; + + /* For upper relations, scanrel must be either a joinrel or a baserel */ + Assert( + !IS_UPPER_REL(context->foreignrel) || IS_JOIN_REL(scanrel) || + IS_SIMPLE_REL(scanrel) + ); + + /* Construct FROM clause */ + appendStringInfoString(buf, " FROM "); + deparseFromExprForRel( + buf, + context->root, + scanrel, + (bms_num_members(scanrel->relids) > 1), + (Index)0, + NULL, + context->params_list + ); + + /* Construct WHERE clause */ + if (quals != NIL) { + appendStringInfoString(buf, " WHERE "); + appendConditions(quals, context); + } } /* @@ -1252,66 +1310,68 @@ deparseFromExpr(List * quals, deparse_expr_cxt * context) * If qualify_col is true, add relation alias before the column name. */ static void -deparseTargetList(StringInfo buf, - RangeTblEntry * rte, - Index rtindex, - Relation rel, - Bitmapset * attrs_used, - bool qualify_col, - List * *retrieved_attrs) -{ - TupleDesc tupdesc = RelationGetDescr(rel); - bool have_wholerow; - bool first; - int i; - - *retrieved_attrs = NIL; - - /* If there's a whole-row reference, we'll need all the columns. */ - have_wholerow = bms_is_member(0 - FirstLowInvalidHeapAttributeNumber, - attrs_used); - - first = true; - for (i = 1; i <= tupdesc->natts; i++) - { - CustomObjectDef *cdef; - Form_pg_attribute attr = TupleDescAttr(tupdesc, i - 1); - - /* Ignore dropped attributes. */ - if (attr->attisdropped) - continue; - - if (have_wholerow || - bms_is_member(i - FirstLowInvalidHeapAttributeNumber, - attrs_used)) - { - if (!first) - appendStringInfoString(buf, ", "); - - first = false; - - cdef = chfdw_check_for_custom_type(attr->atttypid); - deparseColumnRef(buf, cdef, rtindex, i, rte, qualify_col); - - *retrieved_attrs = lappend_int(*retrieved_attrs, i); - } - } - - /* - * check for ctid and oid - */ - if (bms_is_member(SelfItemPointerAttributeNumber - - FirstLowInvalidHeapAttributeNumber, - attrs_used)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("clickhouse does not support system columns"))); - - /* Don't generate bad syntax if no undropped columns */ - if (first) - { - appendStringInfoString(buf, "NULL"); - } +deparseTargetList( + StringInfo buf, + RangeTblEntry* rte, + Index rtindex, + Relation rel, + Bitmapset* attrs_used, + bool qualify_col, + List** retrieved_attrs +) { + TupleDesc tupdesc = RelationGetDescr(rel); + bool have_wholerow; + bool first; + int i; + + *retrieved_attrs = NIL; + + /* If there's a whole-row reference, we'll need all the columns. */ + have_wholerow = bms_is_member(0 - FirstLowInvalidHeapAttributeNumber, attrs_used); + + first = true; + for (i = 1; i <= tupdesc->natts; i++) { + CustomObjectDef* cdef; + Form_pg_attribute attr = TupleDescAttr(tupdesc, i - 1); + + /* Ignore dropped attributes. */ + if (attr->attisdropped) { + continue; + } + + if (have_wholerow || + bms_is_member(i - FirstLowInvalidHeapAttributeNumber, attrs_used)) { + if (!first) { + appendStringInfoString(buf, ", "); + } + + first = false; + + cdef = chfdw_check_for_custom_type(attr->atttypid); + deparseColumnRef(buf, cdef, rtindex, i, rte, qualify_col); + + *retrieved_attrs = lappend_int(*retrieved_attrs, i); + } + } + + /* + * check for ctid and oid + */ + if (bms_is_member( + SelfItemPointerAttributeNumber - FirstLowInvalidHeapAttributeNumber, + attrs_used + )) { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("clickhouse does not support system columns")) + ); + } + + /* Don't generate bad syntax if no undropped columns */ + if (first) { + appendStringInfoString(buf, "NULL"); + } } /* @@ -1324,64 +1384,58 @@ deparseTargetList(StringInfo buf, * or bare clauses. */ static void -appendConditions(List * exprs, deparse_expr_cxt * context) -{ - ListCell *lc; - bool is_first = true; - StringInfo buf = context->buf; - - foreach(lc, exprs) - { - Expr *expr = (Expr *) lfirst(lc); - - /* Extract clause from RestrictInfo, if required */ - if (IsA(expr, RestrictInfo)) - { - expr = ((RestrictInfo *) expr)->clause; - } - - /* Connect expressions with "AND" and parenthesize each condition. */ - if (!is_first) - { - appendStringInfoString(buf, " AND "); - } - - appendStringInfoChar(buf, '('); - deparseExpr(expr, context); - appendStringInfoChar(buf, ')'); - - is_first = false; - } +appendConditions(List* exprs, deparse_expr_cxt* context) { + ListCell* lc; + bool is_first = true; + StringInfo buf = context->buf; + + foreach (lc, exprs) { + Expr* expr = (Expr*)lfirst(lc); + + /* Extract clause from RestrictInfo, if required */ + if (IsA(expr, RestrictInfo)) { + expr = ((RestrictInfo*)expr)->clause; + } + + /* Connect expressions with "AND" and parenthesize each condition. */ + if (!is_first) { + appendStringInfoString(buf, " AND "); + } + + appendStringInfoChar(buf, '('); + deparseExpr(expr, context); + appendStringInfoChar(buf, ')'); + + is_first = false; + } } /* Output join name for given join type */ -const char * -chfdw_get_jointype_name(JoinType jointype) -{ - switch (jointype) - { - case JOIN_INNER: - return "INNER"; +const char* +chfdw_get_jointype_name(JoinType jointype) { + switch (jointype) { + case JOIN_INNER: + return "INNER"; - case JOIN_LEFT: - return "LEFT"; + case JOIN_LEFT: + return "LEFT"; - case JOIN_RIGHT: - return "RIGHT"; + case JOIN_RIGHT: + return "RIGHT"; - case JOIN_FULL: - return "FULL"; + case JOIN_FULL: + return "FULL"; - case JOIN_SEMI: - return "LEFT SEMI"; + case JOIN_SEMI: + return "LEFT SEMI"; - default: - /* Shouldn't come here, but protect from buggy code. */ - elog(ERROR, "unsupported join type %d", jointype); - } + default: + /* Shouldn't come here, but protect from buggy code. */ + elog(ERROR, "unsupported join type %d", jointype); + } - /* Keep compiler happy */ - return NULL; + /* Keep compiler happy */ + return NULL; } /* @@ -1393,31 +1447,33 @@ chfdw_get_jointype_name(JoinType jointype) * from 1. It has same number of entries as tlist. */ static void -deparseExplicitTargetList(List * tlist, - List * *retrieved_attrs, - deparse_expr_cxt * context) -{ - ListCell *lc; - StringInfo buf = context->buf; - int i = 0; +deparseExplicitTargetList( + List* tlist, + List** retrieved_attrs, + deparse_expr_cxt* context +) { + ListCell* lc; + StringInfo buf = context->buf; + int i = 0; - *retrieved_attrs = NIL; + *retrieved_attrs = NIL; - foreach(lc, tlist) - { - TargetEntry *tle = lfirst_node(TargetEntry, lc); + foreach (lc, tlist) { + TargetEntry* tle = lfirst_node(TargetEntry, lc); - if (i > 0) - appendStringInfoString(buf, ", "); + if (i > 0) { + appendStringInfoString(buf, ", "); + } - deparseExpr((Expr *) tle->expr, context); + deparseExpr((Expr*)tle->expr, context); - *retrieved_attrs = lappend_int(*retrieved_attrs, i + 1); - i++; - } + *retrieved_attrs = lappend_int(*retrieved_attrs, i + 1); + i++; + } - if (i == 0) - appendStringInfoString(buf, "NULL"); + if (i == 0) { + appendStringInfoString(buf, "NULL"); + } } /* @@ -1426,32 +1482,32 @@ deparseExplicitTargetList(List * tlist, * This is used for deparsing the given relation as a subquery. */ static void -deparseSubqueryTargetList(deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - RelOptInfo *foreignrel = context->foreignrel; - bool first; - ListCell *lc; +deparseSubqueryTargetList(deparse_expr_cxt* context) { + StringInfo buf = context->buf; + RelOptInfo* foreignrel = context->foreignrel; + bool first; + ListCell* lc; - /* Should only be called in these cases. */ - Assert(IS_SIMPLE_REL(foreignrel) || IS_JOIN_REL(foreignrel)); + /* Should only be called in these cases. */ + Assert(IS_SIMPLE_REL(foreignrel) || IS_JOIN_REL(foreignrel)); - first = true; - foreach(lc, foreignrel->reltarget->exprs) - { - Node *node = (Node *) lfirst(lc); + first = true; + foreach (lc, foreignrel->reltarget->exprs) { + Node* node = (Node*)lfirst(lc); - if (!first) - appendStringInfoString(buf, ", "); + if (!first) { + appendStringInfoString(buf, ", "); + } - first = false; + first = false; - deparseExpr((Expr *) node, context); - } + deparseExpr((Expr*)node, context); + } - /* Don't generate bad syntax if no expressions */ - if (first) - appendStringInfoString(buf, "NULL"); + /* Don't generate bad syntax if no expressions */ + if (first) { + appendStringInfoString(buf, "NULL"); + } } /* @@ -1468,274 +1524,299 @@ deparseSubqueryTargetList(deparse_expr_cxt * context) * the top-level WHERE clause, which is returned to *ignore_conds. */ static void -deparseFromExprForRel(StringInfo buf, PlannerInfo * root, RelOptInfo * foreignrel, - bool use_alias, Index ignore_rel, List * *ignore_conds, - List * *params_list) -{ - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) foreignrel->fdw_private; - - if (IS_JOIN_REL(foreignrel)) - { - StringInfoData join_sql_o; - StringInfoData join_sql_i; - RelOptInfo *outerrel = fpinfo->outerrel; - RelOptInfo *innerrel = fpinfo->innerrel; - bool outerrel_is_target = false; - bool innerrel_is_target = false; - - if (ignore_rel > 0 && bms_is_member(ignore_rel, foreignrel->relids)) - { - /* - * If this is an inner join, add joinclauses to *ignore_conds and - * set it to empty so that those can be deparsed into the WHERE - * clause. Note that since the target relation can never be within - * the nullable side of an outer join, those could safely be - * pulled up into the WHERE clause (see foreign_join_ok()). Note - * also that since the target relation is only inner-joined to any - * other relation in the query, all conditions in the join tree - * mentioning the target relation could be deparsed into the WHERE - * clause by doing this recursively. - */ - if (fpinfo->jointype == JOIN_INNER) - { - *ignore_conds = list_concat(*ignore_conds, - list_copy(fpinfo->joinclauses)); - fpinfo->joinclauses = NIL; - } - - /* - * Check if either of the input relations is the target relation. - */ - if (outerrel->relid == ignore_rel) - { - outerrel_is_target = true; - } - else if (innerrel->relid == ignore_rel) - { - innerrel_is_target = true; - } - } - - /* Deparse outer relation if not the target relation. */ - if (!outerrel_is_target) - { - initStringInfo(&join_sql_o); - deparseRangeTblRef(&join_sql_o, root, outerrel, - fpinfo->make_outerrel_subquery, - ignore_rel, ignore_conds, params_list); - - /* - * If inner relation is the target relation, skip deparsing it. - * Note that since the join of the target relation with any other - * relation in the query is an inner join and can never be within - * the nullable side of an outer join, the join could be - * interchanged with higher-level joins (cf. identity 1 on outer - * join reordering shown in src/backend/optimizer/README), which - * means it's safe to skip the target-relation deparsing here. - */ - if (innerrel_is_target) - { - Assert(fpinfo->jointype == JOIN_INNER); - Assert(fpinfo->joinclauses == NIL); - appendStringInfoString(buf, join_sql_o.data); - return; - } - } - - /* Deparse inner relation if not the target relation. */ - if (!innerrel_is_target) - { - initStringInfo(&join_sql_i); - deparseRangeTblRef(&join_sql_i, root, innerrel, - fpinfo->make_innerrel_subquery, - ignore_rel, ignore_conds, params_list); - - /* - * If outer relation is the target relation, skip deparsing it. - * See the above note about safety. - */ - if (outerrel_is_target) - { - Assert(fpinfo->jointype == JOIN_INNER); - Assert(fpinfo->joinclauses == NIL); - appendStringInfoString(buf, join_sql_i.data); - return; - } - } - - /* Neither of the relations is the target relation. */ - Assert(!outerrel_is_target && !innerrel_is_target); - - /* - * For a join relation FROM clause entry is deparsed as - * - * ((outer relation) (inner relation) ON (joinclauses)) - * - * ClickHouse doesn't use ALL modifier for SEMI joins. - */ - if (fpinfo->jointype == JOIN_SEMI) - appendStringInfo(buf, " %s %s JOIN %s ON ", join_sql_o.data, - chfdw_get_jointype_name(fpinfo->jointype), join_sql_i.data); - else - appendStringInfo(buf, " %s ALL %s JOIN %s ON ", join_sql_o.data, - chfdw_get_jointype_name(fpinfo->jointype), join_sql_i.data); - - /* Append join clause; (TRUE) if no join clause */ - if (fpinfo->joinclauses) - { - deparse_expr_cxt context; - - context.buf = buf; - context.foreignrel = foreignrel; - context.scanrel = foreignrel; - context.root = root; - context.params_list = params_list; - context.func = NULL; - context.interval_op = false; - context.array_as_tuple = false; - context.no_sort_parens = false; - - appendStringInfoChar(buf, '('); - appendConditions(fpinfo->joinclauses, &context); - appendStringInfoChar(buf, ')'); - } - else - { - appendStringInfoString(buf, "(TRUE)"); - } - } - else - { - RangeTblEntry *rte = planner_rt_fetch(foreignrel->relid, root); - - /* - * Core code already has some lock on each rel being planned, so we - * can use NoLock here. - */ - Relation rel = table_open_compat(rte->relid, NoLock); - - deparseRelation(buf, rel); - - /* - * Add a unique alias to avoid any conflict in relation names due to - * pulled up subqueries in the query being built for a pushed down - * join. - */ - if (use_alias) - { - appendStringInfo(buf, " %s%d", REL_ALIAS_PREFIX, foreignrel->relid); - } - - table_close_compat(rel, NoLock); - } +deparseFromExprForRel( + StringInfo buf, + PlannerInfo* root, + RelOptInfo* foreignrel, + bool use_alias, + Index ignore_rel, + List** ignore_conds, + List** params_list +) { + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)foreignrel->fdw_private; + + if (IS_JOIN_REL(foreignrel)) { + StringInfoData join_sql_o; + StringInfoData join_sql_i; + RelOptInfo* outerrel = fpinfo->outerrel; + RelOptInfo* innerrel = fpinfo->innerrel; + bool outerrel_is_target = false; + bool innerrel_is_target = false; + + if (ignore_rel > 0 && bms_is_member(ignore_rel, foreignrel->relids)) { + /* + * If this is an inner join, add joinclauses to *ignore_conds and + * set it to empty so that those can be deparsed into the WHERE + * clause. Note that since the target relation can never be within + * the nullable side of an outer join, those could safely be + * pulled up into the WHERE clause (see foreign_join_ok()). Note + * also that since the target relation is only inner-joined to any + * other relation in the query, all conditions in the join tree + * mentioning the target relation could be deparsed into the WHERE + * clause by doing this recursively. + */ + if (fpinfo->jointype == JOIN_INNER) { + *ignore_conds = + list_concat(*ignore_conds, list_copy(fpinfo->joinclauses)); + fpinfo->joinclauses = NIL; + } + + /* + * Check if either of the input relations is the target relation. + */ + if (outerrel->relid == ignore_rel) { + outerrel_is_target = true; + } else if (innerrel->relid == ignore_rel) { + innerrel_is_target = true; + } + } + + /* Deparse outer relation if not the target relation. */ + if (!outerrel_is_target) { + initStringInfo(&join_sql_o); + deparseRangeTblRef( + &join_sql_o, + root, + outerrel, + fpinfo->make_outerrel_subquery, + ignore_rel, + ignore_conds, + params_list + ); + + /* + * If inner relation is the target relation, skip deparsing it. + * Note that since the join of the target relation with any other + * relation in the query is an inner join and can never be within + * the nullable side of an outer join, the join could be + * interchanged with higher-level joins (cf. identity 1 on outer + * join reordering shown in src/backend/optimizer/README), which + * means it's safe to skip the target-relation deparsing here. + */ + if (innerrel_is_target) { + Assert(fpinfo->jointype == JOIN_INNER); + Assert(fpinfo->joinclauses == NIL); + appendStringInfoString(buf, join_sql_o.data); + return; + } + } + + /* Deparse inner relation if not the target relation. */ + if (!innerrel_is_target) { + initStringInfo(&join_sql_i); + deparseRangeTblRef( + &join_sql_i, + root, + innerrel, + fpinfo->make_innerrel_subquery, + ignore_rel, + ignore_conds, + params_list + ); + + /* + * If outer relation is the target relation, skip deparsing it. + * See the above note about safety. + */ + if (outerrel_is_target) { + Assert(fpinfo->jointype == JOIN_INNER); + Assert(fpinfo->joinclauses == NIL); + appendStringInfoString(buf, join_sql_i.data); + return; + } + } + + /* Neither of the relations is the target relation. */ + Assert(!outerrel_is_target && !innerrel_is_target); + + /* + * For a join relation FROM clause entry is deparsed as + * + * ((outer relation) (inner relation) ON (joinclauses)) + * + * ClickHouse doesn't use ALL modifier for SEMI joins. + */ + if (fpinfo->jointype == JOIN_SEMI) { + appendStringInfo( + buf, + " %s %s JOIN %s ON ", + join_sql_o.data, + chfdw_get_jointype_name(fpinfo->jointype), + join_sql_i.data + ); + } else { + appendStringInfo( + buf, + " %s ALL %s JOIN %s ON ", + join_sql_o.data, + chfdw_get_jointype_name(fpinfo->jointype), + join_sql_i.data + ); + } + + /* Append join clause; (TRUE) if no join clause */ + if (fpinfo->joinclauses) { + deparse_expr_cxt context; + + context.buf = buf; + context.foreignrel = foreignrel; + context.scanrel = foreignrel; + context.root = root; + context.params_list = params_list; + context.func = NULL; + context.interval_op = false; + context.array_as_tuple = false; + context.no_sort_parens = false; + + appendStringInfoChar(buf, '('); + appendConditions(fpinfo->joinclauses, &context); + appendStringInfoChar(buf, ')'); + } else { + appendStringInfoString(buf, "(TRUE)"); + } + } else { + RangeTblEntry* rte = planner_rt_fetch(foreignrel->relid, root); + + /* + * Core code already has some lock on each rel being planned, so we + * can use NoLock here. + */ + Relation rel = table_open_compat(rte->relid, NoLock); + + deparseRelation(buf, rel); + + /* + * Add a unique alias to avoid any conflict in relation names due to + * pulled up subqueries in the query being built for a pushed down + * join. + */ + if (use_alias) { + appendStringInfo(buf, " %s%d", REL_ALIAS_PREFIX, foreignrel->relid); + } + + table_close_compat(rel, NoLock); + } } /* * Append FROM clause entry for the given relation into buf. */ static void -deparseRangeTblRef(StringInfo buf, PlannerInfo * root, RelOptInfo * foreignrel, - bool make_subquery, Index ignore_rel, List * *ignore_conds, - List * *params_list) -{ - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) foreignrel->fdw_private; - - /* Should only be called in these cases. */ - Assert(IS_SIMPLE_REL(foreignrel) || IS_JOIN_REL(foreignrel)); - - Assert(fpinfo->local_conds == NIL); - - /* If make_subquery is true, deparse the relation as a subquery. */ - if (make_subquery) - { - List *retrieved_attrs; - int ncols; - - /* - * The given relation shouldn't contain the target relation, because - * this should only happen for input relations for a full join, and - * such relations can never contain an UPDATE/DELETE target. - */ - Assert(ignore_rel == 0 || - !bms_is_member(ignore_rel, foreignrel->relids)); - - /* Deparse the subquery representing the relation. */ - appendStringInfoChar(buf, '('); - chfdw_deparse_select_stmt_for_rel(buf, root, foreignrel, NIL, - fpinfo->remote_conds, NIL, - false, false, true, - &retrieved_attrs, params_list); - appendStringInfoChar(buf, ')'); - - /* Append the relation alias. */ - appendStringInfo(buf, " %s%d", SUBQUERY_REL_ALIAS_PREFIX, - fpinfo->relation_index); - - /* - * Append the column aliases if needed. Note that the subquery emits - * expressions specified in the relation's reltarget (see - * deparseSubqueryTargetList). - */ - ncols = list_length(foreignrel->reltarget->exprs); - if (ncols > 0) - { - int i; - - appendStringInfoChar(buf, '('); - for (i = 1; i <= ncols; i++) - { - if (i > 1) - { - appendStringInfoString(buf, ", "); - } - - appendStringInfo(buf, "%s%d", SUBQUERY_COL_ALIAS_PREFIX, i); - } - appendStringInfoChar(buf, ')'); - } - } - else - deparseFromExprForRel(buf, root, foreignrel, true, ignore_rel, - ignore_conds, params_list); +deparseRangeTblRef( + StringInfo buf, + PlannerInfo* root, + RelOptInfo* foreignrel, + bool make_subquery, + Index ignore_rel, + List** ignore_conds, + List** params_list +) { + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)foreignrel->fdw_private; + + /* Should only be called in these cases. */ + Assert(IS_SIMPLE_REL(foreignrel) || IS_JOIN_REL(foreignrel)); + + Assert(fpinfo->local_conds == NIL); + + /* If make_subquery is true, deparse the relation as a subquery. */ + if (make_subquery) { + List* retrieved_attrs; + int ncols; + + /* + * The given relation shouldn't contain the target relation, because + * this should only happen for input relations for a full join, and + * such relations can never contain an UPDATE/DELETE target. + */ + Assert(ignore_rel == 0 || !bms_is_member(ignore_rel, foreignrel->relids)); + + /* Deparse the subquery representing the relation. */ + appendStringInfoChar(buf, '('); + chfdw_deparse_select_stmt_for_rel( + buf, + root, + foreignrel, + NIL, + fpinfo->remote_conds, + NIL, + false, + false, + true, + &retrieved_attrs, + params_list + ); + appendStringInfoChar(buf, ')'); + + /* Append the relation alias. */ + appendStringInfo( + buf, " %s%d", SUBQUERY_REL_ALIAS_PREFIX, fpinfo->relation_index + ); + + /* + * Append the column aliases if needed. Note that the subquery emits + * expressions specified in the relation's reltarget (see + * deparseSubqueryTargetList). + */ + ncols = list_length(foreignrel->reltarget->exprs); + if (ncols > 0) { + int i; + + appendStringInfoChar(buf, '('); + for (i = 1; i <= ncols; i++) { + if (i > 1) { + appendStringInfoString(buf, ", "); + } + + appendStringInfo(buf, "%s%d", SUBQUERY_COL_ALIAS_PREFIX, i); + } + appendStringInfoChar(buf, ')'); + } + } else { + deparseFromExprForRel( + buf, root, foreignrel, true, ignore_rel, ignore_conds, params_list + ); + } } /* * deparse remote INSERT statement */ -char * -chfdw_deparse_insert_sql(StringInfo buf, RangeTblEntry * rte, - Index rtindex, Relation rel, - List * targetAttrs) -{ - bool first; - ListCell *lc; - StringInfoData table_name; - - initStringInfo(&table_name); - appendStringInfoString(buf, "INSERT INTO "); - deparseRelation(&table_name, rel); - appendStringInfoString(buf, table_name.data); - - if (targetAttrs) - { - appendStringInfoChar(buf, '('); - - first = true; - foreach(lc, targetAttrs) - { - int attnum = lfirst_int(lc); - - if (!first) - appendStringInfoString(buf, ", "); - - first = false; - - deparseColumnRef(buf, NULL, rtindex, attnum, rte, false); - } - appendStringInfoChar(buf, ')'); - } - - return table_name.data; +char* +chfdw_deparse_insert_sql( + StringInfo buf, + RangeTblEntry* rte, + Index rtindex, + Relation rel, + List* targetAttrs +) { + bool first; + ListCell* lc; + StringInfoData table_name; + + initStringInfo(&table_name); + appendStringInfoString(buf, "INSERT INTO "); + deparseRelation(&table_name, rel); + appendStringInfoString(buf, table_name.data); + + if (targetAttrs) { + appendStringInfoChar(buf, '('); + + first = true; + foreach (lc, targetAttrs) { + int attnum = lfirst_int(lc); + + if (!first) { + appendStringInfoString(buf, ", "); + } + + first = false; + + deparseColumnRef(buf, NULL, rtindex, attnum, rte, false); + } + appendStringInfoChar(buf, ')'); + } + + return table_name.data; } /* @@ -1745,36 +1826,46 @@ chfdw_deparse_insert_sql(StringInfo buf, RangeTblEntry * rte, * If qualify_col is true, qualify column name with the alias of relation. */ static void -deparseColumnRef(StringInfo buf, CustomObjectDef * cdef, - int varno, int varattno, RangeTblEntry * rte, - bool qualify_col) -{ - char *colname = NULL; - CustomColumnInfo *cinfo; - - /* varno must not be any of OUTER_VAR, INNER_VAR and INDEX_VAR. */ - Assert(!IS_SPECIAL_VARNO(varno)); - - if (varattno <= 0) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("ClickHouse does not support system attributes"))); - - /* Get FDW specific options for this column */ - cinfo = chfdw_get_custom_column_info(rte->relid, varattno); - if (cinfo) - colname = cinfo->colname; - - /* - * If it's a column of a regular table or it doesn't have column_name FDW - * option, use attribute name. - */ - if (colname == NULL) - colname = get_attname(rte->relid, varattno, false); - - if (qualify_col) - ADD_REL_QUALIFIER(buf, varno); - appendStringInfoString(buf, quote_identifier(colname)); +deparseColumnRef( + StringInfo buf, + CustomObjectDef* cdef, + int varno, + int varattno, + RangeTblEntry* rte, + bool qualify_col +) { + char* colname = NULL; + CustomColumnInfo* cinfo; + + /* varno must not be any of OUTER_VAR, INNER_VAR and INDEX_VAR. */ + Assert(!IS_SPECIAL_VARNO(varno)); + + if (varattno <= 0) { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ClickHouse does not support system attributes")) + ); + } + + /* Get FDW specific options for this column */ + cinfo = chfdw_get_custom_column_info(rte->relid, varattno); + if (cinfo) { + colname = cinfo->colname; + } + + /* + * If it's a column of a regular table or it doesn't have column_name FDW + * option, use attribute name. + */ + if (colname == NULL) { + colname = get_attname(rte->relid, varattno, false); + } + + if (qualify_col) { + ADD_REL_QUALIFIER(buf, varno); + } + appendStringInfoString(buf, quote_identifier(colname)); } /* @@ -1783,54 +1874,48 @@ deparseColumnRef(StringInfo buf, CustomObjectDef * cdef, * Similarly, schema_name FDW option overrides schema name. */ static void -deparseRelation(StringInfo buf, Relation rel) -{ - ForeignTable *table; - const char *relname = NULL; - char *dbname = "default"; - ForeignServer *server = chfdw_get_foreign_server(rel); - ListCell *lc; - - chfdw_extract_options(server->options, NULL, NULL, NULL, &dbname, NULL, NULL, NULL, NULL, NULL); - - /* obtain additional catalog information. */ - table = GetForeignTable(RelationGetRelid(rel)); - - /* - * Use value of FDW options if any, instead of the name of object itself. - */ - foreach(lc, table->options) - { - DefElem *def = (DefElem *) lfirst(lc); - - if (strcmp(def->defname, "table_name") == 0) - { - relname = defGetString(def); - } - else if (strcmp(def->defname, "database") == 0) - { - dbname = defGetString(def); - } - } - if (relname == NULL) - { - relname = RelationGetRelationName(rel); - } - - appendStringInfo(buf, "%s.%s", quote_identifier(dbname), - quote_identifier(relname)); +deparseRelation(StringInfo buf, Relation rel) { + ForeignTable* table; + const char* relname = NULL; + char* dbname = "default"; + ForeignServer* server = chfdw_get_foreign_server(rel); + ListCell* lc; + + chfdw_extract_options( + server->options, NULL, NULL, NULL, &dbname, NULL, NULL, NULL, NULL, NULL + ); + + /* obtain additional catalog information. */ + table = GetForeignTable(RelationGetRelid(rel)); + + /* + * Use value of FDW options if any, instead of the name of object itself. + */ + foreach (lc, table->options) { + DefElem* def = (DefElem*)lfirst(lc); + + if (strcmp(def->defname, "table_name") == 0) { + relname = defGetString(def); + } else if (strcmp(def->defname, "database") == 0) { + dbname = defGetString(def); + } + } + if (relname == NULL) { + relname = RelationGetRelationName(rel); + } + + appendStringInfo(buf, "%s.%s", quote_identifier(dbname), quote_identifier(relname)); } /* * Append a SQL string literal representing "val" to buf. */ static void -deparseStringLiteral(StringInfo buf, const char *val) -{ - char *quoted = ch_quote_literal(val); +deparseStringLiteral(StringInfo buf, const char* val) { + char* quoted = ch_quote_literal(val); - appendStringInfoString(buf, quoted); - pfree(quoted); + appendStringInfoString(buf, quoted); + pfree(quoted); } /* @@ -1844,86 +1929,82 @@ deparseStringLiteral(StringInfo buf, const char *val) * should be self-parenthesized. */ static void -deparseExpr(Expr * node, deparse_expr_cxt * context) -{ - if (node == NULL) - { - return; - } - - switch (nodeTag(node)) - { - case T_Var: - deparseVar((Var *) node, context); - break; - case T_Const: - deparseConst((Const *) node, context, 0); - break; - case T_Param: - deparseParam((Param *) node, context); - break; - case T_SubscriptingRef: - deparseSubscriptingRef((SubscriptingRef *) node, context); - break; - case T_FuncExpr: - deparseFuncExpr((FuncExpr *) node, context); - break; - case T_SQLValueFunction: - deparseSQLValueFunction((SQLValueFunction *) node, context); - break; - case T_OpExpr: - deparseOpExpr((OpExpr *) node, context); - break; - case T_DistinctExpr: - deparseDistinctExpr((DistinctExpr *) node, context); - break; - case T_NullIfExpr: - deparseNullIfExpr((NullIfExpr *) node, context); - break; - case T_ScalarArrayOpExpr: - deparseScalarArrayOpExpr((ScalarArrayOpExpr *) node, context); - break; - case T_RelabelType: - deparseRelabelType((RelabelType *) node, context); - break; - case T_BoolExpr: - deparseBoolExpr((BoolExpr *) node, context); - break; - case T_NullTest: - deparseNullTest((NullTest *) node, context); - break; - case T_ArrayExpr: - deparseArrayExpr((ArrayExpr *) node, context); - break; - case T_Aggref: - deparseAggref((Aggref *) node, context); - break; - case T_WindowFunc: - deparseWindowFunc((WindowFunc *) node, context); - break; - case T_CaseExpr: - deparseCaseExpr((CaseExpr *) node, context); - break; - case T_CaseWhen: - deparseCaseWhen((CaseWhen *) node, context); - break; - case T_CoalesceExpr: - deparseCoalesceExpr((CoalesceExpr *) node, context); - break; - case T_MinMaxExpr: - deparseMinMaxExpr((MinMaxExpr *) node, context); - break; - case T_CoerceViaIO: - deparseCoerceViaIO((CoerceViaIO *) node, context); - break; - case T_RowExpr: - deparseRowExpr((RowExpr *) node, context); - break; - default: - elog(ERROR, "unsupported expression type for deparse: %d", - (int) nodeTag(node)); - break; - } +deparseExpr(Expr* node, deparse_expr_cxt* context) { + if (node == NULL) { + return; + } + + switch (nodeTag(node)) { + case T_Var: + deparseVar((Var*)node, context); + break; + case T_Const: + deparseConst((Const*)node, context, 0); + break; + case T_Param: + deparseParam((Param*)node, context); + break; + case T_SubscriptingRef: + deparseSubscriptingRef((SubscriptingRef*)node, context); + break; + case T_FuncExpr: + deparseFuncExpr((FuncExpr*)node, context); + break; + case T_SQLValueFunction: + deparseSQLValueFunction((SQLValueFunction*)node, context); + break; + case T_OpExpr: + deparseOpExpr((OpExpr*)node, context); + break; + case T_DistinctExpr: + deparseDistinctExpr((DistinctExpr*)node, context); + break; + case T_NullIfExpr: + deparseNullIfExpr((NullIfExpr*)node, context); + break; + case T_ScalarArrayOpExpr: + deparseScalarArrayOpExpr((ScalarArrayOpExpr*)node, context); + break; + case T_RelabelType: + deparseRelabelType((RelabelType*)node, context); + break; + case T_BoolExpr: + deparseBoolExpr((BoolExpr*)node, context); + break; + case T_NullTest: + deparseNullTest((NullTest*)node, context); + break; + case T_ArrayExpr: + deparseArrayExpr((ArrayExpr*)node, context); + break; + case T_Aggref: + deparseAggref((Aggref*)node, context); + break; + case T_WindowFunc: + deparseWindowFunc((WindowFunc*)node, context); + break; + case T_CaseExpr: + deparseCaseExpr((CaseExpr*)node, context); + break; + case T_CaseWhen: + deparseCaseWhen((CaseWhen*)node, context); + break; + case T_CoalesceExpr: + deparseCoalesceExpr((CoalesceExpr*)node, context); + break; + case T_MinMaxExpr: + deparseMinMaxExpr((MinMaxExpr*)node, context); + break; + case T_CoerceViaIO: + deparseCoerceViaIO((CoerceViaIO*)node, context); + break; + case T_RowExpr: + deparseRowExpr((RowExpr*)node, context); + break; + default: + elog(ERROR, "unsupported expression type for deparse: %d", (int)nodeTag(node)); + break; + } } /* @@ -1934,187 +2015,188 @@ deparseExpr(Expr * node, deparse_expr_cxt * context) * run time). Handle it the same way we handle plain Params. */ static void -deparseVar(Var * node, deparse_expr_cxt * context) -{ - CustomObjectDef *cdef; - Relids relids = context->scanrel->relids; - int relno; - int colno; - - /* Qualify columns when multiple relations are involved. */ - bool qualify_col = (bms_num_members(relids) > 1); - - /* - * If the Var belongs to the foreign relation that is deparsed as a - * subquery, use the relation and column alias to the Var provided by the - * subquery, instead of the remote name. - */ - if (is_subquery_var(node, context->scanrel, &relno, &colno)) - { - appendStringInfo(context->buf, "%s%d.%s%d", - SUBQUERY_REL_ALIAS_PREFIX, relno, - SUBQUERY_COL_ALIAS_PREFIX, colno); - return; - } - - cdef = context->func; - if (!cdef) - cdef = chfdw_check_for_custom_type(node->vartype); - - if (bms_is_member(node->varno, relids) && node->varlevelsup == 0) - deparseColumnRef(context->buf, cdef, - node->varno, node->varattno, - planner_rt_fetch(node->varno, context->root), - qualify_col); - else - { - /* Treat like a Param */ - if (context->params_list) - { - int pindex = 0; - ListCell *lc; - - /* find its index in params_list */ - foreach(lc, *context->params_list) - { - pindex++; - if (equal(node, (Node *) lfirst(lc))) - break; - } - if (lc == NULL) - { - /* not in list, so add it */ - pindex++; - *context->params_list = lappend(*context->params_list, node); - } - - printRemoteParam(pindex, node->vartype, node->vartypmod, context); - } - else - { - printRemotePlaceholder(node->vartype, node->vartypmod, context); - } - } +deparseVar(Var* node, deparse_expr_cxt* context) { + CustomObjectDef* cdef; + Relids relids = context->scanrel->relids; + int relno; + int colno; + + /* Qualify columns when multiple relations are involved. */ + bool qualify_col = (bms_num_members(relids) > 1); + + /* + * If the Var belongs to the foreign relation that is deparsed as a + * subquery, use the relation and column alias to the Var provided by the + * subquery, instead of the remote name. + */ + if (is_subquery_var(node, context->scanrel, &relno, &colno)) { + appendStringInfo( + context->buf, + "%s%d.%s%d", + SUBQUERY_REL_ALIAS_PREFIX, + relno, + SUBQUERY_COL_ALIAS_PREFIX, + colno + ); + return; + } + + cdef = context->func; + if (!cdef) { + cdef = chfdw_check_for_custom_type(node->vartype); + } + + if (bms_is_member(node->varno, relids) && node->varlevelsup == 0) { + deparseColumnRef( + context->buf, + cdef, + node->varno, + node->varattno, + planner_rt_fetch(node->varno, context->root), + qualify_col + ); + } else { + /* Treat like a Param */ + if (context->params_list) { + int pindex = 0; + ListCell* lc; + + /* find its index in params_list */ + foreach (lc, *context->params_list) { + pindex++; + if (equal(node, (Node*)lfirst(lc))) { + break; + } + } + if (lc == NULL) { + /* not in list, so add it */ + pindex++; + *context->params_list = lappend(*context->params_list, node); + } + + printRemoteParam(pindex, node->vartype, node->vartypmod, context); + } else { + printRemotePlaceholder(node->vartype, node->vartypmod, context); + } + } } -#define USE_ISO_DATES 1 +#define USE_ISO_DATES 1 Datum -ch_time_out(PG_FUNCTION_ARGS) -{ - TimeADT time = PG_GETARG_TIMEADT(0); - char *result; - struct pg_tm tt, - *tm = &tt; - fsec_t fsec; - char buf[MAXDATELEN + 1]; - - time2tm(time, tm, &fsec); - EncodeTimeOnly(tm, fsec, false, 0, USE_ISO_DATES, buf); - - result = pstrdup(buf); - PG_RETURN_CSTRING(result); +ch_time_out(PG_FUNCTION_ARGS) { + TimeADT time = PG_GETARG_TIMEADT(0); + char* result; + struct pg_tm tt, *tm = &tt; + fsec_t fsec; + char buf[MAXDATELEN + 1]; + + time2tm(time, tm, &fsec); + EncodeTimeOnly(tm, fsec, false, 0, USE_ISO_DATES, buf); + + result = pstrdup(buf); + PG_RETURN_CSTRING(result); } /* date_out() * Given internal format date, convert to text string. */ Datum -ch_date_out(PG_FUNCTION_ARGS) -{ - DateADT date = PG_GETARG_DATEADT(0); - char *result; - struct pg_tm tt, - *tm = &tt; - char buf[MAXDATELEN + 1]; - - if (DATE_NOT_FINITE(date)) - EncodeSpecialDate(date, buf); - else - { - j2date(date + POSTGRES_EPOCH_JDATE, - &(tm->tm_year), &(tm->tm_mon), &(tm->tm_mday)); - EncodeDateOnly(tm, USE_ISO_DATES, buf); - } - - result = pstrdup(buf); - PG_RETURN_CSTRING(result); +ch_date_out(PG_FUNCTION_ARGS) { + DateADT date = PG_GETARG_DATEADT(0); + char* result; + struct pg_tm tt, *tm = &tt; + char buf[MAXDATELEN + 1]; + + if (DATE_NOT_FINITE(date)) { + EncodeSpecialDate(date, buf); + } else { + j2date( + date + POSTGRES_EPOCH_JDATE, &(tm->tm_year), &(tm->tm_mon), &(tm->tm_mday) + ); + EncodeDateOnly(tm, USE_ISO_DATES, buf); + } + + result = pstrdup(buf); + PG_RETURN_CSTRING(result); } Datum -ch_timestamp_out(PG_FUNCTION_ARGS) -{ - Timestamp timestamp = PG_GETARG_TIMESTAMP(0); - char *result; - struct pg_tm tt, - *tm = &tt; - fsec_t fsec; - char buf[MAXDATELEN + 1]; - - if (TIMESTAMP_NOT_FINITE(timestamp)) - EncodeSpecialTimestamp(timestamp, buf); - else if (timestamp2tm(timestamp, NULL, tm, &fsec, NULL, NULL) == 0) - /* we ignore fractional seconds */ - EncodeDateTime(tm, 0, false, 0, NULL, USE_ISO_DATES, buf); - else - ereport(ERROR, - (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), - errmsg("timestamp out of range"))); - - result = pstrdup(buf); - PG_RETURN_CSTRING(result); +ch_timestamp_out(PG_FUNCTION_ARGS) { + Timestamp timestamp = PG_GETARG_TIMESTAMP(0); + char* result; + struct pg_tm tt, *tm = &tt; + fsec_t fsec; + char buf[MAXDATELEN + 1]; + + if (TIMESTAMP_NOT_FINITE(timestamp)) { + EncodeSpecialTimestamp(timestamp, buf); + } else if (timestamp2tm(timestamp, NULL, tm, &fsec, NULL, NULL) == 0) { + /* we ignore fractional seconds */ + EncodeDateTime(tm, 0, false, 0, NULL, USE_ISO_DATES, buf); + } else { + ereport( + ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range")) + ); + } + + result = pstrdup(buf); + PG_RETURN_CSTRING(result); } static void -emitArrayElement(StringInfo buf, Datum elt, bool isnull, Oid element_type, - Oid typiofunc) -{ - char *extval; - - if (isnull) - { - appendStringInfoString(buf, "NULL"); - return; - } - - extval = OidOutputFunctionCall(typiofunc, elt); - - switch (element_type) - { - case INT2OID: - case INT4OID: - case INT8OID: - case OIDOID: - case FLOAT4OID: - case FLOAT8OID: - case NUMERICOID: - { - /* - * No need to quote unless it's a special value such as 'NaN'. - * See comments in get_const_expr(). - */ - if (strspn(extval, "0123456789+-eE.") == strlen(extval)) - { - if (extval[0] == '+' || extval[0] == '-') - appendStringInfo(buf, "(%s)", extval); - else - appendStringInfoString(buf, extval); - } - else - appendStringInfo(buf, "'%s'", extval); - } - break; - case BOOLOID: - if (strcmp(extval, "t") == 0) - appendStringInfoString(buf, "true"); - else - appendStringInfoString(buf, "false"); - break; - default: - deparseStringLiteral(buf, extval); - break; - } - pfree(extval); +emitArrayElement( + StringInfo buf, + Datum elt, + bool isnull, + Oid element_type, + Oid typiofunc +) { + char* extval; + + if (isnull) { + appendStringInfoString(buf, "NULL"); + return; + } + + extval = OidOutputFunctionCall(typiofunc, elt); + + switch (element_type) { + case INT2OID: + case INT4OID: + case INT8OID: + case OIDOID: + case FLOAT4OID: + case FLOAT8OID: + case NUMERICOID: { + /* + * No need to quote unless it's a special value such as 'NaN'. + * See comments in get_const_expr(). + */ + if (strspn(extval, "0123456789+-eE.") == strlen(extval)) { + if (extval[0] == '+' || extval[0] == '-') { + appendStringInfo(buf, "(%s)", extval); + } else { + appendStringInfoString(buf, extval); + } + } else { + appendStringInfo(buf, "'%s'", extval); + } + } break; + case BOOLOID: + if (strcmp(extval, "t") == 0) { + appendStringInfoString(buf, "true"); + } else { + appendStringInfoString(buf, "false"); + } + break; + default: + deparseStringLiteral(buf, extval); + break; + } + pfree(extval); } /* @@ -2123,115 +2205,149 @@ emitArrayElement(StringInfo buf, Datum elt, bool isnull, Oid element_type, * in row-major order, matching postgres' flat element layout. */ static void -emitArrayLevel(StringInfo buf, int level, int ndims, int *dims, array_iter * iter, - Oid element_type, int16 typlen, bool typbyval, char typalign, - Oid typiofunc, int *nleaf) -{ - appendStringInfoChar(buf, '['); - for (int i = 0; i < dims[level]; i++) - { - if (i > 0) - appendStringInfoChar(buf, ','); - - if (level + 1 < ndims) - emitArrayLevel(buf, level + 1, ndims, dims, iter, element_type, - typlen, typbyval, typalign, typiofunc, nleaf); - else - { - Datum elt; - bool isnull; - int n = (*nleaf)++; +emitArrayLevel( + StringInfo buf, + int level, + int ndims, + int* dims, + array_iter* iter, + Oid element_type, + int16 typlen, + bool typbyval, + char typalign, + Oid typiofunc, + int* nleaf +) { + appendStringInfoChar(buf, '['); + for (int i = 0; i < dims[level]; i++) { + if (i > 0) { + appendStringInfoChar(buf, ','); + } + + if (level + 1 < ndims) { + emitArrayLevel( + buf, + level + 1, + ndims, + dims, + iter, + element_type, + typlen, + typbyval, + typalign, + typiofunc, + nleaf + ); + } else { + Datum elt; + bool isnull; + int n = (*nleaf)++; #if PG_VERSION_NUM < 190000 - elt = array_iter_next(iter, &isnull, n, typlen, typbyval, typalign); + elt = array_iter_next(iter, &isnull, n, typlen, typbyval, typalign); #else - elt = array_iter_next(iter, &isnull, n); + elt = array_iter_next(iter, &isnull, n); #endif - emitArrayElement(buf, elt, isnull, element_type, typiofunc); - } - } - appendStringInfoChar(buf, ']'); + emitArrayElement(buf, elt, isnull, element_type, typiofunc); + } + } + appendStringInfoChar(buf, ']'); } static void -deparseArray(Datum arr, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - AnyArrayType *array = DatumGetAnyArrayP(arr); - int ndims = AARR_NDIM(array); - int *dims = AARR_DIMS(array); - Oid element_type = AARR_ELEMTYPE(array); - - int16 typlen; - bool typbyval; - char typalign; - char typdelim; - Oid typioparam; - Oid typiofunc; - array_iter iter; - - if (context->array_as_tuple && ndims > 1) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_clickhouse: tuple-formatted arrays must be one-dimensional"))); - - get_type_io_data(element_type, IOFunc_output, - &typlen, &typbyval, - &typalign, &typdelim, - &typioparam, &typiofunc); +deparseArray(Datum arr, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + AnyArrayType* array = DatumGetAnyArrayP(arr); + int ndims = AARR_NDIM(array); + int* dims = AARR_DIMS(array); + Oid element_type = AARR_ELEMTYPE(array); + + int16 typlen; + bool typbyval; + char typalign; + char typdelim; + Oid typioparam; + Oid typiofunc; + array_iter iter; + + if (context->array_as_tuple && ndims > 1) { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("pg_clickhouse: tuple-formatted arrays must be one-dimensional")) + ); + } + + get_type_io_data( + element_type, + IOFunc_output, + &typlen, + &typbyval, + &typalign, + &typdelim, + &typioparam, + &typiofunc + ); #if PG_VERSION_NUM < 190000 - array_iter_setup(&iter, array); + array_iter_setup(&iter, array); #else - array_iter_setup(&iter, array, typlen, typbyval, typalign); + array_iter_setup(&iter, array, typlen, typbyval, typalign); #endif - if (ndims > 1) - { - int nleaf = 0; - - emitArrayLevel(buf, 0, ndims, dims, &iter, element_type, - typlen, typbyval, typalign, typiofunc, &nleaf); - } - else - { - int nitems = ArrayGetNItems(ndims, dims); - char open = context->array_as_tuple ? '(' : '['; - char close = context->array_as_tuple ? ')' : ']'; - - appendStringInfoChar(buf, open); - for (int i = 0; i < nitems; i++) - { - Datum elt; - bool isnull; - - if (i > 0) - appendStringInfoChar(buf, ','); + if (ndims > 1) { + int nleaf = 0; + + emitArrayLevel( + buf, + 0, + ndims, + dims, + &iter, + element_type, + typlen, + typbyval, + typalign, + typiofunc, + &nleaf + ); + } else { + int nitems = ArrayGetNItems(ndims, dims); + char open = context->array_as_tuple ? '(' : '['; + char close = context->array_as_tuple ? ')' : ']'; + + appendStringInfoChar(buf, open); + for (int i = 0; i < nitems; i++) { + Datum elt; + bool isnull; + + if (i > 0) { + appendStringInfoChar(buf, ','); + } #if PG_VERSION_NUM < 190000 - elt = array_iter_next(&iter, &isnull, i, typlen, typbyval, typalign); + elt = array_iter_next(&iter, &isnull, i, typlen, typbyval, typalign); #else - elt = array_iter_next(&iter, &isnull, i); + elt = array_iter_next(&iter, &isnull, i); #endif - emitArrayElement(buf, elt, isnull, element_type, typiofunc); - } - appendStringInfoChar(buf, close); - } + emitArrayElement(buf, elt, isnull, element_type, typiofunc); + } + appendStringInfoChar(buf, close); + } } /* * Exported function to convert an array to a ClickHouse string literal array. */ -char * -chfdw_array_to_ch_literal(Datum arr) -{ +char* +chfdw_array_to_ch_literal(Datum arr) { - deparse_expr_cxt context; + deparse_expr_cxt context; - context.array_as_tuple = false; - context.buf = makeStringInfo(); - deparseArray(arr, &context); - return context.buf->data; + context.array_as_tuple = false; + context.buf = makeStringInfo(); + deparseArray(arr, &context); + return context.buf->data; } /* @@ -2243,146 +2359,139 @@ chfdw_array_to_ch_literal(Datum arr) * to be the right type by default. */ static void -deparseConst(Const * node, deparse_expr_cxt * context, int showtype) -{ - StringInfo buf = context->buf; - Oid typoutput; - bool typIsVarlena; - char *extval = NULL; - bool closebr = false; - - if (node->constisnull) - { - appendStringInfoString(buf, "NULL"); - return; - } - - if (showtype > 0) - appendStringInfoString(buf, "cast("); - - getTypeOutputInfo(node->consttype, - &typoutput, &typIsVarlena); - - if (typoutput == F_TIMESTAMPTZ_OUT || typoutput == F_TIMESTAMP_OUT) - { - /* - * We use our own function here, that removes fractional seconds since - * there are not supported in clickhouse - */ - extval = DatumGetCString(DirectFunctionCall1(ch_timestamp_out, node->constvalue)); - } - else if (typoutput == F_INTERVAL_OUT) - { - /* - * basically we can't convert month part since we should know about - * related timestamp first. - * - * for other types we just convert to seconds. - */ - uint64 sec; - Interval *ival = DatumGetIntervalP(node->constvalue); - char bufint8[MAXINT8LEN + 1]; - - if (ival->month != 0) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot convert interval with months into clickhouse"))); - - sec = 86400 /* sec in day */ * ival->day + (int64) (ival->time / 1000000); - pg_lltoa(sec, bufint8); - appendStringInfoString(buf, bufint8); - goto cleanup; - } - else if (typoutput == F_ARRAY_OUT) - { - deparseArray(node->constvalue, context); - goto cleanup; - } - else if (node->consttype == BYTEAOID) - { - /* - * Emit printable ASCII as-is and \xHH for everything else. PG's - * bytea_out hex format (\xHHHH...) doesn't round-trip through CH's - * string lexer past a single byte: \x consumes exactly two hex digits - * and any remaining hex chars become ASCII literals. - */ - bytea *bp = DatumGetByteaPP(node->constvalue); - const char *bytes = VARDATA_ANY(bp); - int len = VARSIZE_ANY_EXHDR(bp); - - appendStringInfoChar(buf, '\''); - for (int i = 0; i < len; i++) - { - unsigned char c = (unsigned char) bytes[i]; - - if (c == '\'') - appendStringInfoString(buf, "\\'"); - else if (c == '\\') - appendStringInfoString(buf, "\\\\"); - else if (c >= 0x20 && c <= 0x7E) - appendStringInfoChar(buf, (char) c); - else - appendStringInfo(buf, "\\x%02x", c); - } - appendStringInfoChar(buf, '\''); - goto cleanup; - } - else - { - extval = OidOutputFunctionCall(typoutput, node->constvalue); - } - - switch (node->consttype) - { - case INT2OID: - case INT4OID: - case INT8OID: - case OIDOID: - case FLOAT4OID: - case FLOAT8OID: - case NUMERICOID: - { - /* - * No need to quote unless it's a special value such as 'NaN'. - * See comments in get_const_expr(). - */ - if (strspn(extval, "0123456789+-eE.") == strlen(extval)) - { - if (extval[0] == '+' || extval[0] == '-') - appendStringInfo(buf, "(%s)", extval); - else - appendStringInfoString(buf, extval); - } - else - deparseStringLiteral(buf, extval); - } - break; - case BITOID: - case VARBITOID: - appendStringInfo(buf, "B'%s'", extval); - break; - case BOOLOID: - if (strcmp(extval, "t") == 0) - appendStringInfoChar(buf, '1'); - else - appendStringInfoChar(buf, '0'); - break; - default: - deparseStringLiteral(buf, extval); - break; - } - - if (closebr) - appendStringInfoChar(buf, ')'); +deparseConst(Const* node, deparse_expr_cxt* context, int showtype) { + StringInfo buf = context->buf; + Oid typoutput; + bool typIsVarlena; + char* extval = NULL; + bool closebr = false; + + if (node->constisnull) { + appendStringInfoString(buf, "NULL"); + return; + } + + if (showtype > 0) { + appendStringInfoString(buf, "cast("); + } + + getTypeOutputInfo(node->consttype, &typoutput, &typIsVarlena); + + if (typoutput == F_TIMESTAMPTZ_OUT || typoutput == F_TIMESTAMP_OUT) { + /* + * We use our own function here, that removes fractional seconds since + * there are not supported in clickhouse + */ + extval = + DatumGetCString(DirectFunctionCall1(ch_timestamp_out, node->constvalue)); + } else if (typoutput == F_INTERVAL_OUT) { + /* + * basically we can't convert month part since we should know about + * related timestamp first. + * + * for other types we just convert to seconds. + */ + uint64 sec; + Interval* ival = DatumGetIntervalP(node->constvalue); + char bufint8[MAXINT8LEN + 1]; + + if (ival->month != 0) { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot convert interval with months into clickhouse")) + ); + } + + sec = 86400 /* sec in day */ * ival->day + (int64)(ival->time / 1000000); + pg_lltoa(sec, bufint8); + appendStringInfoString(buf, bufint8); + goto cleanup; + } else if (typoutput == F_ARRAY_OUT) { + deparseArray(node->constvalue, context); + goto cleanup; + } else if (node->consttype == BYTEAOID) { + /* + * Emit printable ASCII as-is and \xHH for everything else. PG's + * bytea_out hex format (\xHHHH...) doesn't round-trip through CH's + * string lexer past a single byte: \x consumes exactly two hex digits + * and any remaining hex chars become ASCII literals. + */ + bytea* bp = DatumGetByteaPP(node->constvalue); + const char* bytes = VARDATA_ANY(bp); + int len = VARSIZE_ANY_EXHDR(bp); + + appendStringInfoChar(buf, '\''); + for (int i = 0; i < len; i++) { + unsigned char c = (unsigned char)bytes[i]; + + if (c == '\'') { + appendStringInfoString(buf, "\\'"); + } else if (c == '\\') { + appendStringInfoString(buf, "\\\\"); + } else if (c >= 0x20 && c <= 0x7E) { + appendStringInfoChar(buf, (char)c); + } else { + appendStringInfo(buf, "\\x%02x", c); + } + } + appendStringInfoChar(buf, '\''); + goto cleanup; + } else { + extval = OidOutputFunctionCall(typoutput, node->constvalue); + } + + switch (node->consttype) { + case INT2OID: + case INT4OID: + case INT8OID: + case OIDOID: + case FLOAT4OID: + case FLOAT8OID: + case NUMERICOID: { + /* + * No need to quote unless it's a special value such as 'NaN'. + * See comments in get_const_expr(). + */ + if (strspn(extval, "0123456789+-eE.") == strlen(extval)) { + if (extval[0] == '+' || extval[0] == '-') { + appendStringInfo(buf, "(%s)", extval); + } else { + appendStringInfoString(buf, extval); + } + } else { + deparseStringLiteral(buf, extval); + } + } break; + case BITOID: + case VARBITOID: + appendStringInfo(buf, "B'%s'", extval); + break; + case BOOLOID: + if (strcmp(extval, "t") == 0) { + appendStringInfoChar(buf, '1'); + } else { + appendStringInfoChar(buf, '0'); + } + break; + default: + deparseStringLiteral(buf, extval); + break; + } + + if (closebr) { + appendStringInfoChar(buf, ')'); + } cleanup: - if (showtype > 0) - appendStringInfo(buf, " as %s)", - deparse_type_name(node->consttype, - node->consttypmod)); - if (extval) - pfree(extval); - + if (showtype > 0) { + appendStringInfo( + buf, " as %s)", deparse_type_name(node->consttype, node->consttypmod) + ); + } + if (extval) { + pfree(extval); + } } /* @@ -2394,33 +2503,28 @@ deparseConst(Const * node, deparse_expr_cxt * context, int showtype) * no need to identify a parameter number. */ static void -deparseParam(Param * node, deparse_expr_cxt * context) -{ - if (context->params_list) - { - int pindex = 0; - ListCell *lc; - - /* find its index in params_list */ - foreach(lc, *context->params_list) - { - pindex++; - if (equal(node, (Node *) lfirst(lc))) - break; - } - if (lc == NULL) - { - /* not in list, so add it */ - pindex++; - *context->params_list = lappend(*context->params_list, node); - } - - printRemoteParam(pindex, node->paramtype, node->paramtypmod, context); - } - else - { - printRemotePlaceholder(node->paramtype, node->paramtypmod, context); - } +deparseParam(Param* node, deparse_expr_cxt* context) { + if (context->params_list) { + int pindex = 0; + ListCell* lc; + + /* find its index in params_list */ + foreach (lc, *context->params_list) { + pindex++; + if (equal(node, (Node*)lfirst(lc))) { + break; + } + } + if (lc == NULL) { + /* not in list, so add it */ + pindex++; + *context->params_list = lappend(*context->params_list, node); + } + + printRemoteParam(pindex, node->paramtype, node->paramtypmod, context); + } else { + printRemotePlaceholder(node->paramtype, node->paramtypmod, context); + } } /* @@ -2428,13 +2532,16 @@ deparseParam(Param * node, deparse_expr_cxt * context) * by param number and remote data type. */ static void -printRemoteParam(int paramindex, Oid paramtype, int32 paramtypmod, - deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - char *ptypename = deparse_type_name(paramtype, paramtypmod); - - appendStringInfo(buf, "{p%d:%s}", paramindex, ptypename); +printRemoteParam( + int paramindex, + Oid paramtype, + int32 paramtypmod, + deparse_expr_cxt* context +) { + StringInfo buf = context->buf; + char* ptypename = deparse_type_name(paramtype, paramtypmod); + + appendStringInfo(buf, "{p%d:%s}", paramindex, ptypename); } /* @@ -2451,72 +2558,66 @@ printRemoteParam(int paramindex, Oid paramtype, int32 paramtypmod, * subqueries. */ static void -printRemotePlaceholder(Oid paramtype, int32 paramtypmod, - deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - char *ptypename = deparse_type_name(paramtype, paramtypmod); +printRemotePlaceholder(Oid paramtype, int32 paramtypmod, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + char* ptypename = deparse_type_name(paramtype, paramtypmod); - appendStringInfo(buf, "((SELECT CAST(null AS Nullable(%s))", ptypename); + appendStringInfo(buf, "((SELECT CAST(null AS Nullable(%s))", ptypename); } /* * Deparse an array subscript expression. */ static void -deparseSubscriptingRef(SubscriptingRef * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - - if (node->reflowerindexpr != NIL) - { - /* - * Slice: CH doesn't support [L:U] syntax, emit arraySlice(). PG slice - * is inclusive both ends, CH arraySlice takes (arr, offset, length). - * Only 1D arrays are supported (enforced elsewhere). - */ - Expr *lower = (Expr *) linitial(node->reflowerindexpr); - Expr *upper = (Expr *) linitial(node->refupperindexpr); - - appendStringInfoString(buf, "arraySlice("); - deparseExpr(node->refexpr, context); - appendStringInfoString(buf, ", "); - - if (lower) - deparseExpr(lower, context); - else - appendStringInfoChar(buf, '1'); - - if (upper) - { - appendStringInfoString(buf, ", ("); - deparseExpr(upper, context); - appendStringInfoString(buf, ") - ("); - if (lower) - deparseExpr(lower, context); - else - appendStringInfoChar(buf, '1'); - appendStringInfoString(buf, ") + 1"); - } - - appendStringInfoChar(buf, ')'); - } - else - { - /* Single element: emit arr[idx] */ - appendStringInfoChar(buf, '('); - if (IsA(node->refexpr, Var)) - deparseExpr(node->refexpr, context); - else - { - appendStringInfoChar(buf, '('); - deparseExpr(node->refexpr, context); - appendStringInfoChar(buf, ')'); - } - appendStringInfoChar(buf, '['); - deparseExpr((Expr *) linitial(node->refupperindexpr), context); - appendStringInfoString(buf, "])"); - } +deparseSubscriptingRef(SubscriptingRef* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + + if (node->reflowerindexpr != NIL) { + /* + * Slice: CH doesn't support [L:U] syntax, emit arraySlice(). PG slice + * is inclusive both ends, CH arraySlice takes (arr, offset, length). + * Only 1D arrays are supported (enforced elsewhere). + */ + Expr* lower = (Expr*)linitial(node->reflowerindexpr); + Expr* upper = (Expr*)linitial(node->refupperindexpr); + + appendStringInfoString(buf, "arraySlice("); + deparseExpr(node->refexpr, context); + appendStringInfoString(buf, ", "); + + if (lower) { + deparseExpr(lower, context); + } else { + appendStringInfoChar(buf, '1'); + } + + if (upper) { + appendStringInfoString(buf, ", ("); + deparseExpr(upper, context); + appendStringInfoString(buf, ") - ("); + if (lower) { + deparseExpr(lower, context); + } else { + appendStringInfoChar(buf, '1'); + } + appendStringInfoString(buf, ") + 1"); + } + + appendStringInfoChar(buf, ')'); + } else { + /* Single element: emit arr[idx] */ + appendStringInfoChar(buf, '('); + if (IsA(node->refexpr, Var)) { + deparseExpr(node->refexpr, context); + } else { + appendStringInfoChar(buf, '('); + deparseExpr(node->refexpr, context); + appendStringInfoChar(buf, ')'); + } + appendStringInfoChar(buf, '['); + deparseExpr((Expr*)linitial(node->refupperindexpr), context); + appendStringInfoString(buf, "])"); + } } /* @@ -2530,37 +2631,35 @@ deparseSubscriptingRef(SubscriptingRef * node, deparse_expr_cxt * context) * is a text[] constant holding the path keys. */ static void -deparseJsonbExtractPath(FuncExpr * node, deparse_expr_cxt * context, - bool wrap_json) -{ - StringInfo buf = context->buf; - Const *arr = (Const *) lsecond(node->args); - ArrayType *array; - Datum *elems; - bool *nulls; - int nelems; - - if (wrap_json) - appendStringInfoString(buf, "toJSONString("); - - /* First arg is the JSONB column expression. */ - deparseExpr((Expr *) linitial(node->args), context); - - /* Second arg is text[] with path keys → dot notation. */ - array = DatumGetArrayTypeP(arr->constvalue); - deconstruct_array(array, TEXTOID, -1, false, TYPALIGN_INT, - &elems, &nulls, &nelems); - - for (int i = 0; i < nelems; i++) - { - char *key = TextDatumGetCString(elems[i]); - - appendStringInfoChar(buf, '.'); - appendStringInfoString(buf, quote_identifier(key)); - } - - if (wrap_json) - appendStringInfoChar(buf, ')'); +deparseJsonbExtractPath(FuncExpr* node, deparse_expr_cxt* context, bool wrap_json) { + StringInfo buf = context->buf; + Const* arr = (Const*)lsecond(node->args); + ArrayType* array; + Datum* elems; + bool* nulls; + int nelems; + + if (wrap_json) { + appendStringInfoString(buf, "toJSONString("); + } + + /* First arg is the JSONB column expression. */ + deparseExpr((Expr*)linitial(node->args), context); + + /* Second arg is text[] with path keys → dot notation. */ + array = DatumGetArrayTypeP(arr->constvalue); + deconstruct_array(array, TEXTOID, -1, false, TYPALIGN_INT, &elems, &nulls, &nelems); + + for (int i = 0; i < nelems; i++) { + char* key = TextDatumGetCString(elems[i]); + + appendStringInfoChar(buf, '.'); + appendStringInfoString(buf, quote_identifier(key)); + } + + if (wrap_json) { + appendStringInfoChar(buf, ')'); + } } /* @@ -2580,69 +2679,70 @@ deparseJsonbExtractPath(FuncExpr * node, deparse_expr_cxt * context, * */ static void -appendRegexFlags(Const * arg, deparse_expr_cxt * context) -{ - char *str = TextDatumGetCString(arg->constvalue); - enum - { - flag_0 = 0, - flag_i = 1, - flag_m = 2, - flag_p = 4, - flag_s = 8, - flag_w = 16, - }; - uint16 flags = flag_0; - - /* Iterate over the flags. */ - while (*str) - { - switch (*str) - { - case 'i': - flags |= flag_i; - break; - case 'm': - case 'n': /* Postgres new name for m. */ - flags |= flag_m; - flags &= ~flag_s; /* Cancels out s. */ - flags &= ~flag_p; /* Cancels out p. */ - flags &= ~flag_w; /* Cancels out w. */ - break; - case 'p': - flags |= flag_p; - flags &= ~flag_m; /* Cancels out m. */ - flags &= ~flag_s; /* Cancels out s. */ - flags &= ~flag_w; /* Cancels out w. */ - break; - case 's': - flags |= flag_s; - flags &= ~flag_m; /* Cancels out m. */ - flags &= ~flag_p; /* Cancels out p. */ - flags &= ~flag_w; /* Cancels out w. */ - break; - case 't': /* Always tight syntax, skip. */ - break; - case 'w': - flags |= flag_w; - flags &= ~flag_m; /* Cancels out m. */ - flags &= ~flag_p; /* Cancels out p. */ - flags &= ~flag_s; /* Cancels out s. */ - break; - } - str++; - } - - if (flags & flag_i) - appendStringInfoChar(context->buf, 'i'); - if (flags & flag_m) - appendStringInfoString(context->buf, "m-s"); - if (flags & flag_p) - appendStringInfoString(context->buf, "-s"); - if (flags & flag_s) - appendStringInfoChar(context->buf, 's'); - if (flags & flag_w) - appendStringInfoChar(context->buf, 'm'); +appendRegexFlags(Const* arg, deparse_expr_cxt* context) { + char* str = TextDatumGetCString(arg->constvalue); + enum { + flag_0 = 0, + flag_i = 1, + flag_m = 2, + flag_p = 4, + flag_s = 8, + flag_w = 16, + }; + uint16 flags = flag_0; + + /* Iterate over the flags. */ + while (*str) { + switch (*str) { + case 'i': + flags |= flag_i; + break; + case 'm': + case 'n': /* Postgres new name for m. */ + flags |= flag_m; + flags &= ~flag_s; /* Cancels out s. */ + flags &= ~flag_p; /* Cancels out p. */ + flags &= ~flag_w; /* Cancels out w. */ + break; + case 'p': + flags |= flag_p; + flags &= ~flag_m; /* Cancels out m. */ + flags &= ~flag_s; /* Cancels out s. */ + flags &= ~flag_w; /* Cancels out w. */ + break; + case 's': + flags |= flag_s; + flags &= ~flag_m; /* Cancels out m. */ + flags &= ~flag_p; /* Cancels out p. */ + flags &= ~flag_w; /* Cancels out w. */ + break; + case 't': /* Always tight syntax, skip. */ + break; + case 'w': + flags |= flag_w; + flags &= ~flag_m; /* Cancels out m. */ + flags &= ~flag_p; /* Cancels out p. */ + flags &= ~flag_s; /* Cancels out s. */ + break; + } + str++; + } + + if (flags & flag_i) { + appendStringInfoChar(context->buf, 'i'); + } + if (flags & flag_m) { + appendStringInfoString(context->buf, "m-s"); + } + if (flags & flag_p) { + appendStringInfoString(context->buf, "-s"); + } + if (flags & flag_s) { + appendStringInfoChar(context->buf, 's'); + } + if (flags & flag_w) { + appendStringInfoChar(context->buf, 'm'); + } } /* @@ -2652,24 +2752,22 @@ appendRegexFlags(Const * arg, deparse_expr_cxt * context) * are no flags it simply appends the regexp expression. If there are flags, * it emits the regular expression as `concat('(?$flags), $regexp)`, * delegating the actually flag output to `appendRegexFlags()`. -*/ + */ static void -appendRegex(List * args, deparse_expr_cxt * context) -{ - - if (list_length(args) <= 2) - { - /* No flags argument, just append the regexp expression. */ - deparseExpr((Expr *) list_nth(args, 1), context); - return; - } - - /* Concatenate the flags with the regexp expression. */ - appendStringInfoString(context->buf, "concat('(?"); - appendRegexFlags((Const *) list_nth(args, 2), context); - appendStringInfoString(context->buf, ")', "); - deparseExpr((Expr *) list_nth(args, 1), context); - appendStringInfoChar(context->buf, ')'); +appendRegex(List* args, deparse_expr_cxt* context) { + + if (list_length(args) <= 2) { + /* No flags argument, just append the regexp expression. */ + deparseExpr((Expr*)list_nth(args, 1), context); + return; + } + + /* Concatenate the flags with the regexp expression. */ + appendStringInfoString(context->buf, "concat('(?"); + appendRegexFlags((Const*)list_nth(args, 2), context); + appendStringInfoString(context->buf, ")', "); + deparseExpr((Expr*)list_nth(args, 1), context); + appendStringInfoChar(context->buf, ')'); } /* @@ -2685,548 +2783,578 @@ appendRegex(List * args, deparse_expr_cxt * context) * See PG `DCH_keywords` in `src/backend/utils/adt/formatting.c` for the * full PG token grammar. */ -typedef struct -{ - const char *pg; - const char *ch; /* NULL → recognized keyword we refuse */ -} ToCharTok; +typedef struct { + const char* pg; + const char* ch; /* NULL → recognized keyword we refuse */ +} ToCharTok; static const ToCharTok to_char_toks[] = { - {"Y,YYY", NULL}, {"y,yyy", NULL}, - {"SSSSS", NULL}, {"sssss", NULL}, - {"MONTH", NULL}, {"Month", NULL}, {"month", NULL}, - {"YYYY", "%Y"}, {"yyyy", "%Y"}, - {"HH24", "%H"}, {"hh24", "%H"}, - {"HH12", "%I"}, {"hh12", "%I"}, - {"IYYY", NULL}, {"iyyy", NULL}, - {"IDDD", NULL}, {"iddd", NULL}, - {"SSSS", NULL}, {"ssss", NULL}, - {"A.D.", NULL}, {"a.d.", NULL}, - {"B.C.", NULL}, {"b.c.", NULL}, - {"P.M.", NULL}, {"p.m.", NULL}, - {"A.M.", NULL}, {"a.m.", NULL}, - {"YYY", NULL}, {"yyy", NULL}, - {"DDD", "%j"}, {"ddd", "%j"}, - {"DAY", NULL}, {"day", NULL}, {"Day", NULL}, - {"Mon", "%b"}, - {"MON", NULL}, {"mon", NULL}, - {"IYY", NULL}, {"iyy", NULL}, - {"FF1", NULL}, {"FF2", NULL}, {"FF3", NULL}, - {"FF4", NULL}, {"FF5", NULL}, {"FF6", NULL}, - {"ff1", NULL}, {"ff2", NULL}, {"ff3", NULL}, - {"ff4", NULL}, {"ff5", NULL}, {"ff6", NULL}, - {"TZH", NULL}, {"tzh", NULL}, - {"TZM", NULL}, {"tzm", NULL}, - {"AD", NULL}, {"ad", NULL}, - {"AM", "%p"}, {"PM", "%p"}, - {"am", NULL}, {"pm", NULL}, - {"BC", NULL}, {"bc", NULL}, - {"CC", NULL}, {"cc", NULL}, - {"DD", "%d"}, {"dd", "%d"}, - {"Dy", "%a"}, - {"DY", NULL}, {"dy", NULL}, - {"FM", NULL}, {"fm", NULL}, - {"FX", NULL}, {"fx", NULL}, - {"HH", "%I"}, {"hh", "%I"}, - {"ID", NULL}, {"id", NULL}, - {"IW", NULL}, {"iw", NULL}, - {"IY", NULL}, {"iy", NULL}, - {"MI", "%i"}, {"mi", "%i"}, - {"MM", "%m"}, {"mm", "%m"}, - {"MS", NULL}, {"ms", NULL}, - {"OF", NULL}, {"of", NULL}, - {"RM", NULL}, {"rm", NULL}, - {"SS", "%S"}, {"ss", "%S"}, - {"TM", NULL}, {"tm", NULL}, - {"TZ", NULL}, {"tz", NULL}, - {"US", NULL}, {"us", NULL}, - {"WW", NULL}, {"ww", NULL}, - {"YY", "%y"}, {"yy", "%y"}, - {"D", NULL}, {"d", NULL}, - {"I", NULL}, {"i", NULL}, - {"J", NULL}, {"j", NULL}, - {"Q", "%Q"}, {"q", "%Q"}, - {"W", NULL}, {"w", NULL}, - {"Y", NULL}, {"y", NULL}, - {NULL, NULL} + { "Y,YYY", NULL }, + { "y,yyy", NULL }, + { "SSSSS", NULL }, + { "sssss", NULL }, + { "MONTH", NULL }, + { "Month", NULL }, + { "month", NULL }, + { "YYYY", "%Y" }, + { "yyyy", "%Y" }, + { "HH24", "%H" }, + { "hh24", "%H" }, + { "HH12", "%I" }, + { "hh12", "%I" }, + { "IYYY", NULL }, + { "iyyy", NULL }, + { "IDDD", NULL }, + { "iddd", NULL }, + { "SSSS", NULL }, + { "ssss", NULL }, + { "A.D.", NULL }, + { "a.d.", NULL }, + { "B.C.", NULL }, + { "b.c.", NULL }, + { "P.M.", NULL }, + { "p.m.", NULL }, + { "A.M.", NULL }, + { "a.m.", NULL }, + { "YYY", NULL }, + { "yyy", NULL }, + { "DDD", "%j" }, + { "ddd", "%j" }, + { "DAY", NULL }, + { "day", NULL }, + { "Day", NULL }, + { "Mon", "%b" }, + { "MON", NULL }, + { "mon", NULL }, + { "IYY", NULL }, + { "iyy", NULL }, + { "FF1", NULL }, + { "FF2", NULL }, + { "FF3", NULL }, + { "FF4", NULL }, + { "FF5", NULL }, + { "FF6", NULL }, + { "ff1", NULL }, + { "ff2", NULL }, + { "ff3", NULL }, + { "ff4", NULL }, + { "ff5", NULL }, + { "ff6", NULL }, + { "TZH", NULL }, + { "tzh", NULL }, + { "TZM", NULL }, + { "tzm", NULL }, + { "AD", NULL }, + { "ad", NULL }, + { "AM", "%p" }, + { "PM", "%p" }, + { "am", NULL }, + { "pm", NULL }, + { "BC", NULL }, + { "bc", NULL }, + { "CC", NULL }, + { "cc", NULL }, + { "DD", "%d" }, + { "dd", "%d" }, + { "Dy", "%a" }, + { "DY", NULL }, + { "dy", NULL }, + { "FM", NULL }, + { "fm", NULL }, + { "FX", NULL }, + { "fx", NULL }, + { "HH", "%I" }, + { "hh", "%I" }, + { "ID", NULL }, + { "id", NULL }, + { "IW", NULL }, + { "iw", NULL }, + { "IY", NULL }, + { "iy", NULL }, + { "MI", "%i" }, + { "mi", "%i" }, + { "MM", "%m" }, + { "mm", "%m" }, + { "MS", NULL }, + { "ms", NULL }, + { "OF", NULL }, + { "of", NULL }, + { "RM", NULL }, + { "rm", NULL }, + { "SS", "%S" }, + { "ss", "%S" }, + { "TM", NULL }, + { "tm", NULL }, + { "TZ", NULL }, + { "tz", NULL }, + { "US", NULL }, + { "us", NULL }, + { "WW", NULL }, + { "ww", NULL }, + { "YY", "%y" }, + { "yy", "%y" }, + { "D", NULL }, + { "d", NULL }, + { "I", NULL }, + { "i", NULL }, + { "J", NULL }, + { "j", NULL }, + { "Q", "%Q" }, + { "q", "%Q" }, + { "W", NULL }, + { "w", NULL }, + { "Y", NULL }, + { "y", NULL }, + { NULL, NULL } }; bool -chfdw_translate_to_char_format(const char *pgfmt, StringInfo out) -{ - const char *p = pgfmt; - bool just_keyword = false; - - while (*p) - { - const ToCharTok *t; - bool matched = false; - - /* - * Ordinal postfix (TH/th) attaches to a preceding numeric keyword; CH - * has no ordinal output, so refuse rather than diverge. - */ - if (just_keyword - && (p[0] == 'T' || p[0] == 't') - && (p[1] == 'H' || p[1] == 'h')) - return false; - just_keyword = false; - - /* Quoted literal: pass inner chars through, doubling % for CH. */ - if (*p == '"') - { - p++; - while (*p) - { - if (*p == '\\' && p[1]) - p++; - else if (*p == '"') - { - p++; - break; - } - if (out) - { - if (*p == '%') - appendStringInfoString(out, "%%"); - else - appendStringInfoChar(out, *p); - } - p++; - } - continue; - } - - /* Outside quotes, backslash escapes only a literal " */ - if (p[0] == '\\' && p[1] == '"') - { - if (out) - appendStringInfoChar(out, '"'); - p += 2; - continue; - } - - for (t = to_char_toks; t->pg; t++) - { - size_t l = strlen(t->pg); - - if (strncmp(p, t->pg, l) == 0) - { - if (!t->ch) - return false; - if (out) - appendStringInfoString(out, t->ch); - p += l; - just_keyword = true; - matched = true; - break; - } - } - if (matched) - continue; - - if (out) - { - if (*p == '%') - appendStringInfoString(out, "%%"); - else - appendStringInfoChar(out, *p); - } - p++; - } - - return true; +chfdw_translate_to_char_format(const char* pgfmt, StringInfo out) { + const char* p = pgfmt; + bool just_keyword = false; + + while (*p) { + const ToCharTok* t; + bool matched = false; + + /* + * Ordinal postfix (TH/th) attaches to a preceding numeric keyword; CH + * has no ordinal output, so refuse rather than diverge. + */ + if (just_keyword && (p[0] == 'T' || p[0] == 't') && + (p[1] == 'H' || p[1] == 'h')) { + return false; + } + just_keyword = false; + + /* Quoted literal: pass inner chars through, doubling % for CH. */ + if (*p == '"') { + p++; + while (*p) { + if (*p == '\\' && p[1]) { + p++; + } else if (*p == '"') { + p++; + break; + } + if (out) { + if (*p == '%') { + appendStringInfoString(out, "%%"); + } else { + appendStringInfoChar(out, *p); + } + } + p++; + } + continue; + } + + /* Outside quotes, backslash escapes only a literal " */ + if (p[0] == '\\' && p[1] == '"') { + if (out) { + appendStringInfoChar(out, '"'); + } + p += 2; + continue; + } + + for (t = to_char_toks; t->pg; t++) { + size_t l = strlen(t->pg); + + if (strncmp(p, t->pg, l) == 0) { + if (!t->ch) { + return false; + } + if (out) { + appendStringInfoString(out, t->ch); + } + p += l; + just_keyword = true; + matched = true; + break; + } + } + if (matched) { + continue; + } + + if (out) { + if (*p == '%') { + appendStringInfoString(out, "%%"); + } else { + appendStringInfoChar(out, *p); + } + } + p++; + } + + return true; } /* * Deparse a function call. */ static void -deparseFuncExpr(FuncExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - bool first; - ListCell *arg; - CustomObjectDef *cdef, - *old_cdef; - - /* - * If the function call came from an implicit coercion, then just show the - * first argument. - */ - if (node->funcformat == COERCE_IMPLICIT_CAST) - { - deparseExpr((Expr *) linitial(node->args), context); - return; - } - - /* - * If the function call came from a cast, then show the first argument - * plus an explicit cast operation. - */ - if (node->funcformat == COERCE_EXPLICIT_CAST) - { - Oid rettype = node->funcresulttype; - int32 coercedTypmod; - - /* Get the typmod if this is a length-coercion function */ - (void) exprIsLengthCoercion((Node *) node, &coercedTypmod); - - appendStringInfoString(buf, "cast("); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfo(buf, ", 'Nullable(%s)')", - deparse_type_name(rettype, coercedTypmod)); - return; - } - - /* - * Normal function: display as proname(args). CF_ARRAY_SORT_DESC name - * depends on boolean arg, resolve before printing. - */ - cdef = appendFunctionName(node->funcid, context); - - if (cdef) - { - /* Special casses. */ - switch (cdef->cf_type) - { - case CF_JSON_EXTRACT_PATH_TEXT: - case CF_JSON_EXTRACT_PATH: - { - deparseJsonbExtractPath(node, context, - cdef->cf_type == CF_JSON_EXTRACT_PATH); - return; - } - case CF_CURRENT_DATABASE: - { - SQLValueFunction svf; - - svf.op = SVFOP_CURRENT_CATALOG; - deparseSQLValueFunction(&svf, context); - return; - } - case CF_CURRENT_SCHEMA: - { - SQLValueFunction svf; - - svf.op = SVFOP_CURRENT_SCHEMA; - deparseSQLValueFunction(&svf, context); - return; - } - case CF_CLOCK_TIMESTAMP: - { - appendStringInfo(buf, "nowInBlock64(6, %s)", QUOTED_TZ); - return; - } - case CF_DATE_TRUNC: - { - Const *arg = (Const *) linitial(node->args); - char *trunctype = TextDatumGetCString(arg->constvalue); - - CSTRING_TOLOWER(trunctype); - int cast_to_datetime64 = 0; - - if (strcmp(trunctype, "week") == 0) - { - appendStringInfoString(buf, "toMonday"); - } - else if (strcmp(trunctype, "second") == 0) - { - cast_to_datetime64 = 1; - appendStringInfoString(buf, "toStartOfSecond"); - } - else if (strcmp(trunctype, "minute") == 0) - { - appendStringInfoString(buf, "toStartOfMinute"); - } - else if (strcmp(trunctype, "hour") == 0) - { - appendStringInfoString(buf, "toStartOfHour"); - } - else if (strcmp(trunctype, "day") == 0) - { - appendStringInfoString(buf, "toStartOfDay"); - } - else if (strcmp(trunctype, "month") == 0) - { - appendStringInfoString(buf, "toStartOfMonth"); - } - else if (strcmp(trunctype, "quarter") == 0) - { - appendStringInfoString(buf, "toStartOfQuarter"); - } - else if (strcmp(trunctype, "year") == 0) - { - appendStringInfoString(buf, "toStartOfYear"); - } - else - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("date_trunc cannot be exported for: %s", trunctype))); - } - - pfree(trunctype); - if (cast_to_datetime64) - { - appendStringInfoString(buf, "(toDateTime64("); - deparseExpr(list_nth(node->args, 1), context); - appendStringInfoString(buf, ", 1))"); - } - else - { - appendStringInfoChar(buf, '('); - deparseExpr(list_nth(node->args, 1), context); - appendStringInfoChar(buf, ')'); - } - return; - } - case CF_DATE_PART: - { - Const *arg = (Const *) linitial(node->args); - char *parttype = TextDatumGetCString(arg->constvalue); - - CSTRING_TOLOWER(parttype); - - if (strcmp(parttype, "day") == 0) - appendStringInfoString(buf, "toDayOfMonth"); - else if (strcmp(parttype, "doy") == 0) - appendStringInfoString(buf, "toDayOfYear"); - else if (strcmp(parttype, "dow") == 0) - appendStringInfoString(buf, "toDayOfWeek"); - else if (strcmp(parttype, "year") == 0) - appendStringInfoString(buf, "toYear"); - else if (strcmp(parttype, "month") == 0) - appendStringInfoString(buf, "toMonth"); - else if (strcmp(parttype, "hour") == 0) - appendStringInfoString(buf, "toHour"); - else if (strcmp(parttype, "minute") == 0) - appendStringInfoString(buf, "toMinute"); - else if (strcmp(parttype, "second") == 0) - appendStringInfoString(buf, "toSecond"); - else if (strcmp(parttype, "quarter") == 0) - appendStringInfoString(buf, "toQuarter"); - else if (strcmp(parttype, "isoyear") == 0) - appendStringInfoString(buf, "toISOYear"); - else if (strcmp(parttype, "week") == 0) - appendStringInfoString(buf, "toISOWeek"); - else if (strcmp(parttype, "epoch") == 0) - appendStringInfoString(buf, "toUnixTimestamp"); - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("date_part cannot be exported for: %s", parttype))); - - pfree(parttype); - appendStringInfoChar(buf, '('); - deparseExpr(list_nth(node->args, 1), context); - appendStringInfoChar(buf, ')'); - return; - } - case CF_TIMEZONE: - case CF_ARRAY_PREPEND: - case CF_STRING_TO_ARRAY: - case CF_STRING_TO_ARRAY_PART: - { - /* Arguments are reversed. */ - appendStringInfoChar(buf, '('); - deparseExpr((Expr *) list_nth(node->args, 1), context); - appendStringInfoString(buf, ", "); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoChar(buf, ')'); - if (cdef->cf_type == CF_STRING_TO_ARRAY_PART) - { - /* Use array subscript to extract item. */ - appendStringInfoChar(buf, '['); - deparseExpr((Expr *) list_nth(node->args, 2), context); - appendStringInfoChar(buf, ']'); - } - return; - } - case CF_MATCH: - { - /* match(haystack, pattern) */ - appendStringInfoChar(buf, '('); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoString(buf, ", "); - appendRegex(node->args, context); - appendStringInfoChar(buf, ')'); - return; - } - case CF_SPLIT_BY_REGEX: - { - /* splitByRegexp(regexp, s) */ - appendStringInfoChar(buf, '('); - appendRegex(node->args, context); - appendStringInfoString(buf, ", "); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoChar(buf, ')'); - return; - } - case CF_REPLACE_REGEX: - { - /* replaceRegexpOne() or replaceRegexpAll() */ - char *flags = NULL; - - if (list_length(node->args) >= 4) - { - /* Determine function name from "g" in flags. */ - /* XXX May not be a constant. Enforce elsewhere. */ - Const *arg = (Const *) list_nth(node->args, 3); - - flags = TextDatumGetCString(arg->constvalue); - char *c = strchr(flags, 'g'); - - if (c) - { - appendStringInfoString(buf, "replaceRegexpAll"); - /* Remove any and all g flags. */ - while (c[0]) - { - while (c[1] == 'g') - ++c; - c[0] = c[1]; - ++c; - } - } - else - appendStringInfoString(buf, "replaceRegexpOne"); - } - else - appendStringInfoString(buf, "replaceRegexpOne"); - - appendStringInfoChar(buf, '('); - - /* Emit the first string to search ("haystack"). */ - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoString(buf, ", "); - - /* Emit the regular expression. */ - if (flags && strlen(flags)) - { - /* Concatenate flags. */ - appendStringInfo(context->buf, "concat('(?%s)', ", flags); - deparseExpr((Expr *) list_nth(node->args, 1), context); - appendStringInfoChar(buf, ')'); - } - else - deparseExpr((Expr *) list_nth(node->args, 1), context); - - /* Emit the replacement string and finish. */ - appendStringInfoString(buf, ", "); - deparseExpr((Expr *) list_nth(node->args, 2), context); - appendStringInfoChar(buf, ')'); - return; - } - case CF_REGEX_PG_MATCH: - { - /* Parse the regex so we can determine if it has captures. */ - Const *arg = (Const *) list_nth(((FuncExpr *) node)->args, 1); - pg_regex_t *regex = RE_compile_and_cache(DatumGetTextP(arg->constvalue), - REG_ADVANCED, node->inputcollid); - - /* - * If no captures, use arraySlice(extractAll(), otherwise - * use extractGroups(). - */ - appendStringInfoString(buf, regex->re_nsub ? "extractGroups(" : "arraySlice(extractAll("); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoString(buf, ", "); - appendRegex(node->args, context); - appendStringInfoString(buf, regex->re_nsub ? ")" : "), 1, 1)"); - return; - } - case CF_ARRAY_LENGTH: - appendStringInfoChar(buf, '('); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoChar(buf, ')'); - return; - case CF_TRIM_ARRAY: - /* arrayResize(arr, length(arr) - n) */ - appendStringInfoChar(buf, '('); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoString(buf, ", length("); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoString(buf, ") - "); - deparseExpr((Expr *) list_nth(node->args, 1), context); - appendStringInfoChar(buf, ')'); - return; - case CF_ARRAY_SORT_DESC: - { - /* Determine function name from reverse boolean arg. */ - Const *desc_arg = (Const *) list_nth(node->args, 1); - - appendStringInfoString(buf, !desc_arg->constisnull && DatumGetBool(desc_arg->constvalue) ? "arrayReverseSort" : "arraySort"); - appendStringInfoChar(buf, '('); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoChar(buf, ')'); - return; - } - - case CF_ARRAY_FILL: - /* arrayWithConstant(n, val): swap args */ - appendStringInfoChar(buf, '('); - deparseExpr((Expr *) list_nth(node->args, 1), context); - appendStringInfoString(buf, ", "); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoChar(buf, ')'); - return; - case CF_TO_CHAR: - { - /* - * formatDateTime(ts, ''). Shippability - * already validated the format string is a non-NULL Const - * whose tokens all translate; redoing the translation - * here is just to materialize the result for SQL-literal - * quoting. - */ - Const *fmt_const = (Const *) list_nth(node->args, 1); - char *pgfmt = TextDatumGetCString(fmt_const->constvalue); - StringInfoData chfmt; - - initStringInfo(&chfmt); - if (!chfdw_translate_to_char_format(pgfmt, &chfmt)) - elog(ERROR, "to_char format unexpectedly rejected during deparse: %s", pgfmt); - - appendStringInfoChar(buf, '('); - deparseExpr((Expr *) linitial(node->args), context); - appendStringInfoString(buf, ", "); - deparseStringLiteral(buf, chfmt.data); - appendStringInfoChar(buf, ')'); - - pfree(chfmt.data); - pfree(pgfmt); - return; - } - default: - break; - } - } - - appendStringInfoChar(buf, '('); - old_cdef = context->func; - - /* ... and all the arguments */ - first = true; - foreach(arg, node->args) - { - if (!first) - appendStringInfoString(buf, ", "); - - deparseExpr((Expr *) lfirst(arg), context); - first = false; - } - - context->func = old_cdef; - if (cdef) - { - for (int i = 0; i < cdef->paren_count; i++) - { - appendStringInfoChar(buf, ')'); - } - } - else - appendStringInfoChar(buf, ')'); +deparseFuncExpr(FuncExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + bool first; + ListCell* arg; + CustomObjectDef *cdef, *old_cdef; + + /* + * If the function call came from an implicit coercion, then just show the + * first argument. + */ + if (node->funcformat == COERCE_IMPLICIT_CAST) { + deparseExpr((Expr*)linitial(node->args), context); + return; + } + + /* + * If the function call came from a cast, then show the first argument + * plus an explicit cast operation. + */ + if (node->funcformat == COERCE_EXPLICIT_CAST) { + Oid rettype = node->funcresulttype; + int32 coercedTypmod; + + /* Get the typmod if this is a length-coercion function */ + (void)exprIsLengthCoercion((Node*)node, &coercedTypmod); + + appendStringInfoString(buf, "cast("); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfo( + buf, ", 'Nullable(%s)')", deparse_type_name(rettype, coercedTypmod) + ); + return; + } + + /* + * Normal function: display as proname(args). CF_ARRAY_SORT_DESC name + * depends on boolean arg, resolve before printing. + */ + cdef = appendFunctionName(node->funcid, context); + + if (cdef) { + /* Special casses. */ + switch (cdef->cf_type) { + case CF_JSON_EXTRACT_PATH_TEXT: + case CF_JSON_EXTRACT_PATH: { + deparseJsonbExtractPath( + node, context, cdef->cf_type == CF_JSON_EXTRACT_PATH + ); + return; + } + case CF_CURRENT_DATABASE: { + SQLValueFunction svf; + + svf.op = SVFOP_CURRENT_CATALOG; + deparseSQLValueFunction(&svf, context); + return; + } + case CF_CURRENT_SCHEMA: { + SQLValueFunction svf; + + svf.op = SVFOP_CURRENT_SCHEMA; + deparseSQLValueFunction(&svf, context); + return; + } + case CF_CLOCK_TIMESTAMP: { + appendStringInfo(buf, "nowInBlock64(6, %s)", QUOTED_TZ); + return; + } + case CF_DATE_TRUNC: { + Const* arg = (Const*)linitial(node->args); + char* trunctype = TextDatumGetCString(arg->constvalue); + + CSTRING_TOLOWER(trunctype); + int cast_to_datetime64 = 0; + + if (strcmp(trunctype, "week") == 0) { + appendStringInfoString(buf, "toMonday"); + } else if (strcmp(trunctype, "second") == 0) { + cast_to_datetime64 = 1; + appendStringInfoString(buf, "toStartOfSecond"); + } else if (strcmp(trunctype, "minute") == 0) { + appendStringInfoString(buf, "toStartOfMinute"); + } else if (strcmp(trunctype, "hour") == 0) { + appendStringInfoString(buf, "toStartOfHour"); + } else if (strcmp(trunctype, "day") == 0) { + appendStringInfoString(buf, "toStartOfDay"); + } else if (strcmp(trunctype, "month") == 0) { + appendStringInfoString(buf, "toStartOfMonth"); + } else if (strcmp(trunctype, "quarter") == 0) { + appendStringInfoString(buf, "toStartOfQuarter"); + } else if (strcmp(trunctype, "year") == 0) { + appendStringInfoString(buf, "toStartOfYear"); + } else { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("date_trunc cannot be exported for: %s", trunctype)) + ); + } + + pfree(trunctype); + if (cast_to_datetime64) { + appendStringInfoString(buf, "(toDateTime64("); + deparseExpr(list_nth(node->args, 1), context); + appendStringInfoString(buf, ", 1))"); + } else { + appendStringInfoChar(buf, '('); + deparseExpr(list_nth(node->args, 1), context); + appendStringInfoChar(buf, ')'); + } + return; + } + case CF_DATE_PART: { + Const* arg = (Const*)linitial(node->args); + char* parttype = TextDatumGetCString(arg->constvalue); + + CSTRING_TOLOWER(parttype); + + if (strcmp(parttype, "day") == 0) { + appendStringInfoString(buf, "toDayOfMonth"); + } else if (strcmp(parttype, "doy") == 0) { + appendStringInfoString(buf, "toDayOfYear"); + } else if (strcmp(parttype, "dow") == 0) { + appendStringInfoString(buf, "toDayOfWeek"); + } else if (strcmp(parttype, "year") == 0) { + appendStringInfoString(buf, "toYear"); + } else if (strcmp(parttype, "month") == 0) { + appendStringInfoString(buf, "toMonth"); + } else if (strcmp(parttype, "hour") == 0) { + appendStringInfoString(buf, "toHour"); + } else if (strcmp(parttype, "minute") == 0) { + appendStringInfoString(buf, "toMinute"); + } else if (strcmp(parttype, "second") == 0) { + appendStringInfoString(buf, "toSecond"); + } else if (strcmp(parttype, "quarter") == 0) { + appendStringInfoString(buf, "toQuarter"); + } else if (strcmp(parttype, "isoyear") == 0) { + appendStringInfoString(buf, "toISOYear"); + } else if (strcmp(parttype, "week") == 0) { + appendStringInfoString(buf, "toISOWeek"); + } else if (strcmp(parttype, "epoch") == 0) { + appendStringInfoString(buf, "toUnixTimestamp"); + } else { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("date_part cannot be exported for: %s", parttype)) + ); + } + + pfree(parttype); + appendStringInfoChar(buf, '('); + deparseExpr(list_nth(node->args, 1), context); + appendStringInfoChar(buf, ')'); + return; + } + case CF_TIMEZONE: + case CF_ARRAY_PREPEND: + case CF_STRING_TO_ARRAY: + case CF_STRING_TO_ARRAY_PART: { + /* Arguments are reversed. */ + appendStringInfoChar(buf, '('); + deparseExpr((Expr*)list_nth(node->args, 1), context); + appendStringInfoString(buf, ", "); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoChar(buf, ')'); + if (cdef->cf_type == CF_STRING_TO_ARRAY_PART) { + /* Use array subscript to extract item. */ + appendStringInfoChar(buf, '['); + deparseExpr((Expr*)list_nth(node->args, 2), context); + appendStringInfoChar(buf, ']'); + } + return; + } + case CF_MATCH: { + /* match(haystack, pattern) */ + appendStringInfoChar(buf, '('); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoString(buf, ", "); + appendRegex(node->args, context); + appendStringInfoChar(buf, ')'); + return; + } + case CF_SPLIT_BY_REGEX: { + /* splitByRegexp(regexp, s) */ + appendStringInfoChar(buf, '('); + appendRegex(node->args, context); + appendStringInfoString(buf, ", "); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoChar(buf, ')'); + return; + } + case CF_REPLACE_REGEX: { + /* replaceRegexpOne() or replaceRegexpAll() */ + char* flags = NULL; + + if (list_length(node->args) >= 4) { + /* Determine function name from "g" in flags. */ + /* XXX May not be a constant. Enforce elsewhere. */ + Const* arg = (Const*)list_nth(node->args, 3); + + flags = TextDatumGetCString(arg->constvalue); + char* c = strchr(flags, 'g'); + + if (c) { + appendStringInfoString(buf, "replaceRegexpAll"); + /* Remove any and all g flags. */ + while (c[0]) { + while (c[1] == 'g') { + ++c; + } + c[0] = c[1]; + ++c; + } + } else { + appendStringInfoString(buf, "replaceRegexpOne"); + } + } else { + appendStringInfoString(buf, "replaceRegexpOne"); + } + + appendStringInfoChar(buf, '('); + + /* Emit the first string to search ("haystack"). */ + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoString(buf, ", "); + + /* Emit the regular expression. */ + if (flags && strlen(flags)) { + /* Concatenate flags. */ + appendStringInfo(context->buf, "concat('(?%s)', ", flags); + deparseExpr((Expr*)list_nth(node->args, 1), context); + appendStringInfoChar(buf, ')'); + } else { + deparseExpr((Expr*)list_nth(node->args, 1), context); + } + + /* Emit the replacement string and finish. */ + appendStringInfoString(buf, ", "); + deparseExpr((Expr*)list_nth(node->args, 2), context); + appendStringInfoChar(buf, ')'); + return; + } + case CF_REGEX_PG_MATCH: { + /* Parse the regex so we can determine if it has captures. */ + Const* arg = (Const*)list_nth(((FuncExpr*)node)->args, 1); + pg_regex_t* regex = RE_compile_and_cache( + DatumGetTextP(arg->constvalue), REG_ADVANCED, node->inputcollid + ); + + /* + * If no captures, use arraySlice(extractAll(), otherwise + * use extractGroups(). + */ + appendStringInfoString( + buf, regex->re_nsub ? "extractGroups(" : "arraySlice(extractAll(" + ); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoString(buf, ", "); + appendRegex(node->args, context); + appendStringInfoString(buf, regex->re_nsub ? ")" : "), 1, 1)"); + return; + } + case CF_ARRAY_LENGTH: + appendStringInfoChar(buf, '('); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoChar(buf, ')'); + return; + case CF_TRIM_ARRAY: + /* arrayResize(arr, length(arr) - n) */ + appendStringInfoChar(buf, '('); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoString(buf, ", length("); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoString(buf, ") - "); + deparseExpr((Expr*)list_nth(node->args, 1), context); + appendStringInfoChar(buf, ')'); + return; + case CF_ARRAY_SORT_DESC: { + /* Determine function name from reverse boolean arg. */ + Const* desc_arg = (Const*)list_nth(node->args, 1); + + appendStringInfoString( + buf, + !desc_arg->constisnull && DatumGetBool(desc_arg->constvalue) + ? "arrayReverseSort" + : "arraySort" + ); + appendStringInfoChar(buf, '('); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoChar(buf, ')'); + return; + } + + case CF_ARRAY_FILL: + /* arrayWithConstant(n, val): swap args */ + appendStringInfoChar(buf, '('); + deparseExpr((Expr*)list_nth(node->args, 1), context); + appendStringInfoString(buf, ", "); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoChar(buf, ')'); + return; + case CF_TO_CHAR: { + /* + * formatDateTime(ts, ''). Shippability + * already validated the format string is a non-NULL Const + * whose tokens all translate; redoing the translation + * here is just to materialize the result for SQL-literal + * quoting. + */ + Const* fmt_const = (Const*)list_nth(node->args, 1); + char* pgfmt = TextDatumGetCString(fmt_const->constvalue); + StringInfoData chfmt; + + initStringInfo(&chfmt); + if (!chfdw_translate_to_char_format(pgfmt, &chfmt)) { + elog( + ERROR, + "to_char format unexpectedly rejected during deparse: %s", + pgfmt + ); + } + + appendStringInfoChar(buf, '('); + deparseExpr((Expr*)linitial(node->args), context); + appendStringInfoString(buf, ", "); + deparseStringLiteral(buf, chfmt.data); + appendStringInfoChar(buf, ')'); + + pfree(chfmt.data); + pfree(pgfmt); + return; + } + default: + break; + } + } + + appendStringInfoChar(buf, '('); + old_cdef = context->func; + + /* ... and all the arguments */ + first = true; + foreach (arg, node->args) { + if (!first) { + appendStringInfoString(buf, ", "); + } + + deparseExpr((Expr*)lfirst(arg), context); + first = false; + } + + context->func = old_cdef; + if (cdef) { + for (int i = 0; i < cdef->paren_count; i++) { + appendStringInfoChar(buf, ')'); + } + } else { + appendStringInfoChar(buf, ')'); + } } /* @@ -3239,191 +3367,187 @@ deparseFuncExpr(FuncExpr * node, deparse_expr_cxt * context) * `src/backend/executor/execExprInterp.c` Postgres source file for reference. */ static void -deparseSQLValueFunction(SQLValueFunction * svf, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - - LOCAL_FCINFO(fcinfo, 0); - Datum datum; - - /* - * ClickHouse does not support TZs as part of DateTimes, so current and - * local variants both render to the session time zone. - */ - switch (svf->op) - { - case SVFOP_CURRENT_DATE: - appendStringInfo(buf, "toDate(now(%s))", QUOTED_TZ); - break; - case SVFOP_CURRENT_TIME: - case SVFOP_LOCALTIME: - appendStringInfo(buf, "toTime64(now64(6, %s), 6)", QUOTED_TZ); - break; - case SVFOP_CURRENT_TIME_N: - case SVFOP_LOCALTIME_N: - appendStringInfo(buf, "toTime64(now64(%1$d, %2$s), %1$d)", svf->typmod, QUOTED_TZ); - break; - case SVFOP_CURRENT_TIMESTAMP: - case SVFOP_LOCALTIMESTAMP: - appendStringInfo(buf, "now64(6, %s)", QUOTED_TZ); - break; - case SVFOP_CURRENT_TIMESTAMP_N: - case SVFOP_LOCALTIMESTAMP_N: - appendStringInfo(buf, "now64(%d, %s)", svf->typmod, QUOTED_TZ); - break; - case SVFOP_CURRENT_ROLE: - case SVFOP_CURRENT_USER: - case SVFOP_USER: - InitFunctionCallInfoData(*fcinfo, NULL, 0, InvalidOid, NULL, NULL); - datum = current_user(fcinfo); - if (fcinfo->isnull) - appendStringInfoString(buf, "NULL"); - else - appendStringInfoString(buf, ch_quote_literal(DatumGetCString(datum))); - break; - case SVFOP_SESSION_USER: - InitFunctionCallInfoData(*fcinfo, NULL, 0, InvalidOid, NULL, NULL); - datum = session_user(fcinfo); - if (fcinfo->isnull) - appendStringInfoString(buf, "NULL"); - else - appendStringInfoString(buf, ch_quote_literal(DatumGetCString(datum))); - break; - case SVFOP_CURRENT_CATALOG: - InitFunctionCallInfoData(*fcinfo, NULL, 0, InvalidOid, NULL, NULL); - datum = current_database(fcinfo); - appendStringInfoString(buf, ch_quote_literal(DatumGetCString(datum))); - break; - case SVFOP_CURRENT_SCHEMA: - InitFunctionCallInfoData(*fcinfo, NULL, 0, InvalidOid, NULL, NULL); - datum = current_schema(fcinfo); - if (fcinfo->isnull) - appendStringInfoString(buf, "NULL"); - else - appendStringInfoString(buf, ch_quote_literal(DatumGetCString(datum))); - break; - default: - elog(ERROR, "unknown SQL Value function: %i", svf->op); - } +deparseSQLValueFunction(SQLValueFunction* svf, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + + LOCAL_FCINFO(fcinfo, 0); + Datum datum; + + /* + * ClickHouse does not support TZs as part of DateTimes, so current and + * local variants both render to the session time zone. + */ + switch (svf->op) { + case SVFOP_CURRENT_DATE: + appendStringInfo(buf, "toDate(now(%s))", QUOTED_TZ); + break; + case SVFOP_CURRENT_TIME: + case SVFOP_LOCALTIME: + appendStringInfo(buf, "toTime64(now64(6, %s), 6)", QUOTED_TZ); + break; + case SVFOP_CURRENT_TIME_N: + case SVFOP_LOCALTIME_N: + appendStringInfo( + buf, "toTime64(now64(%1$d, %2$s), %1$d)", svf->typmod, QUOTED_TZ + ); + break; + case SVFOP_CURRENT_TIMESTAMP: + case SVFOP_LOCALTIMESTAMP: + appendStringInfo(buf, "now64(6, %s)", QUOTED_TZ); + break; + case SVFOP_CURRENT_TIMESTAMP_N: + case SVFOP_LOCALTIMESTAMP_N: + appendStringInfo(buf, "now64(%d, %s)", svf->typmod, QUOTED_TZ); + break; + case SVFOP_CURRENT_ROLE: + case SVFOP_CURRENT_USER: + case SVFOP_USER: + InitFunctionCallInfoData(*fcinfo, NULL, 0, InvalidOid, NULL, NULL); + datum = current_user(fcinfo); + if (fcinfo->isnull) { + appendStringInfoString(buf, "NULL"); + } else { + appendStringInfoString(buf, ch_quote_literal(DatumGetCString(datum))); + } + break; + case SVFOP_SESSION_USER: + InitFunctionCallInfoData(*fcinfo, NULL, 0, InvalidOid, NULL, NULL); + datum = session_user(fcinfo); + if (fcinfo->isnull) { + appendStringInfoString(buf, "NULL"); + } else { + appendStringInfoString(buf, ch_quote_literal(DatumGetCString(datum))); + } + break; + case SVFOP_CURRENT_CATALOG: + InitFunctionCallInfoData(*fcinfo, NULL, 0, InvalidOid, NULL, NULL); + datum = current_database(fcinfo); + appendStringInfoString(buf, ch_quote_literal(DatumGetCString(datum))); + break; + case SVFOP_CURRENT_SCHEMA: + InitFunctionCallInfoData(*fcinfo, NULL, 0, InvalidOid, NULL, NULL); + datum = current_schema(fcinfo); + if (fcinfo->isnull) { + appendStringInfoString(buf, "NULL"); + } else { + appendStringInfoString(buf, ch_quote_literal(DatumGetCString(datum))); + } + break; + default: + elog(ERROR, "unknown SQL Value function: %i", svf->op); + } } static void -deparseIntervalOp(Node * first, Node * second, deparse_expr_cxt * context, bool plus) -{ - StringInfo buf = context->buf; - Const *constval; - Interval *span; - char ibuf[MAXINT8LEN + 1]; - - if (!IsA(second, Const)) - { - bool old_op = context->interval_op; - - /* first argument */ - deparseExpr((Expr *) first, context); - - if (plus) - appendStringInfoString(buf, " + "); - else - appendStringInfoString(buf, " - "); - - appendStringInfoString(buf, "INTERVAL "); - - /* second */ - context->interval_op = true; - deparseExpr((Expr *) second, context); - context->interval_op = old_op; - return; - } - - constval = (Const *) second; - span = DatumGetIntervalP(constval->constvalue); - - /* top function is always addSeconds */ - appendStringInfoString(buf, "addSeconds("); - - if (span->day) - appendStringInfoString(buf, "addDays("); - - if (span->month) - appendStringInfoString(buf, "addMonths("); - - /* first argument here */ - deparseExpr((Expr *) first, context); - - if (span->month) - { - /* addMonths arg */ - appendStringInfoChar(buf, ','); - snprintf(ibuf, sizeof(ibuf), "%d", span->month); - if (!plus) - { - appendStringInfoString(buf, "-("); - appendStringInfoString(buf, ibuf); - appendStringInfoChar(buf, ')'); - } - else - appendStringInfoString(buf, ibuf); - appendStringInfoChar(buf, ')'); - } - - if (span->day) - { - /* addDays arg */ - appendStringInfoChar(buf, ','); - snprintf(ibuf, sizeof(ibuf), "%d", span->day); - if (!plus) - { - appendStringInfoString(buf, "-("); - appendStringInfoString(buf, ibuf); - appendStringInfoChar(buf, ')'); - } - else - appendStringInfoString(buf, ibuf); - appendStringInfoChar(buf, ')'); - } - - /* addSeconds arg */ - appendStringInfoChar(buf, ','); - pg_lltoa((int64) (span->time / 1000000), ibuf); - if (!plus) - { - appendStringInfoString(buf, "-("); - appendStringInfoString(buf, ibuf); - appendStringInfoChar(buf, ')'); - } - else - appendStringInfoString(buf, ibuf); - appendStringInfoChar(buf, ')'); +deparseIntervalOp(Node* first, Node* second, deparse_expr_cxt* context, bool plus) { + StringInfo buf = context->buf; + Const* constval; + Interval* span; + char ibuf[MAXINT8LEN + 1]; + + if (!IsA(second, Const)) { + bool old_op = context->interval_op; + + /* first argument */ + deparseExpr((Expr*)first, context); + + if (plus) { + appendStringInfoString(buf, " + "); + } else { + appendStringInfoString(buf, " - "); + } + + appendStringInfoString(buf, "INTERVAL "); + + /* second */ + context->interval_op = true; + deparseExpr((Expr*)second, context); + context->interval_op = old_op; + return; + } + + constval = (Const*)second; + span = DatumGetIntervalP(constval->constvalue); + + /* top function is always addSeconds */ + appendStringInfoString(buf, "addSeconds("); + + if (span->day) { + appendStringInfoString(buf, "addDays("); + } + + if (span->month) { + appendStringInfoString(buf, "addMonths("); + } + + /* first argument here */ + deparseExpr((Expr*)first, context); + + if (span->month) { + /* addMonths arg */ + appendStringInfoChar(buf, ','); + snprintf(ibuf, sizeof(ibuf), "%d", span->month); + if (!plus) { + appendStringInfoString(buf, "-("); + appendStringInfoString(buf, ibuf); + appendStringInfoChar(buf, ')'); + } else { + appendStringInfoString(buf, ibuf); + } + appendStringInfoChar(buf, ')'); + } + + if (span->day) { + /* addDays arg */ + appendStringInfoChar(buf, ','); + snprintf(ibuf, sizeof(ibuf), "%d", span->day); + if (!plus) { + appendStringInfoString(buf, "-("); + appendStringInfoString(buf, ibuf); + appendStringInfoChar(buf, ')'); + } else { + appendStringInfoString(buf, ibuf); + } + appendStringInfoChar(buf, ')'); + } + + /* addSeconds arg */ + appendStringInfoChar(buf, ','); + pg_lltoa((int64)(span->time / 1000000), ibuf); + if (!plus) { + appendStringInfoString(buf, "-("); + appendStringInfoString(buf, ibuf); + appendStringInfoChar(buf, ')'); + } else { + appendStringInfoString(buf, ibuf); + } + appendStringInfoChar(buf, ')'); } static Oid -findFunction(Oid typoid, char *name) -{ - int i; - Oid result = InvalidOid; - HeapTuple proctup; - Form_pg_proc procform; - CatCList *catlist; - - catlist = SearchSysCacheList1(PROCNAMEARGSNSP, - CStringGetDatum(name)); - - for (i = 0; i < catlist->n_members; i++) - { - proctup = &catlist->members[i]->tuple; - procform = (Form_pg_proc) GETSTRUCT(proctup); - if (procform->proargtypes.values[0] == typoid) +findFunction(Oid typoid, char* name) { + int i; + Oid result = InvalidOid; + HeapTuple proctup; + Form_pg_proc procform; + CatCList* catlist; + + catlist = SearchSysCacheList1(PROCNAMEARGSNSP, CStringGetDatum(name)); + + for (i = 0; i < catlist->n_members; i++) { + proctup = &catlist->members[i]->tuple; + procform = (Form_pg_proc)GETSTRUCT(proctup); + if (procform->proargtypes.values[0] == typoid) #if PG_VERSION_NUM < 120000 - result = HeapTupleGetOid(proctup); + result = HeapTupleGetOid(proctup); #else - result = procform->oid; + result = procform->oid; #endif - } + } - ReleaseSysCacheList(catlist); + ReleaseSysCacheList(catlist); - return result; + return result; } /* @@ -3431,314 +3555,301 @@ findFunction(Oid typoid, char *name) * operations, we always parenthesize the arguments. */ static void -deparseOpExpr(OpExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - HeapTuple tuple; - Form_pg_operator form; - char oprkind; - ListCell *arg; - CustomObjectDef *cdef; - - /* Retrieve information about the operator from system catalog. */ - tuple = SearchSysCache1(OPEROID, ObjectIdGetDatum(node->opno)); - if (!HeapTupleIsValid(tuple)) - elog(ERROR, "cache lookup failed for operator %u", node->opno); - - form = (Form_pg_operator) GETSTRUCT(tuple); - oprkind = form->oprkind; - - /* Sanity check. */ - Assert((oprkind == 'r' && list_length(node->args) == 1) || - (oprkind == 'l' && list_length(node->args) == 1) || - (oprkind == 'b' && list_length(node->args) == 2)); - - cdef = chfdw_check_for_custom_operator(node->opno, form); - if (cdef) - { - switch (cdef->cf_type) - { - case CF_REGEX_MATCH: - case CF_REGEX_NO_MATCH: - case CF_REGEX_ICASE_MATCH: - case CF_REGEX_ICASE_NO_MATCH: - { - bool negated = (cdef->cf_type == CF_REGEX_NO_MATCH || - cdef->cf_type == CF_REGEX_ICASE_NO_MATCH); - bool icase = (cdef->cf_type == CF_REGEX_ICASE_MATCH || - cdef->cf_type == CF_REGEX_ICASE_NO_MATCH); - - if (negated) - appendStringInfoString(buf, "(NOT "); - appendStringInfoString(buf, "match("); - deparseExpr(linitial(node->args), context); - appendStringInfo(buf, ", concat('(?%s-s)', ", icase ? "i" : ""); - deparseExpr(lsecond(node->args), context); - appendStringInfoString(buf, "))"); - if (negated) - appendStringInfoChar(buf, ')'); - goto cleanup; - } - break; - case CF_TIMESTAMPTZ_PL_INTERVAL: - { - deparseIntervalOp(linitial(node->args), - list_nth(node->args, 1), context, true); - goto cleanup; - } - break; - case CF_HSTORE_FETCHVAL: - { - Expr *arg1 = linitial(node->args); - Expr *arg2 = list_nth(node->args, 1); - - if (IsA(arg1, Const)) - { - Const *constval = (Const *) arg1; - Oid akeys = findFunction(constval->consttype, "akeys"); - Oid avalues = findFunction(constval->consttype, "avals"); - - /* vals[nullif(indexOf(keys,toString(arg1)), 0)] */ - appendStringInfoChar(buf, '('); - deparseArray(OidFunctionCall1(avalues, constval->constvalue), context); - appendStringInfoString(buf, "[nullif(indexOf("); - deparseArray(OidFunctionCall1(akeys, constval->constvalue), context); - appendStringInfoChar(buf, ','); - deparseExpr(arg2, context); - appendStringInfoString(buf, "), 0)])"); - } - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_clickhouse supports hstore fetchval " - "only for scalars"))); - - goto cleanup; - } - break; - case CF_ARRAY_CONTAINS: - { - appendStringInfoString(buf, "hasAll("); - deparseExpr(linitial(node->args), context); - appendStringInfoString(buf, ", "); - deparseExpr(lsecond(node->args), context); - appendStringInfoChar(buf, ')'); - goto cleanup; - } - break; - case CF_ARRAY_CONTAINED_BY: - { - appendStringInfoString(buf, "hasAll("); - deparseExpr(lsecond(node->args), context); - appendStringInfoString(buf, ", "); - deparseExpr(linitial(node->args), context); - appendStringInfoChar(buf, ')'); - goto cleanup; - } - break; - case CF_ARRAY_OVERLAP: - { - appendStringInfoString(buf, "hasAny("); - deparseExpr(linitial(node->args), context); - appendStringInfoString(buf, ", "); - deparseExpr(lsecond(node->args), context); - appendStringInfoChar(buf, ')'); - goto cleanup; - } - break; - case CF_JSON_FETCHVAL: - case CF_JSON_FETCHVAL_TEXT: - { - Expr *arg1 = linitial(node->args); - Expr *arg2 = lsecond(node->args); - - /* - * Convert jsonb ->> 'key' to ClickHouse dot notation: - * column.key Convert jsonb -> 'key' to JSON-wrapped dot - * notation: toJSONString(column.key) The -> operator - * returns jsonb, so we need to wrap the result with - * toJSONString() for type compatibility. - */ - if (IsA(arg2, Const)) - { - Const *key = (Const *) arg2; - - if (key->consttype == TEXTOID && !key->constisnull) - { - char *keyval = TextDatumGetCString(key->constvalue); - - if (cdef->cf_type == CF_JSON_FETCHVAL) - appendStringInfoString(buf, "toJSONString("); - - deparseExpr(arg1, context); - appendStringInfoChar(buf, '.'); - appendStringInfoString(buf, - quote_identifier(keyval)); - - if (cdef->cf_type == CF_JSON_FETCHVAL) - appendStringInfoChar(buf, ')'); - - pfree(keyval); - goto cleanup; - } - } - /* Non-text key: fall through to standard deparse */ - } - break; - default: /* keep compiler quiet */ ; - } - } - - if ((node->opresulttype == INT2OID || - node->opresulttype == INT4OID || - node->opresulttype == INT8OID) && - strcmp(NameStr(form->oprname), "/") == 0) - { - char *s = ch_format_type_extended(node->opresulttype, 0, 0); - - appendStringInfo(buf, "to%s", s); - } - - /* Always parenthesize the expression. */ - appendStringInfoChar(buf, '('); - - /* Deparse left operand. */ - if (oprkind == 'r' || oprkind == 'b') - { - arg = list_head(node->args); - - /* - * Check for TestCaseExpr, in statements like CASE expr WHEN . - * Basically they would look like, OpExpr->(TestCaseExpr, Const). We - * should just skip first arg and deparse second. - */ - if (IsA(lfirst(arg), CaseTestExpr)) - { - arg = list_tail(node->args); - deparseExpr(lfirst(arg), context); - appendStringInfoChar(buf, ')'); - goto cleanup; - } - - deparseExpr(lfirst(arg), context); - appendStringInfoChar(buf, ' '); - } - - /* - * Here we add support of special case like ( || ' days')::interval. - * We convert it to () day. INTERVAL keyword added earlier - */ - if (context->interval_op && strcmp(NameStr(form->oprname), "||") == 0) - { - Const *right = lfirst(list_tail(node->args)); - - if (IsA(right, Const) && right->consttype == TEXTOID) - { - char *s = TextDatumGetCString(right->constvalue); - - if (strstr(s, "day") != NULL) - appendStringInfoString(buf, ") day"); - else if (strstr(s, "year") != NULL) - appendStringInfoString(buf, ") year"); - else if (strstr(s, "month") != NULL) - appendStringInfoString(buf, ") month"); - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("unsupported type of interval"))); - pfree(s); - - goto cleanup; - } - } - - /* Deparse operator name. */ - deparseOperatorName(buf, form); - - /* Deparse right operand. */ - if (oprkind == 'l' || oprkind == 'b') - { - arg = list_tail(node->args); - appendStringInfoChar(buf, ' '); - deparseExpr(lfirst(arg), context); - } - - appendStringInfoChar(buf, ')'); +deparseOpExpr(OpExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + HeapTuple tuple; + Form_pg_operator form; + char oprkind; + ListCell* arg; + CustomObjectDef* cdef; + + /* Retrieve information about the operator from system catalog. */ + tuple = SearchSysCache1(OPEROID, ObjectIdGetDatum(node->opno)); + if (!HeapTupleIsValid(tuple)) { + elog(ERROR, "cache lookup failed for operator %u", node->opno); + } + + form = (Form_pg_operator)GETSTRUCT(tuple); + oprkind = form->oprkind; + + /* Sanity check. */ + Assert( + (oprkind == 'r' && list_length(node->args) == 1) || + (oprkind == 'l' && list_length(node->args) == 1) || + (oprkind == 'b' && list_length(node->args) == 2) + ); + + cdef = chfdw_check_for_custom_operator(node->opno, form); + if (cdef) { + switch (cdef->cf_type) { + case CF_REGEX_MATCH: + case CF_REGEX_NO_MATCH: + case CF_REGEX_ICASE_MATCH: + case CF_REGEX_ICASE_NO_MATCH: { + bool negated = + (cdef->cf_type == CF_REGEX_NO_MATCH || + cdef->cf_type == CF_REGEX_ICASE_NO_MATCH); + bool icase = + (cdef->cf_type == CF_REGEX_ICASE_MATCH || + cdef->cf_type == CF_REGEX_ICASE_NO_MATCH); + + if (negated) { + appendStringInfoString(buf, "(NOT "); + } + appendStringInfoString(buf, "match("); + deparseExpr(linitial(node->args), context); + appendStringInfo(buf, ", concat('(?%s-s)', ", icase ? "i" : ""); + deparseExpr(lsecond(node->args), context); + appendStringInfoString(buf, "))"); + if (negated) { + appendStringInfoChar(buf, ')'); + } + goto cleanup; + } break; + case CF_TIMESTAMPTZ_PL_INTERVAL: { + deparseIntervalOp( + linitial(node->args), list_nth(node->args, 1), context, true + ); + goto cleanup; + } break; + case CF_HSTORE_FETCHVAL: { + Expr* arg1 = linitial(node->args); + Expr* arg2 = list_nth(node->args, 1); + + if (IsA(arg1, Const)) { + Const* constval = (Const*)arg1; + Oid akeys = findFunction(constval->consttype, "akeys"); + Oid avalues = findFunction(constval->consttype, "avals"); + + /* vals[nullif(indexOf(keys,toString(arg1)), 0)] */ + appendStringInfoChar(buf, '('); + deparseArray(OidFunctionCall1(avalues, constval->constvalue), context); + appendStringInfoString(buf, "[nullif(indexOf("); + deparseArray(OidFunctionCall1(akeys, constval->constvalue), context); + appendStringInfoChar(buf, ','); + deparseExpr(arg2, context); + appendStringInfoString(buf, "), 0)])"); + } else { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "pg_clickhouse supports hstore fetchval " + "only for scalars" + )) + ); + } + + goto cleanup; + } break; + case CF_ARRAY_CONTAINS: { + appendStringInfoString(buf, "hasAll("); + deparseExpr(linitial(node->args), context); + appendStringInfoString(buf, ", "); + deparseExpr(lsecond(node->args), context); + appendStringInfoChar(buf, ')'); + goto cleanup; + } break; + case CF_ARRAY_CONTAINED_BY: { + appendStringInfoString(buf, "hasAll("); + deparseExpr(lsecond(node->args), context); + appendStringInfoString(buf, ", "); + deparseExpr(linitial(node->args), context); + appendStringInfoChar(buf, ')'); + goto cleanup; + } break; + case CF_ARRAY_OVERLAP: { + appendStringInfoString(buf, "hasAny("); + deparseExpr(linitial(node->args), context); + appendStringInfoString(buf, ", "); + deparseExpr(lsecond(node->args), context); + appendStringInfoChar(buf, ')'); + goto cleanup; + } break; + case CF_JSON_FETCHVAL: + case CF_JSON_FETCHVAL_TEXT: { + Expr* arg1 = linitial(node->args); + Expr* arg2 = lsecond(node->args); + + /* + * Convert jsonb ->> 'key' to ClickHouse dot notation: + * column.key Convert jsonb -> 'key' to JSON-wrapped dot + * notation: toJSONString(column.key) The -> operator + * returns jsonb, so we need to wrap the result with + * toJSONString() for type compatibility. + */ + if (IsA(arg2, Const)) { + Const* key = (Const*)arg2; + + if (key->consttype == TEXTOID && !key->constisnull) { + char* keyval = TextDatumGetCString(key->constvalue); + + if (cdef->cf_type == CF_JSON_FETCHVAL) { + appendStringInfoString(buf, "toJSONString("); + } + + deparseExpr(arg1, context); + appendStringInfoChar(buf, '.'); + appendStringInfoString(buf, quote_identifier(keyval)); + + if (cdef->cf_type == CF_JSON_FETCHVAL) { + appendStringInfoChar(buf, ')'); + } + + pfree(keyval); + goto cleanup; + } + } + /* Non-text key: fall through to standard deparse */ + } break; + default: /* keep compiler quiet */; + } + } + + if ((node->opresulttype == INT2OID || node->opresulttype == INT4OID || + node->opresulttype == INT8OID) && + strcmp(NameStr(form->oprname), "/") == 0) { + char* s = ch_format_type_extended(node->opresulttype, 0, 0); + + appendStringInfo(buf, "to%s", s); + } + + /* Always parenthesize the expression. */ + appendStringInfoChar(buf, '('); + + /* Deparse left operand. */ + if (oprkind == 'r' || oprkind == 'b') { + arg = list_head(node->args); + + /* + * Check for TestCaseExpr, in statements like CASE expr WHEN . + * Basically they would look like, OpExpr->(TestCaseExpr, Const). We + * should just skip first arg and deparse second. + */ + if (IsA(lfirst(arg), CaseTestExpr)) { + arg = list_tail(node->args); + deparseExpr(lfirst(arg), context); + appendStringInfoChar(buf, ')'); + goto cleanup; + } + + deparseExpr(lfirst(arg), context); + appendStringInfoChar(buf, ' '); + } + + /* + * Here we add support of special case like ( || ' days')::interval. + * We convert it to () day. INTERVAL keyword added earlier + */ + if (context->interval_op && strcmp(NameStr(form->oprname), "||") == 0) { + Const* right = lfirst(list_tail(node->args)); + + if (IsA(right, Const) && right->consttype == TEXTOID) { + char* s = TextDatumGetCString(right->constvalue); + + if (strstr(s, "day") != NULL) { + appendStringInfoString(buf, ") day"); + } else if (strstr(s, "year") != NULL) { + appendStringInfoString(buf, ") year"); + } else if (strstr(s, "month") != NULL) { + appendStringInfoString(buf, ") month"); + } else { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("unsupported type of interval")) + ); + } + pfree(s); + + goto cleanup; + } + } + + /* Deparse operator name. */ + deparseOperatorName(buf, form); + + /* Deparse right operand. */ + if (oprkind == 'l' || oprkind == 'b') { + arg = list_tail(node->args); + appendStringInfoChar(buf, ' '); + deparseExpr(lfirst(arg), context); + } + + appendStringInfoChar(buf, ')'); cleanup: - ReleaseSysCache(tuple); + ReleaseSysCache(tuple); } /* * Print the name of an operator. */ static void -deparseOperatorName(StringInfo buf, Form_pg_operator opform) -{ - char *opname; - - opname = NameStr(opform->oprname); - - if (strcmp(opname, "~~") == 0) - appendStringInfoString(buf, "LIKE"); - else if (strcmp(opname, "~~*") == 0) - appendStringInfoString(buf, "ILIKE"); - else if (strcmp(opname, "!~~") == 0) - appendStringInfoString(buf, "NOT LIKE"); - else if (strcmp(opname, "!~~*") == 0) - appendStringInfoString(buf, "NOT ILIKE"); - else - appendStringInfoString(buf, opname); +deparseOperatorName(StringInfo buf, Form_pg_operator opform) { + char* opname; + + opname = NameStr(opform->oprname); + + if (strcmp(opname, "~~") == 0) { + appendStringInfoString(buf, "LIKE"); + } else if (strcmp(opname, "~~*") == 0) { + appendStringInfoString(buf, "ILIKE"); + } else if (strcmp(opname, "!~~") == 0) { + appendStringInfoString(buf, "NOT LIKE"); + } else if (strcmp(opname, "!~~*") == 0) { + appendStringInfoString(buf, "NOT ILIKE"); + } else { + appendStringInfoString(buf, opname); + } } /* * Deparse IS DISTINCT FROM. */ static void -deparseDistinctExpr(DistinctExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; +deparseDistinctExpr(DistinctExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; - Assert(list_length(node->args) == 2); + Assert(list_length(node->args) == 2); - appendStringInfoChar(buf, '('); - deparseExpr(linitial(node->args), context); - appendStringInfoString(buf, " IS DISTINCT FROM "); - deparseExpr(lsecond(node->args), context); - appendStringInfoChar(buf, ')'); + appendStringInfoChar(buf, '('); + deparseExpr(linitial(node->args), context); + appendStringInfoString(buf, " IS DISTINCT FROM "); + deparseExpr(lsecond(node->args), context); + appendStringInfoChar(buf, ')'); } static void -deparseNullIfExpr(NullIfExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; +deparseNullIfExpr(NullIfExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; - Assert(list_length(node->args) == 2); + Assert(list_length(node->args) == 2); - appendStringInfoString(buf, "NULLIF("); - deparseExpr(linitial(node->args), context); - appendStringInfoChar(buf, ','); - deparseExpr(lsecond(node->args), context); - appendStringInfoChar(buf, ')'); + appendStringInfoString(buf, "NULLIF("); + deparseExpr(linitial(node->args), context); + appendStringInfoChar(buf, ','); + deparseExpr(lsecond(node->args), context); + appendStringInfoChar(buf, ')'); } static void -deparseAsIn(ScalarArrayOpExpr * node, deparse_expr_cxt * context, int optype) -{ - StringInfo buf = context->buf; - Expr *arg1 = linitial(node->args); - Expr *arg2 = lsecond(node->args); - - deparseExpr(arg1, context); - if (optype == 1) - appendStringInfoString(buf, " IN "); - else - appendStringInfoString(buf, " NOT IN "); - - Assert(IsA(arg2, Const)); - context->array_as_tuple = true; - deparseExpr(arg2, context); - context->array_as_tuple = false; +deparseAsIn(ScalarArrayOpExpr* node, deparse_expr_cxt* context, int optype) { + StringInfo buf = context->buf; + Expr* arg1 = linitial(node->args); + Expr* arg2 = lsecond(node->args); + + deparseExpr(arg1, context); + if (optype == 1) { + appendStringInfoString(buf, " IN "); + } else { + appendStringInfoString(buf, " NOT IN "); + } + + Assert(IsA(arg2, Const)); + context->array_as_tuple = true; + deparseExpr(arg2, context); + context->array_as_tuple = false; } /* @@ -3746,210 +3857,200 @@ deparseAsIn(ScalarArrayOpExpr * node, deparse_expr_cxt * context, int optype) * priority of operations, we always parenthesize the arguments. */ static void -deparseScalarArrayOpExpr(ScalarArrayOpExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - Expr *arg1; - Expr *arg2; - - /* Retrieve information about the operator from system catalog. */ - int optype = chfdw_is_equal_op(node->opno); - - if (optype == 0) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_clickhouse supports only equal (not equal) operations on ANY/ALL functions"))); - - /* Sanity check. */ - Assert(list_length(node->args) == 2); - - appendStringInfoChar(buf, '('); - if (node->useOr) - { - arg2 = lsecond(node->args); - - /* very narrow case for = ANY(ARRAY) */ - if (optype == 1 && IsA(arg2, Const)) - deparseAsIn(node, context, optype); - else - { - if (optype == 1) - appendStringInfoString(buf, "has("); - else - appendStringInfoString(buf, "not has("); - - /* Deparse right operand. */ - deparseExpr(arg2, context); - appendStringInfoChar(buf, ','); - - /* Deparse left operand. */ - arg1 = linitial(node->args); - deparseExpr(arg1, context); - - /* Close function call */ - appendStringInfoChar(buf, ')'); - } - } - else - { - arg2 = lsecond(node->args); - - /* very narrow case for <> ALL(ARRAY) */ - if (optype == 2 && IsA(arg2, Const)) - deparseAsIn(node, context, optype); - else - { - appendStringInfoString(buf, "countEqual("); - - /* Deparse right operand. */ - arg2 = lsecond(node->args); - deparseExpr(arg2, context); - appendStringInfoChar(buf, ','); - - /* Deparse left operand. */ - arg1 = linitial(node->args); - deparseExpr(arg1, context); - - /* Close function call */ - if (optype == 1) - { - appendStringInfoString(buf, ") = length("); - - /* Deparse right operand again */ - deparseExpr(arg2, context); - appendStringInfoChar(buf, ')'); - } - else - { - appendStringInfoString(buf, ") = 0"); - } - } - } - - appendStringInfoChar(buf, ')'); +deparseScalarArrayOpExpr(ScalarArrayOpExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + Expr* arg1; + Expr* arg2; + + /* Retrieve information about the operator from system catalog. */ + int optype = chfdw_is_equal_op(node->opno); + + if (optype == 0) { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "pg_clickhouse supports only equal (not equal) operations on ANY/ALL " + "functions" + )) + ); + } + + /* Sanity check. */ + Assert(list_length(node->args) == 2); + + appendStringInfoChar(buf, '('); + if (node->useOr) { + arg2 = lsecond(node->args); + + /* very narrow case for = ANY(ARRAY) */ + if (optype == 1 && IsA(arg2, Const)) { + deparseAsIn(node, context, optype); + } else { + if (optype == 1) { + appendStringInfoString(buf, "has("); + } else { + appendStringInfoString(buf, "not has("); + } + + /* Deparse right operand. */ + deparseExpr(arg2, context); + appendStringInfoChar(buf, ','); + + /* Deparse left operand. */ + arg1 = linitial(node->args); + deparseExpr(arg1, context); + + /* Close function call */ + appendStringInfoChar(buf, ')'); + } + } else { + arg2 = lsecond(node->args); + + /* very narrow case for <> ALL(ARRAY) */ + if (optype == 2 && IsA(arg2, Const)) { + deparseAsIn(node, context, optype); + } else { + appendStringInfoString(buf, "countEqual("); + + /* Deparse right operand. */ + arg2 = lsecond(node->args); + deparseExpr(arg2, context); + appendStringInfoChar(buf, ','); + + /* Deparse left operand. */ + arg1 = linitial(node->args); + deparseExpr(arg1, context); + + /* Close function call */ + if (optype == 1) { + appendStringInfoString(buf, ") = length("); + + /* Deparse right operand again */ + deparseExpr(arg2, context); + appendStringInfoChar(buf, ')'); + } else { + appendStringInfoString(buf, ") = 0"); + } + } + } + + appendStringInfoChar(buf, ')'); } /* * Deparse a RelabelType (binary-compatible cast) node. */ static void -deparseRelabelType(RelabelType * node, deparse_expr_cxt * context) -{ - deparseExpr(node->arg, context); +deparseRelabelType(RelabelType* node, deparse_expr_cxt* context) { + deparseExpr(node->arg, context); } /* * Deparse a BoolExpr node. */ static void -deparseBoolExpr(BoolExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - const char *op = NULL; /* keep compiler quiet */ - bool first; - ListCell *lc; - - switch (node->boolop) - { - case AND_EXPR: - op = "AND"; - break; - case OR_EXPR: - op = "OR"; - break; - case NOT_EXPR: - appendStringInfoString(buf, "(NOT "); - deparseExpr(linitial(node->args), context); - appendStringInfoChar(buf, ')'); - return; - } - - appendStringInfoChar(buf, '('); - first = true; - foreach(lc, node->args) - { - if (!first) - { - appendStringInfo(buf, " %s ", op); - } - deparseExpr((Expr *) lfirst(lc), context); - first = false; - } - appendStringInfoChar(buf, ')'); +deparseBoolExpr(BoolExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + const char* op = NULL; /* keep compiler quiet */ + bool first; + ListCell* lc; + + switch (node->boolop) { + case AND_EXPR: + op = "AND"; + break; + case OR_EXPR: + op = "OR"; + break; + case NOT_EXPR: + appendStringInfoString(buf, "(NOT "); + deparseExpr(linitial(node->args), context); + appendStringInfoChar(buf, ')'); + return; + } + + appendStringInfoChar(buf, '('); + first = true; + foreach (lc, node->args) { + if (!first) { + appendStringInfo(buf, " %s ", op); + } + deparseExpr((Expr*)lfirst(lc), context); + first = false; + } + appendStringInfoChar(buf, ')'); } /* * Deparse IS [NOT] NULL expression. */ static void -deparseNullTest(NullTest * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - - appendStringInfoChar(buf, '('); - deparseExpr(node->arg, context); - - /* - * For scalar inputs, we prefer to print as IS [NOT] NULL, which is - * shorter and traditional. If it's a rowtype input but we're applying a - * scalar test, must print IS [NOT] DISTINCT FROM NULL to be semantically - * correct. - */ - if (node->argisrow || !type_is_rowtype(exprType((Node *) node->arg))) - { - if (node->nulltesttype == IS_NULL) - appendStringInfoString(buf, " IS NULL)"); - else - appendStringInfoString(buf, " IS NOT NULL)"); - } - else - { - if (node->nulltesttype == IS_NULL) - appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL)"); - else - appendStringInfoString(buf, " IS DISTINCT FROM NULL)"); - } +deparseNullTest(NullTest* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + + appendStringInfoChar(buf, '('); + deparseExpr(node->arg, context); + + /* + * For scalar inputs, we prefer to print as IS [NOT] NULL, which is + * shorter and traditional. If it's a rowtype input but we're applying a + * scalar test, must print IS [NOT] DISTINCT FROM NULL to be semantically + * correct. + */ + if (node->argisrow || !type_is_rowtype(exprType((Node*)node->arg))) { + if (node->nulltesttype == IS_NULL) { + appendStringInfoString(buf, " IS NULL)"); + } else { + appendStringInfoString(buf, " IS NOT NULL)"); + } + } else { + if (node->nulltesttype == IS_NULL) { + appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL)"); + } else { + appendStringInfoString(buf, " IS DISTINCT FROM NULL)"); + } + } } /* * Deparse ARRAY[...] construct. */ static void -deparseArrayExpr(ArrayExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; +deparseArrayExpr(ArrayExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; - if (node->elements == NIL) - appendStringInfoString(buf, "CAST("); + if (node->elements == NIL) { + appendStringInfoString(buf, "CAST("); + } - appendStringInfoString(buf, "["); - deparseArrayList(node, context); - appendStringInfoChar(buf, ']'); + appendStringInfoString(buf, "["); + deparseArrayList(node, context); + appendStringInfoChar(buf, ']'); - /* If the array is empty, we need an explicit cast to the array type. */ - if (node->elements == NIL) - appendStringInfo(buf, ", '%s')", - deparse_type_name(node->array_typeid, -1)); + /* If the array is empty, we need an explicit cast to the array type. */ + if (node->elements == NIL) { + appendStringInfo(buf, ", '%s')", deparse_type_name(node->array_typeid, -1)); + } } /* * Deparse an array to a list, as in converting a variadic function call's * array to the list of individual values. -*/ + */ static void -deparseArrayList(ArrayExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - bool first = true; - ListCell *lc; - - foreach(lc, node->elements) - { - if (!first) - appendStringInfoString(buf, ", "); - deparseExpr(lfirst(lc), context); - first = false; - } +deparseArrayList(ArrayExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + bool first = true; + ListCell* lc; + + foreach (lc, node->elements) { + if (!first) { + appendStringInfoString(buf, ", "); + } + deparseExpr(lfirst(lc), context); + first = false; + } } /* @@ -3958,81 +4059,91 @@ deparseArrayList(ArrayExpr * node, deparse_expr_cxt * context) * they're not (yet) supported by ClickHouse aggregates. */ static void -appendAggOrderBySuffix(Oid sortop, Oid sortcoltype, bool nulls_first, - deparse_expr_cxt * context) -{ - /* StringInfo buf = context->buf; */ - TypeCacheEntry *typentry; - - /* See whether operator is default < or > for sort expr's datatype. */ - typentry = lookup_type_cache(sortcoltype, - TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); - - if (sortop == typentry->lt_opr) - { - /* - * Okay. - * - * appendStringInfoString(buf, " ASC"); - */ - ; - } - else if (sortop == typentry->gt_opr) - { - /* appendStringInfoString(buf, " DESC"); */ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_clickhouse: ClickHouse does not support \"DESC\" in aggregate expressions"))); - } - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_clickhouse: ClickHouse does not support \"USING\" in aggregate expressions"))); - - if (nulls_first) - { - /* appendStringInfoString(buf, " NULLS FIRST"); */ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_clickhouse: ClickHouse does not support \"NULLS FIRST\" in aggregate expressions"))); - } - else - { - /* - * Okay unless DESC support added. - * - * appendStringInfoString(buf, " NULLS LAST"); - */ - } +appendAggOrderBySuffix( + Oid sortop, + Oid sortcoltype, + bool nulls_first, + deparse_expr_cxt* context +) { + /* StringInfo buf = context->buf; */ + TypeCacheEntry* typentry; + + /* See whether operator is default < or > for sort expr's datatype. */ + typentry = lookup_type_cache(sortcoltype, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); + + if (sortop == typentry->lt_opr) { + /* + * Okay. + * + * appendStringInfoString(buf, " ASC"); + */ + ; + } else if (sortop == typentry->gt_opr) { + /* appendStringInfoString(buf, " DESC"); */ + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "pg_clickhouse: ClickHouse does not support \"DESC\" in aggregate " + "expressions" + )) + ); + } else { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "pg_clickhouse: ClickHouse does not support \"USING\" in aggregate " + "expressions" + )) + ); + } + + if (nulls_first) { + /* appendStringInfoString(buf, " NULLS FIRST"); */ + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "pg_clickhouse: ClickHouse does not support \"NULLS FIRST\" in " + "aggregate expressions" + )) + ); + } else { + /* + * Okay unless DESC support added. + * + * appendStringInfoString(buf, " NULLS LAST"); + */ + } } - /* * Append ORDER BY arguments to aggregate function arguments. */ static void -appendAggOrderBy(List * orderList, List * targetList, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - ListCell *lc; - bool first = true; - - foreach(lc, orderList) - { - SortGroupClause *srt = (SortGroupClause *) lfirst(lc); - Node *sortexpr; - - if (!first) - appendStringInfoString(buf, ", "); - first = false; - - /* Deparse the sort expression proper. */ - sortexpr = deparseSortGroupClause(srt->tleSortGroupRef, targetList, - false, context); - /* Add decoration as needed. */ - appendAggOrderBySuffix(srt->sortop, exprType(sortexpr), srt->nulls_first, - context); - } +appendAggOrderBy(List* orderList, List* targetList, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + ListCell* lc; + bool first = true; + + foreach (lc, orderList) { + SortGroupClause* srt = (SortGroupClause*)lfirst(lc); + Node* sortexpr; + + if (!first) { + appendStringInfoString(buf, ", "); + } + first = false; + + /* Deparse the sort expression proper. */ + sortexpr = + deparseSortGroupClause(srt->tleSortGroupRef, targetList, false, context); + /* Add decoration as needed. */ + appendAggOrderBySuffix( + srt->sortop, exprType(sortexpr), srt->nulls_first, context + ); + } } /* @@ -4046,282 +4157,265 @@ appendAggOrderBy(List * orderList, List * targetList, deparse_expr_cxt * context * and has "aggregatefunction" FDW column option set. */ static bool -aggref_on_aggregate_function(Aggref * node, deparse_expr_cxt * context) -{ - List *vars = pull_var_clause((Node *) node->args, 0); - ListCell *lc; - Relids relids = context->scanrel->relids; - bool found = false; - - foreach(lc, vars) - { - Var *var = (Var *) lfirst(lc); - - if (bms_is_member(var->varno, relids) && var->varlevelsup == 0) - { - RangeTblEntry *rte = planner_rt_fetch(var->varno, context->root); - - /* - * Check FDW column options directly rather than relying on - * custom_columns_cache, which can be invalidated between - * GetForeignRelSize and deparse. - */ - List *options = GetForeignColumnOptions(rte->relid, - var->varattno); - ListCell *olc; - - foreach(olc, options) - { - DefElem *def = (DefElem *) lfirst(olc); - - if (strcmp(def->defname, "aggregatefunction") == 0) - { - found = true; - break; - } - } - if (found) - break; - } - } - list_free(vars); - return found; +aggref_on_aggregate_function(Aggref* node, deparse_expr_cxt* context) { + List* vars = pull_var_clause((Node*)node->args, 0); + ListCell* lc; + Relids relids = context->scanrel->relids; + bool found = false; + + foreach (lc, vars) { + Var* var = (Var*)lfirst(lc); + + if (bms_is_member(var->varno, relids) && var->varlevelsup == 0) { + RangeTblEntry* rte = planner_rt_fetch(var->varno, context->root); + + /* + * Check FDW column options directly rather than relying on + * custom_columns_cache, which can be invalidated between + * GetForeignRelSize and deparse. + */ + List* options = GetForeignColumnOptions(rte->relid, var->varattno); + ListCell* olc; + + foreach (olc, options) { + DefElem* def = (DefElem*)lfirst(olc); + + if (strcmp(def->defname, "aggregatefunction") == 0) { + found = true; + break; + } + } + if (found) { + break; + } + } + } + list_free(vars); + return found; } /* * Deparse an Aggref node. */ static void -deparseAggref(Aggref * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - CustomObjectDef *cdef; - CHFdwRelationInfo *fpinfo = context->scanrel->fdw_private; - bool aggfilter = false; - bool sign_count_filter = false; - uint8 brcount = 1; - bool use_variadic; - char *name = get_func_name(node->aggfnoid); - - /* Only basic, non-split aggregation accepted. */ - Assert(node->aggsplit == AGGSPLIT_SIMPLE); - - /* Check if need to print expand VARIADIC (cf. ruleutils.c) */ - use_variadic = node->aggvariadic; - - /* Find aggregate name from aggfnoid which is a pg_proc entry */ - cdef = context->func; - context->func = appendFunctionName(node->aggfnoid, context); - - /* 'If' part */ - if (context->func && context->func->cf_type == CF_SIGN_COUNT && !node->aggstar) - sign_count_filter = true; - - if (aggref_on_aggregate_function(node, context)) - appendStringInfoString(buf, "Merge"); - - if (node->aggfilter || sign_count_filter) - { - aggfilter = true; - appendStringInfoString(buf, "If"); - } - - /* - * groupConcat(delimiter)(expr): emit delimiter as parameterized arg, then - * only the first non-junk arg (the expression). - */ - if (context->func && context->func->cf_type == CF_STRING_AGG) - { - ListCell *arg; - - /* Emit delimiter (2nd non-junk arg) as parameterized arg. */ - appendStringInfoChar(buf, '('); - foreach(arg, node->args) - { - TargetEntry *tle = (TargetEntry *) lfirst(arg); - - if (tle->resjunk) - continue; - if (arg == list_head(node->args)) - continue; - deparseExpr((Expr *) tle->expr, context); - break; - } - appendStringInfoString(buf, ")("); - - /* Emit expression (1st non-junk arg). */ - if (node->aggdistinct != NIL) - appendStringInfoString(buf, "DISTINCT "); - deparseExpr((Expr *) ((TargetEntry *) linitial(node->args))->expr, - context); - appendStringInfoChar(buf, ')'); - - context->func = cdef; - return; - } - - appendStringInfoChar(buf, '('); - - /* Explained below. */ - bool omit_star = false; - - if (AGGKIND_IS_ORDERED_SET(node->aggkind)) - { - /* Emit direct args as ClickHouse parameterized args. */ - ListCell *arg; - bool first = true; - - Assert(!node->aggvariadic); - Assert(node->aggorder != NIL); - - foreach(arg, node->aggdirectargs) - { - if (!first) - appendStringInfoString(buf, ", "); - first = false; - - deparseExpr((Expr *) lfirst(arg), context); - } - - /* Close parameter args and start regular args. */ - appendStringInfoString(buf, ")("); - /* Emit `WITHIN GROUP (ORDER BY ..)` args with no closing paren. */ - context->no_sort_parens = true; - appendAggOrderBy(node->aggorder, node->args, context); - context->no_sort_parens = false; - } - else - { - /* Add DISTINCT */ - if (node->aggdistinct != NIL) - appendStringInfoString(buf, "DISTINCT "); - - /* aggstar can be set only in zero-argument aggregates */ - if (node->aggstar) - { - /* - * Omit * for COUNT(*) but not COUNT(DISTINCT *) - * https://github.com/ClickHouse/pg_clickhouse/issues/25 To be - * fixed in ClickHouse 25.11, so can be omitted once released. - * https://github.com/ClickHouse/ClickHouse/pull/89373 - * - * XXX Once ClickHouse has made a release fixing this issue, - * consider adding the Client::ServerInfo struct returned from - * Client::GetServerInfo() to deparse_expr_cxt so we can allow * - * to be passed through for the fixed version. - */ - omit_star = node->aggfilter && node->aggdistinct == NIL && !strcmp(name, "count"); - if (context->func && context->func->cf_type == CF_SIGN_COUNT) - { - Assert(fpinfo && fpinfo->ch_table_engine == CH_COLLAPSING_MERGE_TREE); - appendStringInfoString(buf, fpinfo->ch_table_sign_field); - } - /* Omit * for COUNT (*) but not COUNT(DISTINCT *). */ - else if (!omit_star) - { - appendStringInfoChar(buf, '*'); - } - } - else - { - ListCell *arg; - bool first = true; - bool signMultiply = (context->func && - (context->func->cf_type == CF_SIGN_AVG || - context->func->cf_type == CF_SIGN_SUM)); - - /* Add all the arguments */ - if (sign_count_filter) - - /* - * in case if COUNT(col) we should get countIf(sign, col is - * not null) - */ - appendStringInfoString(buf, fpinfo->ch_table_sign_field); - else - { - /* default arguments output */ - foreach(arg, node->args) - { - TargetEntry *tle = (TargetEntry *) lfirst(arg); - Node *n = (Node *) tle->expr; - - if (tle->resjunk) - continue; - - if (!first) - appendStringInfoString(buf, ", "); - - first = false; - - if (use_variadic && lnext(node->args, arg) == NULL) - { - /* Convert variadic array to list of arguments. */ - Assert(nodeTag(n) == T_ArrayExpr); - deparseArrayList((ArrayExpr *) n, context); - } - else - { - deparseExpr((Expr *) n, context); - } - } - - if (signMultiply) - { - Assert(fpinfo->ch_table_engine == CH_COLLAPSING_MERGE_TREE); - appendStringInfoString(buf, " * "); - appendStringInfoString(buf, fpinfo->ch_table_sign_field); - } - } - } - } - - /* Add 'If' part condition */ - if (aggfilter) - { - /* No argument output for COUNT (*). */ - if (!omit_star) - appendStringInfoChar(buf, ','); - - if (node->aggfilter) - { - appendStringInfoString(buf, "(("); - deparseExpr((Expr *) node->aggfilter, context); - appendStringInfoString(buf, ") > 0)"); - } - - if (sign_count_filter) - { - if (node->aggfilter) - appendStringInfoString(buf, " AND "); - - appendStringInfoChar(buf, '('); - deparseExpr((Expr *) ((TargetEntry *) linitial(node->args))->expr, context); - appendStringInfoString(buf, ") IS NOT NULL"); - } - } - - while (brcount--) - appendStringInfoChar(buf, ')'); - - /* AVG stuff */ - if (context->func && context->func->cf_type == CF_SIGN_AVG) - { - appendStringInfoString(buf, " / sumIf("); - appendStringInfoString(buf, fpinfo->ch_table_sign_field); - appendStringInfoChar(buf, ','); - if (node->aggfilter) - { - deparseExpr((Expr *) node->aggfilter, context); - appendStringInfoString(buf, " AND "); - } - deparseExpr((Expr *) ((TargetEntry *) linitial(node->args))->expr, context); - appendStringInfoString(buf, " IS NOT NULL"); - appendStringInfoChar(buf, ')'); - } - - /* original */ - context->func = cdef; +deparseAggref(Aggref* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + CustomObjectDef* cdef; + CHFdwRelationInfo* fpinfo = context->scanrel->fdw_private; + bool aggfilter = false; + bool sign_count_filter = false; + uint8 brcount = 1; + bool use_variadic; + char* name = get_func_name(node->aggfnoid); + + /* Only basic, non-split aggregation accepted. */ + Assert(node->aggsplit == AGGSPLIT_SIMPLE); + + /* Check if need to print expand VARIADIC (cf. ruleutils.c) */ + use_variadic = node->aggvariadic; + + /* Find aggregate name from aggfnoid which is a pg_proc entry */ + cdef = context->func; + context->func = appendFunctionName(node->aggfnoid, context); + + /* 'If' part */ + if (context->func && context->func->cf_type == CF_SIGN_COUNT && !node->aggstar) { + sign_count_filter = true; + } + + if (aggref_on_aggregate_function(node, context)) { + appendStringInfoString(buf, "Merge"); + } + + if (node->aggfilter || sign_count_filter) { + aggfilter = true; + appendStringInfoString(buf, "If"); + } + + /* + * groupConcat(delimiter)(expr): emit delimiter as parameterized arg, then + * only the first non-junk arg (the expression). + */ + if (context->func && context->func->cf_type == CF_STRING_AGG) { + ListCell* arg; + + /* Emit delimiter (2nd non-junk arg) as parameterized arg. */ + appendStringInfoChar(buf, '('); + foreach (arg, node->args) { + TargetEntry* tle = (TargetEntry*)lfirst(arg); + + if (tle->resjunk) { + continue; + } + if (arg == list_head(node->args)) { + continue; + } + deparseExpr((Expr*)tle->expr, context); + break; + } + appendStringInfoString(buf, ")("); + + /* Emit expression (1st non-junk arg). */ + if (node->aggdistinct != NIL) { + appendStringInfoString(buf, "DISTINCT "); + } + deparseExpr((Expr*)((TargetEntry*)linitial(node->args))->expr, context); + appendStringInfoChar(buf, ')'); + + context->func = cdef; + return; + } + + appendStringInfoChar(buf, '('); + + /* Explained below. */ + bool omit_star = false; + + if (AGGKIND_IS_ORDERED_SET(node->aggkind)) { + /* Emit direct args as ClickHouse parameterized args. */ + ListCell* arg; + bool first = true; + + Assert(!node->aggvariadic); + Assert(node->aggorder != NIL); + + foreach (arg, node->aggdirectargs) { + if (!first) { + appendStringInfoString(buf, ", "); + } + first = false; + + deparseExpr((Expr*)lfirst(arg), context); + } + + /* Close parameter args and start regular args. */ + appendStringInfoString(buf, ")("); + /* Emit `WITHIN GROUP (ORDER BY ..)` args with no closing paren. */ + context->no_sort_parens = true; + appendAggOrderBy(node->aggorder, node->args, context); + context->no_sort_parens = false; + } else { + /* Add DISTINCT */ + if (node->aggdistinct != NIL) { + appendStringInfoString(buf, "DISTINCT "); + } + + /* aggstar can be set only in zero-argument aggregates */ + if (node->aggstar) { + /* + * Omit * for COUNT(*) but not COUNT(DISTINCT *) + * https://github.com/ClickHouse/pg_clickhouse/issues/25 To be + * fixed in ClickHouse 25.11, so can be omitted once released. + * https://github.com/ClickHouse/ClickHouse/pull/89373 + * + * XXX Once ClickHouse has made a release fixing this issue, + * consider adding the Client::ServerInfo struct returned from + * Client::GetServerInfo() to deparse_expr_cxt so we can allow * + * to be passed through for the fixed version. + */ + omit_star = + node->aggfilter && node->aggdistinct == NIL && !strcmp(name, "count"); + if (context->func && context->func->cf_type == CF_SIGN_COUNT) { + Assert(fpinfo && fpinfo->ch_table_engine == CH_COLLAPSING_MERGE_TREE); + appendStringInfoString(buf, fpinfo->ch_table_sign_field); + } + /* Omit * for COUNT (*) but not COUNT(DISTINCT *). */ + else if (!omit_star) { + appendStringInfoChar(buf, '*'); + } + } else { + ListCell* arg; + bool first = true; + bool signMultiply = + (context->func && (context->func->cf_type == CF_SIGN_AVG || + context->func->cf_type == CF_SIGN_SUM)); + + /* Add all the arguments */ + if (sign_count_filter) { + + /* + * in case if COUNT(col) we should get countIf(sign, col is + * not null) + */ + appendStringInfoString(buf, fpinfo->ch_table_sign_field); + } else { + /* default arguments output */ + foreach (arg, node->args) { + TargetEntry* tle = (TargetEntry*)lfirst(arg); + Node* n = (Node*)tle->expr; + + if (tle->resjunk) { + continue; + } + + if (!first) { + appendStringInfoString(buf, ", "); + } + + first = false; + + if (use_variadic && lnext(node->args, arg) == NULL) { + /* Convert variadic array to list of arguments. */ + Assert(nodeTag(n) == T_ArrayExpr); + deparseArrayList((ArrayExpr*)n, context); + } else { + deparseExpr((Expr*)n, context); + } + } + + if (signMultiply) { + Assert(fpinfo->ch_table_engine == CH_COLLAPSING_MERGE_TREE); + appendStringInfoString(buf, " * "); + appendStringInfoString(buf, fpinfo->ch_table_sign_field); + } + } + } + } + + /* Add 'If' part condition */ + if (aggfilter) { + /* No argument output for COUNT (*). */ + if (!omit_star) { + appendStringInfoChar(buf, ','); + } + + if (node->aggfilter) { + appendStringInfoString(buf, "(("); + deparseExpr((Expr*)node->aggfilter, context); + appendStringInfoString(buf, ") > 0)"); + } + + if (sign_count_filter) { + if (node->aggfilter) { + appendStringInfoString(buf, " AND "); + } + + appendStringInfoChar(buf, '('); + deparseExpr((Expr*)((TargetEntry*)linitial(node->args))->expr, context); + appendStringInfoString(buf, ") IS NOT NULL"); + } + } + + while (brcount--) { + appendStringInfoChar(buf, ')'); + } + + /* AVG stuff */ + if (context->func && context->func->cf_type == CF_SIGN_AVG) { + appendStringInfoString(buf, " / sumIf("); + appendStringInfoString(buf, fpinfo->ch_table_sign_field); + appendStringInfoChar(buf, ','); + if (node->aggfilter) { + deparseExpr((Expr*)node->aggfilter, context); + appendStringInfoString(buf, " AND "); + } + deparseExpr((Expr*)((TargetEntry*)linitial(node->args))->expr, context); + appendStringInfoString(buf, " IS NOT NULL"); + appendStringInfoChar(buf, ')'); + } + + /* original */ + context->func = cdef; } /* @@ -4330,389 +4424,369 @@ deparseAggref(Aggref * node, deparse_expr_cxt * context) * Generates: func_name(args) OVER (PARTITION BY ... ORDER BY ... frame) */ static void -deparseWindowFunc(WindowFunc * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - Query *query = context->root->parse; - WindowClause *wc; - ListCell *lc; - bool first; - char *funcname; - - /* Find the WindowClause referenced by this WindowFunc */ - wc = (WindowClause *) list_nth(query->windowClause, node->winref - 1); - - /* Emit function name */ - funcname = get_func_name(node->winfnoid); - CSTRING_TOLOWER(funcname); - appendStringInfoString(buf, funcname); - appendStringInfoChar(buf, '('); - - /* Emit arguments */ - first = true; - foreach(lc, node->args) - { - if (!first) - appendStringInfoString(buf, ", "); - first = false; - deparseExpr((Expr *) lfirst(lc), context); - } - - appendStringInfoString(buf, ") OVER ("); - - /* PARTITION BY */ - if (wc->partitionClause) - { - appendStringInfoString(buf, "PARTITION BY "); - first = true; - foreach(lc, wc->partitionClause) - { - SortGroupClause *sgc = (SortGroupClause *) lfirst(lc); - TargetEntry *tle = get_sortgroupref_tle(sgc->tleSortGroupRef, - query->targetList); - - if (!first) - appendStringInfoString(buf, ", "); - first = false; - deparseExpr((Expr *) tle->expr, context); - } - } - - /* ORDER BY */ - if (wc->orderClause) - { - if (wc->partitionClause) - appendStringInfoChar(buf, ' '); - appendStringInfoString(buf, "ORDER BY "); - first = true; - foreach(lc, wc->orderClause) - { - SortGroupClause *sgc = (SortGroupClause *) lfirst(lc); - TargetEntry *tle = get_sortgroupref_tle(sgc->tleSortGroupRef, - query->targetList); - TypeCacheEntry *typentry; - - if (!first) - appendStringInfoString(buf, ", "); - first = false; - deparseExpr((Expr *) tle->expr, context); - - /* Determine sort direction from the sort operator */ - typentry = lookup_type_cache(exprType((Node *) tle->expr), - TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); - if (sgc->sortop == typentry->gt_opr) - appendStringInfoString(buf, " DESC"); - else - appendStringInfoString(buf, " ASC"); - - if (sgc->nulls_first) - appendStringInfoString(buf, " NULLS FIRST"); - } - } - - /* - * Frame clause. Skip for ranking functions (row_number, rank, - * dense_rank, ntile, cume_dist, percent_rank) since ClickHouse does not - * accept frame specifications for these. For other functions, emit - * non-default frames only. - */ - if (node->winfnoid != F_ROW_NUMBER - && node->winfnoid != F_RANK_ - && node->winfnoid != F_DENSE_RANK_ - && node->winfnoid != F_NTILE - && node->winfnoid != F_CUME_DIST_ - && node->winfnoid != F_PERCENT_RANK_ - && wc->frameOptions != (FRAMEOPTION_DEFAULTS | FRAMEOPTION_NONDEFAULT) - && (wc->frameOptions & FRAMEOPTION_NONDEFAULT)) - { - appendStringInfoChar(buf, ' '); - - /* Frame type */ - if (wc->frameOptions & FRAMEOPTION_ROWS) - appendStringInfoString(buf, "ROWS "); - else if (wc->frameOptions & FRAMEOPTION_RANGE) - appendStringInfoString(buf, "RANGE "); - else if (wc->frameOptions & FRAMEOPTION_GROUPS) - appendStringInfoString(buf, "GROUPS "); - - /* Frame start and end */ - if (wc->frameOptions & FRAMEOPTION_BETWEEN) - { - appendStringInfoString(buf, "BETWEEN "); - - /* Start bound */ - if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) - appendStringInfoString(buf, "UNBOUNDED PRECEDING"); - else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) - appendStringInfoString(buf, "CURRENT ROW"); - else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) - { - deparseExpr((Expr *) wc->startOffset, context); - appendStringInfoString(buf, " PRECEDING"); - } - else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) - { - deparseExpr((Expr *) wc->startOffset, context); - appendStringInfoString(buf, " FOLLOWING"); - } - - appendStringInfoString(buf, " AND "); - - /* End bound */ - if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) - appendStringInfoString(buf, "UNBOUNDED FOLLOWING"); - else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) - appendStringInfoString(buf, "CURRENT ROW"); - else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) - { - deparseExpr((Expr *) wc->endOffset, context); - appendStringInfoString(buf, " PRECEDING"); - } - else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) - { - deparseExpr((Expr *) wc->endOffset, context); - appendStringInfoString(buf, " FOLLOWING"); - } - } - else - { - /* No BETWEEN — single start bound */ - if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) - appendStringInfoString(buf, "UNBOUNDED PRECEDING"); - else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) - appendStringInfoString(buf, "CURRENT ROW"); - else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) - { - deparseExpr((Expr *) wc->startOffset, context); - appendStringInfoString(buf, " PRECEDING"); - } - else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) - { - deparseExpr((Expr *) wc->startOffset, context); - appendStringInfoString(buf, " FOLLOWING"); - } - } - } - - appendStringInfoChar(buf, ')'); +deparseWindowFunc(WindowFunc* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + Query* query = context->root->parse; + WindowClause* wc; + ListCell* lc; + bool first; + char* funcname; + + /* Find the WindowClause referenced by this WindowFunc */ + wc = (WindowClause*)list_nth(query->windowClause, node->winref - 1); + + /* Emit function name */ + funcname = get_func_name(node->winfnoid); + CSTRING_TOLOWER(funcname); + appendStringInfoString(buf, funcname); + appendStringInfoChar(buf, '('); + + /* Emit arguments */ + first = true; + foreach (lc, node->args) { + if (!first) { + appendStringInfoString(buf, ", "); + } + first = false; + deparseExpr((Expr*)lfirst(lc), context); + } + + appendStringInfoString(buf, ") OVER ("); + + /* PARTITION BY */ + if (wc->partitionClause) { + appendStringInfoString(buf, "PARTITION BY "); + first = true; + foreach (lc, wc->partitionClause) { + SortGroupClause* sgc = (SortGroupClause*)lfirst(lc); + TargetEntry* tle = + get_sortgroupref_tle(sgc->tleSortGroupRef, query->targetList); + + if (!first) { + appendStringInfoString(buf, ", "); + } + first = false; + deparseExpr((Expr*)tle->expr, context); + } + } + + /* ORDER BY */ + if (wc->orderClause) { + if (wc->partitionClause) { + appendStringInfoChar(buf, ' '); + } + appendStringInfoString(buf, "ORDER BY "); + first = true; + foreach (lc, wc->orderClause) { + SortGroupClause* sgc = (SortGroupClause*)lfirst(lc); + TargetEntry* tle = + get_sortgroupref_tle(sgc->tleSortGroupRef, query->targetList); + TypeCacheEntry* typentry; + + if (!first) { + appendStringInfoString(buf, ", "); + } + first = false; + deparseExpr((Expr*)tle->expr, context); + + /* Determine sort direction from the sort operator */ + typentry = lookup_type_cache( + exprType((Node*)tle->expr), TYPECACHE_LT_OPR | TYPECACHE_GT_OPR + ); + if (sgc->sortop == typentry->gt_opr) { + appendStringInfoString(buf, " DESC"); + } else { + appendStringInfoString(buf, " ASC"); + } + + if (sgc->nulls_first) { + appendStringInfoString(buf, " NULLS FIRST"); + } + } + } + + /* + * Frame clause. Skip for ranking functions (row_number, rank, + * dense_rank, ntile, cume_dist, percent_rank) since ClickHouse does not + * accept frame specifications for these. For other functions, emit + * non-default frames only. + */ + if (node->winfnoid != F_ROW_NUMBER && node->winfnoid != F_RANK_ && + node->winfnoid != F_DENSE_RANK_ && node->winfnoid != F_NTILE && + node->winfnoid != F_CUME_DIST_ && node->winfnoid != F_PERCENT_RANK_ && + wc->frameOptions != (FRAMEOPTION_DEFAULTS | FRAMEOPTION_NONDEFAULT) && + (wc->frameOptions & FRAMEOPTION_NONDEFAULT)) { + appendStringInfoChar(buf, ' '); + + /* Frame type */ + if (wc->frameOptions & FRAMEOPTION_ROWS) { + appendStringInfoString(buf, "ROWS "); + } else if (wc->frameOptions & FRAMEOPTION_RANGE) { + appendStringInfoString(buf, "RANGE "); + } else if (wc->frameOptions & FRAMEOPTION_GROUPS) { + appendStringInfoString(buf, "GROUPS "); + } + + /* Frame start and end */ + if (wc->frameOptions & FRAMEOPTION_BETWEEN) { + appendStringInfoString(buf, "BETWEEN "); + + /* Start bound */ + if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) { + appendStringInfoString(buf, "UNBOUNDED PRECEDING"); + } else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) { + appendStringInfoString(buf, "CURRENT ROW"); + } else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) { + deparseExpr((Expr*)wc->startOffset, context); + appendStringInfoString(buf, " PRECEDING"); + } else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) { + deparseExpr((Expr*)wc->startOffset, context); + appendStringInfoString(buf, " FOLLOWING"); + } + + appendStringInfoString(buf, " AND "); + + /* End bound */ + if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) { + appendStringInfoString(buf, "UNBOUNDED FOLLOWING"); + } else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) { + appendStringInfoString(buf, "CURRENT ROW"); + } else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) { + deparseExpr((Expr*)wc->endOffset, context); + appendStringInfoString(buf, " PRECEDING"); + } else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) { + deparseExpr((Expr*)wc->endOffset, context); + appendStringInfoString(buf, " FOLLOWING"); + } + } else { + /* No BETWEEN — single start bound */ + if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) { + appendStringInfoString(buf, "UNBOUNDED PRECEDING"); + } else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) { + appendStringInfoString(buf, "CURRENT ROW"); + } else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) { + deparseExpr((Expr*)wc->startOffset, context); + appendStringInfoString(buf, " PRECEDING"); + } else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) { + deparseExpr((Expr*)wc->startOffset, context); + appendStringInfoString(buf, " FOLLOWING"); + } + } + } + + appendStringInfoChar(buf, ')'); } static void -deparseCaseExpr(CaseExpr * node, deparse_expr_cxt * context) -{ -#define DEPARSE_WRAPPED(node) \ -do { \ - bool isnull = (IsA(node, Const) && ((Const *) (node))->constisnull); \ - if (conv && !isnull) \ - appendStringInfoString(buf, conv); \ - deparseExpr(node, context); \ - if (conv && !isnull) \ - appendStringInfoChar(buf, ')'); \ -} while (0) - - StringInfo buf = context->buf; - ListCell *lc; - char *conv = NULL; - - if (node->casetype == INT2OID || - node->casetype == INT4OID || - node->casetype == INT8OID) - { - conv = ch_format_type_extended(node->casetype, 0, 0); - conv = psprintf("to%s(", conv); - } - - appendStringInfoString(buf, "CASE"); - if (node->arg) - { - appendStringInfoChar(buf, ' '); - deparseExpr(node->arg, context); - } - - foreach(lc, node->args) - { - CaseWhen *arg = lfirst(lc); - - Assert(IsA(arg, CaseWhen)); - appendStringInfoString(buf, " WHEN "); - deparseExpr(arg->expr, context); - - /* - * in simple cases like WHEN val THEN we should extend the condition - * for WHEN val = 1 since there is no bool type in ClickHouse - */ - if (IsA(arg->expr, Var)) - appendStringInfoString(buf, " = 1"); - - appendStringInfoString(buf, " THEN "); - DEPARSE_WRAPPED(arg->result); - } - - if (node->defresult) - { - appendStringInfoString(buf, " ELSE "); - DEPARSE_WRAPPED(node->defresult); - } - - if (conv) - pfree(conv); - appendStringInfoString(buf, " END"); +deparseCaseExpr(CaseExpr* node, deparse_expr_cxt* context) { +#define DEPARSE_WRAPPED(node) \ + do { \ + bool isnull = (IsA(node, Const) && ((Const*)(node))->constisnull); \ + if (conv && !isnull) \ + appendStringInfoString(buf, conv); \ + deparseExpr(node, context); \ + if (conv && !isnull) \ + appendStringInfoChar(buf, ')'); \ + } while (0) + + StringInfo buf = context->buf; + ListCell* lc; + char* conv = NULL; + + if (node->casetype == INT2OID || node->casetype == INT4OID || + node->casetype == INT8OID) { + conv = ch_format_type_extended(node->casetype, 0, 0); + conv = psprintf("to%s(", conv); + } + + appendStringInfoString(buf, "CASE"); + if (node->arg) { + appendStringInfoChar(buf, ' '); + deparseExpr(node->arg, context); + } + + foreach (lc, node->args) { + CaseWhen* arg = lfirst(lc); + + Assert(IsA(arg, CaseWhen)); + appendStringInfoString(buf, " WHEN "); + deparseExpr(arg->expr, context); + + /* + * in simple cases like WHEN val THEN we should extend the condition + * for WHEN val = 1 since there is no bool type in ClickHouse + */ + if (IsA(arg->expr, Var)) { + appendStringInfoString(buf, " = 1"); + } + + appendStringInfoString(buf, " THEN "); + DEPARSE_WRAPPED(arg->result); + } + + if (node->defresult) { + appendStringInfoString(buf, " ELSE "); + DEPARSE_WRAPPED(node->defresult); + } + + if (conv) { + pfree(conv); + } + appendStringInfoString(buf, " END"); } static void -deparseCaseWhen(CaseWhen * node, deparse_expr_cxt * context) -{ - /* - * XXX Needs implementation. - * - * StringInfo buf = context->buf; - * - * ListCell * lc; - */ +deparseCaseWhen(CaseWhen* node, deparse_expr_cxt* context) { + /* + * XXX Needs implementation. + * + * StringInfo buf = context->buf; + * + * ListCell * lc; + */ } static void -deparseRowExpr(RowExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - ListCell *lc; - - bool first = true; - - appendStringInfoChar(buf, '('); - foreach(lc, node->args) - { - if (!first) - appendStringInfoChar(buf, ','); - - first = false; - if (IsA(lfirst(lc), Const)) - deparseConst((Const *) lfirst(lc), context, 1); - else - deparseExpr(lfirst(lc), context); - } - appendStringInfoChar(buf, ')'); +deparseRowExpr(RowExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + ListCell* lc; + + bool first = true; + + appendStringInfoChar(buf, '('); + foreach (lc, node->args) { + if (!first) { + appendStringInfoChar(buf, ','); + } + + first = false; + if (IsA(lfirst(lc), Const)) { + deparseConst((Const*)lfirst(lc), context, 1); + } else { + deparseExpr(lfirst(lc), context); + } + } + appendStringInfoChar(buf, ')'); } static void -deparseCoerceViaIO(CoerceViaIO * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; +deparseCoerceViaIO(CoerceViaIO* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; - if (node->resulttype == INTERVALOID) - deparseExpr(node->arg, context); - else - { - appendStringInfoString(buf, "CAST("); - deparseExpr(node->arg, context); - appendStringInfoString(buf, " AS "); + if (node->resulttype == INTERVALOID) { + deparseExpr(node->arg, context); + } else { + appendStringInfoString(buf, "CAST("); + deparseExpr(node->arg, context); + appendStringInfoString(buf, " AS "); - if (node->resultcollid == 1) - appendStringInfoString(buf, "Nullable("); + if (node->resultcollid == 1) { + appendStringInfoString(buf, "Nullable("); + } - appendStringInfoString(buf, deparse_type_name(node->resulttype, 0)); + appendStringInfoString(buf, deparse_type_name(node->resulttype, 0)); - if (node->resultcollid == 1) - appendStringInfoChar(buf, ')'); + if (node->resultcollid == 1) { + appendStringInfoChar(buf, ')'); + } - appendStringInfoChar(buf, ')'); - } + appendStringInfoChar(buf, ')'); + } } static void -deparseCoalesceExpr(CoalesceExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - ListCell *lc; - bool first; - - appendStringInfoString(buf, "COALESCE("); - - first = true; - foreach(lc, node->args) - { - Expr *arg = lfirst(lc); - - if (!first) - appendStringInfoString(buf, ", "); - - /* first arg should be nullable */ - if (IsA(arg, CoerceViaIO)) - { - CoerceViaIO *vio = (CoerceViaIO *) arg; - - if (arg != llast(node->args)) - vio->resultcollid = 1; - else - vio->resultcollid = InvalidOid; - } - - first = false; - deparseExpr(arg, context); - } - appendStringInfoChar(buf, ')'); +deparseCoalesceExpr(CoalesceExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + ListCell* lc; + bool first; + + appendStringInfoString(buf, "COALESCE("); + + first = true; + foreach (lc, node->args) { + Expr* arg = lfirst(lc); + + if (!first) { + appendStringInfoString(buf, ", "); + } + + /* first arg should be nullable */ + if (IsA(arg, CoerceViaIO)) { + CoerceViaIO* vio = (CoerceViaIO*)arg; + + if (arg != llast(node->args)) { + vio->resultcollid = 1; + } else { + vio->resultcollid = InvalidOid; + } + } + + first = false; + deparseExpr(arg, context); + } + appendStringInfoChar(buf, ')'); } static void -deparseMinMaxExpr(MinMaxExpr * node, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - ListCell *lc; - bool first; - - if (node->op == IS_GREATEST) - appendStringInfoString(buf, "greatest"); - else - appendStringInfoString(buf, "least"); - - appendStringInfoChar(buf, '('); - first = true; - foreach(lc, node->args) - { - if (!first) - appendStringInfoString(buf, ", "); - - first = false; - - deparseExpr(lfirst(lc), context); - } - appendStringInfoChar(buf, ')'); +deparseMinMaxExpr(MinMaxExpr* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + ListCell* lc; + bool first; + + if (node->op == IS_GREATEST) { + appendStringInfoString(buf, "greatest"); + } else { + appendStringInfoString(buf, "least"); + } + + appendStringInfoChar(buf, '('); + first = true; + foreach (lc, node->args) { + if (!first) { + appendStringInfoString(buf, ", "); + } + + first = false; + + deparseExpr(lfirst(lc), context); + } + appendStringInfoChar(buf, ')'); } /* * Deparse GROUP BY clause. */ static void -appendGroupByClause(List * tlist, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - Query *query = context->root->parse; - ListCell *lc; - bool first = true; +appendGroupByClause(List* tlist, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + Query* query = context->root->parse; + ListCell* lc; + bool first = true; - /* Nothing to be done, if there's no GROUP BY clause in the query. */ - if (!query->groupClause) - return; + /* Nothing to be done, if there's no GROUP BY clause in the query. */ + if (!query->groupClause) { + return; + } - appendStringInfoString(buf, " GROUP BY "); + appendStringInfoString(buf, " GROUP BY "); - /* - * Queries with grouping sets are not pushed down, so we don't expect - * grouping sets here. - */ - Assert(!query->groupingSets); + /* + * Queries with grouping sets are not pushed down, so we don't expect + * grouping sets here. + */ + Assert(!query->groupingSets); - foreach(lc, query->groupClause) - { - SortGroupClause *grp = (SortGroupClause *) lfirst(lc); + foreach (lc, query->groupClause) { + SortGroupClause* grp = (SortGroupClause*)lfirst(lc); - if (!first) - appendStringInfoString(buf, ", "); + if (!first) { + appendStringInfoString(buf, ", "); + } - first = false; + first = false; - deparseSortGroupClause(grp->tleSortGroupRef, tlist, true, context); - } + deparseSortGroupClause(grp->tleSortGroupRef, tlist, true, context); + } } /* @@ -4721,92 +4795,85 @@ appendGroupByClause(List * tlist, deparse_expr_cxt * context) * base relation are obtained and deparsed. */ static void -appendOrderByClause(List * pathkeys, bool has_final_sort, - deparse_expr_cxt * context) -{ - ListCell *lcell; - char *delim = " "; - RelOptInfo *baserel = context->scanrel; - StringInfo buf = context->buf; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) context->foreignrel->fdw_private; - - appendStringInfoString(buf, " ORDER BY"); - foreach(lcell, pathkeys) - { - PathKey *pathkey = lfirst(lcell); - Expr *em_expr; - - if (has_final_sort) - { - /* - * By construction, context->foreignrel is the input relation to - * the final sort. Upper rels may have an empty reltarget; fall - * back to the planner's upper target for that stage. - */ - PathTarget *target = context->foreignrel->reltarget; - - if (target->exprs == NIL) - target = context->root->upper_targets[fpinfo->stage]; - em_expr = chfdw_find_em_expr_for_input_target(context->root, - pathkey->pk_eclass, - target); - } - else if (IS_JOIN_REL(context->foreignrel) && - fpinfo->jointype == JOIN_SEMI) - { - /* - * For SEMI JOINs, prefer expressions from the outer relation - * since inner relation columns are not visible in the output. - */ - em_expr = chfdw_find_em_expr_for_rel(pathkey->pk_eclass, - fpinfo->outerrel); - if (em_expr == NULL) - em_expr = chfdw_find_em_expr_for_rel(pathkey->pk_eclass, baserel); - } - else - em_expr = chfdw_find_em_expr_for_rel(pathkey->pk_eclass, baserel); - - Assert(em_expr != NULL); - - appendStringInfoString(buf, delim); - deparseExpr(em_expr, context); +appendOrderByClause(List* pathkeys, bool has_final_sort, deparse_expr_cxt* context) { + ListCell* lcell; + char* delim = " "; + RelOptInfo* baserel = context->scanrel; + StringInfo buf = context->buf; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)context->foreignrel->fdw_private; + + appendStringInfoString(buf, " ORDER BY"); + foreach (lcell, pathkeys) { + PathKey* pathkey = lfirst(lcell); + Expr* em_expr; + + if (has_final_sort) { + /* + * By construction, context->foreignrel is the input relation to + * the final sort. Upper rels may have an empty reltarget; fall + * back to the planner's upper target for that stage. + */ + PathTarget* target = context->foreignrel->reltarget; + + if (target->exprs == NIL) { + target = context->root->upper_targets[fpinfo->stage]; + } + em_expr = chfdw_find_em_expr_for_input_target( + context->root, pathkey->pk_eclass, target + ); + } else if (IS_JOIN_REL(context->foreignrel) && fpinfo->jointype == JOIN_SEMI) { + /* + * For SEMI JOINs, prefer expressions from the outer relation + * since inner relation columns are not visible in the output. + */ + em_expr = chfdw_find_em_expr_for_rel(pathkey->pk_eclass, fpinfo->outerrel); + if (em_expr == NULL) { + em_expr = chfdw_find_em_expr_for_rel(pathkey->pk_eclass, baserel); + } + } else { + em_expr = chfdw_find_em_expr_for_rel(pathkey->pk_eclass, baserel); + } + + Assert(em_expr != NULL); + + appendStringInfoString(buf, delim); + deparseExpr(em_expr, context); #if PG_VERSION_NUM >= 180000 - if (pathkey->pk_cmptype == BTLessStrategyNumber) + if (pathkey->pk_cmptype == BTLessStrategyNumber) #else - if (pathkey->pk_strategy == BTLessStrategyNumber) + if (pathkey->pk_strategy == BTLessStrategyNumber) #endif - appendStringInfoString(buf, " ASC"); - else - appendStringInfoString(buf, " DESC"); - - if (pathkey->pk_nulls_first) - appendStringInfoString(buf, " NULLS FIRST"); - else - appendStringInfoString(buf, " NULLS LAST"); - - delim = ", "; - } + appendStringInfoString(buf, " ASC"); + else { + appendStringInfoString(buf, " DESC"); + } + + if (pathkey->pk_nulls_first) { + appendStringInfoString(buf, " NULLS FIRST"); + } else { + appendStringInfoString(buf, " NULLS LAST"); + } + + delim = ", "; + } } /* * Deparse LIMIT/OFFSET clause. */ static void -appendLimitClause(deparse_expr_cxt * context) -{ - PlannerInfo *root = context->root; - StringInfo buf = context->buf; - - if (root->parse->limitCount) - { - appendStringInfoString(buf, " LIMIT "); - deparseExpr((Expr *) root->parse->limitCount, context); - } - if (root->parse->limitOffset) - { - appendStringInfoString(buf, " OFFSET "); - deparseExpr((Expr *) root->parse->limitOffset, context); - } +appendLimitClause(deparse_expr_cxt* context) { + PlannerInfo* root = context->root; + StringInfo buf = context->buf; + + if (root->parse->limitCount) { + appendStringInfoString(buf, " LIMIT "); + deparseExpr((Expr*)root->parse->limitCount, context); + } + if (root->parse->limitOffset) { + appendStringInfoString(buf, " OFFSET "); + deparseExpr((Expr*)root->parse->limitOffset, context); + } } /* @@ -4814,81 +4881,75 @@ appendLimitClause(deparse_expr_cxt * context) * Deparses function name from given function oid. * Returns was custom or not. */ -static CustomObjectDef * -appendFunctionName(Oid funcid, deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - HeapTuple proctup; - Form_pg_proc procform; - const char *proname; - CustomObjectDef *cdef; - CHFdwRelationInfo *fpinfo = context->scanrel->fdw_private; - - cdef = chfdw_check_for_custom_function(funcid); - if (cdef && cdef->custom_name[0] != '\0') - { - if (cdef->custom_name[0] != '\1') - appendStringInfoString(buf, cdef->custom_name); - return cdef; - } - - proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); - if (!HeapTupleIsValid(proctup)) - elog(ERROR, "cache lookup failed for function %u", funcid); - - procform = (Form_pg_proc) GETSTRUCT(proctup); - proname = NameStr(procform->proname); - - /* we have some additional conditions on aggregation functions */ - if (chfdw_is_builtin(funcid) && procform->prokind == PROKIND_AGGREGATE - && fpinfo->ch_table_engine == CH_COLLAPSING_MERGE_TREE) - { - if (strcmp(proname, "sum") == 0) - { - cdef = palloc(sizeof(CustomObjectDef)); - cdef->cf_oid = funcid; - cdef->cf_type = CF_SIGN_SUM; - } - else if (strcmp(proname, "avg") == 0) - { - cdef = palloc(sizeof(CustomObjectDef)); - cdef->cf_oid = funcid; - cdef->cf_type = CF_SIGN_AVG;; - proname = "sum"; - } - else if (strcmp(proname, "count") == 0) - { - cdef = palloc(sizeof(CustomObjectDef)); - cdef->cf_oid = funcid; - cdef->cf_type = CF_SIGN_COUNT; - proname = "sum"; - } - } - - /* Map PG aggregate names to ClickHouse equivalents */ - switch (funcid) - { - case F_BOOL_AND: - case F_EVERY: - proname = "groupBitAnd"; - break; - case F_BOOL_OR: - proname = "groupBitOr"; - break; - case F_STRING_AGG_TEXT_TEXT: - proname = "groupConcat"; - cdef = palloc0(sizeof(CustomObjectDef)); - cdef->cf_oid = funcid; - cdef->cf_type = CF_STRING_AGG; - break; - } - - /* Always print the function name */ - appendStringInfoString(buf, proname); - - ReleaseSysCache(proctup); - - return cdef; +static CustomObjectDef* +appendFunctionName(Oid funcid, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + HeapTuple proctup; + Form_pg_proc procform; + const char* proname; + CustomObjectDef* cdef; + CHFdwRelationInfo* fpinfo = context->scanrel->fdw_private; + + cdef = chfdw_check_for_custom_function(funcid); + if (cdef && cdef->custom_name[0] != '\0') { + if (cdef->custom_name[0] != '\1') { + appendStringInfoString(buf, cdef->custom_name); + } + return cdef; + } + + proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(proctup)) { + elog(ERROR, "cache lookup failed for function %u", funcid); + } + + procform = (Form_pg_proc)GETSTRUCT(proctup); + proname = NameStr(procform->proname); + + /* we have some additional conditions on aggregation functions */ + if (chfdw_is_builtin(funcid) && procform->prokind == PROKIND_AGGREGATE && + fpinfo->ch_table_engine == CH_COLLAPSING_MERGE_TREE) { + if (strcmp(proname, "sum") == 0) { + cdef = palloc(sizeof(CustomObjectDef)); + cdef->cf_oid = funcid; + cdef->cf_type = CF_SIGN_SUM; + } else if (strcmp(proname, "avg") == 0) { + cdef = palloc(sizeof(CustomObjectDef)); + cdef->cf_oid = funcid; + cdef->cf_type = CF_SIGN_AVG; + ; + proname = "sum"; + } else if (strcmp(proname, "count") == 0) { + cdef = palloc(sizeof(CustomObjectDef)); + cdef->cf_oid = funcid; + cdef->cf_type = CF_SIGN_COUNT; + proname = "sum"; + } + } + + /* Map PG aggregate names to ClickHouse equivalents */ + switch (funcid) { + case F_BOOL_AND: + case F_EVERY: + proname = "groupBitAnd"; + break; + case F_BOOL_OR: + proname = "groupBitOr"; + break; + case F_STRING_AGG_TEXT_TEXT: + proname = "groupConcat"; + cdef = palloc0(sizeof(CustomObjectDef)); + cdef->cf_oid = funcid; + cdef->cf_type = CF_STRING_AGG; + break; + } + + /* Always print the function name */ + appendStringInfoString(buf, proname); + + ReleaseSysCache(proctup); + + return cdef; } /* @@ -4897,103 +4958,96 @@ appendFunctionName(Oid funcid, deparse_expr_cxt * context) * Like get_rule_sortgroupclause(), returns the expression tree, so caller * need not find it again. */ -static Node * -deparseSortGroupClause(Index ref, List * tlist, bool force_colno, - deparse_expr_cxt * context) -{ - StringInfo buf = context->buf; - TargetEntry *tle; - Expr *expr; - - tle = get_sortgroupref_tle(ref, tlist); - expr = tle->expr; - - if (expr && IsA(expr, Const)) - { - /* - * Force a typecast here so that we don't emit something like "GROUP - * BY 2", which will be misconstrued as a column position rather than - * a constant. - */ - deparseConst((Const *) expr, context, 1); - } - else if (!expr || IsA(expr, Var) || context->no_sort_parens) - { - deparseExpr(expr, context); - } - else - { - /* Always parenthesize the expression. */ - appendStringInfoChar(buf, '('); - deparseExpr(expr, context); - appendStringInfoChar(buf, ')'); - } - - return (Node *) expr; +static Node* +deparseSortGroupClause( + Index ref, + List* tlist, + bool force_colno, + deparse_expr_cxt* context +) { + StringInfo buf = context->buf; + TargetEntry* tle; + Expr* expr; + + tle = get_sortgroupref_tle(ref, tlist); + expr = tle->expr; + + if (expr && IsA(expr, Const)) { + /* + * Force a typecast here so that we don't emit something like "GROUP + * BY 2", which will be misconstrued as a column position rather than + * a constant. + */ + deparseConst((Const*)expr, context, 1); + } else if (!expr || IsA(expr, Var) || context->no_sort_parens) { + deparseExpr(expr, context); + } else { + /* Always parenthesize the expression. */ + appendStringInfoChar(buf, '('); + deparseExpr(expr, context); + appendStringInfoChar(buf, ')'); + } + + return (Node*)expr; } - /* * Returns true if given Var is deparsed as a subquery output column, in * which case, *relno and *colno are set to the IDs for the relation and * column alias to the Var provided by the subquery. */ static bool -is_subquery_var(Var * node, RelOptInfo * foreignrel, int *relno, int *colno) -{ - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) foreignrel->fdw_private; - RelOptInfo *outerrel = fpinfo->outerrel; - RelOptInfo *innerrel = fpinfo->innerrel; - - /* Should only be called in these cases. */ - Assert(IS_SIMPLE_REL(foreignrel) || IS_JOIN_REL(foreignrel)); - - /* - * If the given relation isn't a join relation, it doesn't have any lower - * subqueries, so the Var isn't a subquery output column. - */ - if (!IS_JOIN_REL(foreignrel)) - return false; - - /* - * If the Var doesn't belong to any lower subqueries, it isn't a subquery - * output column. - */ - if (!bms_is_member(node->varno, fpinfo->lower_subquery_rels)) - return false; - - if (bms_is_member(node->varno, outerrel->relids)) - { - /* - * If outer relation is deparsed as a subquery, the Var is an output - * column of the subquery; get the IDs for the relation/column alias. - */ - if (fpinfo->make_outerrel_subquery) - { - get_relation_column_alias_ids(node, outerrel, relno, colno); - return true; - } - - /* Otherwise, recurse into the outer relation. */ - return is_subquery_var(node, outerrel, relno, colno); - } - else - { - Assert(bms_is_member(node->varno, innerrel->relids)); - - /* - * If inner relation is deparsed as a subquery, the Var is an output - * column of the subquery; get the IDs for the relation/column alias. - */ - if (fpinfo->make_innerrel_subquery) - { - get_relation_column_alias_ids(node, innerrel, relno, colno); - return true; - } - - /* Otherwise, recurse into the inner relation. */ - return is_subquery_var(node, innerrel, relno, colno); - } +is_subquery_var(Var* node, RelOptInfo* foreignrel, int* relno, int* colno) { + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)foreignrel->fdw_private; + RelOptInfo* outerrel = fpinfo->outerrel; + RelOptInfo* innerrel = fpinfo->innerrel; + + /* Should only be called in these cases. */ + Assert(IS_SIMPLE_REL(foreignrel) || IS_JOIN_REL(foreignrel)); + + /* + * If the given relation isn't a join relation, it doesn't have any lower + * subqueries, so the Var isn't a subquery output column. + */ + if (!IS_JOIN_REL(foreignrel)) { + return false; + } + + /* + * If the Var doesn't belong to any lower subqueries, it isn't a subquery + * output column. + */ + if (!bms_is_member(node->varno, fpinfo->lower_subquery_rels)) { + return false; + } + + if (bms_is_member(node->varno, outerrel->relids)) { + /* + * If outer relation is deparsed as a subquery, the Var is an output + * column of the subquery; get the IDs for the relation/column alias. + */ + if (fpinfo->make_outerrel_subquery) { + get_relation_column_alias_ids(node, outerrel, relno, colno); + return true; + } + + /* Otherwise, recurse into the outer relation. */ + return is_subquery_var(node, outerrel, relno, colno); + } else { + Assert(bms_is_member(node->varno, innerrel->relids)); + + /* + * If inner relation is deparsed as a subquery, the Var is an output + * column of the subquery; get the IDs for the relation/column alias. + */ + if (fpinfo->make_innerrel_subquery) { + get_relation_column_alias_ids(node, innerrel, relno, colno); + return true; + } + + /* Otherwise, recurse into the inner relation. */ + return is_subquery_var(node, innerrel, relno, colno); + } } /* @@ -5001,28 +5055,29 @@ is_subquery_var(Var * node, RelOptInfo * foreignrel, int *relno, int *colno) * given relation, which are returned into *relno and *colno. */ static void -get_relation_column_alias_ids(Var * node, RelOptInfo * foreignrel, - int *relno, int *colno) -{ - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) foreignrel->fdw_private; - int i; - ListCell *lc; - - /* Get the relation alias ID */ - *relno = fpinfo->relation_index; - - /* Get the column alias ID */ - i = 1; - foreach(lc, foreignrel->reltarget->exprs) - { - if (equal(lfirst(lc), (Node *) node)) - { - *colno = i; - return; - } - i++; - } - - /* Shouldn't get here */ - elog(ERROR, "unexpected expression in subquery output"); +get_relation_column_alias_ids( + Var* node, + RelOptInfo* foreignrel, + int* relno, + int* colno +) { + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)foreignrel->fdw_private; + int i; + ListCell* lc; + + /* Get the relation alias ID */ + *relno = fpinfo->relation_index; + + /* Get the column alias ID */ + i = 1; + foreach (lc, foreignrel->reltarget->exprs) { + if (equal(lfirst(lc), (Node*)node)) { + *colno = i; + return; + } + i++; + } + + /* Shouldn't get here */ + elog(ERROR, "unexpected expression in subquery output"); } diff --git a/src/fdw.c b/src/fdw.c index c872ec4..1e0063d 100644 --- a/src/fdw.c +++ b/src/fdw.c @@ -1,11 +1,12 @@ /* - A PostgreSQL extension for connecting to ClickHouse servers. + A PostgreSQL extension for connecting to ClickHouse servers. */ #include /* PostgreSQL includes. */ #include "postgres.h" + #include "catalog/pg_class_d.h" #include "commands/defrem.h" #include "foreign/fdwapi.h" @@ -39,29 +40,28 @@ #endif /* extension includes. */ -#include "utils/builtins.h" #include "fdw.h" +#include "utils/builtins.h" #include "version.h" /* Extension metadata for the server. */ #ifdef PG_MODULE_MAGIC_EXT -PG_MODULE_MAGIC_EXT(.name = "pg_clickhouse", - .version = PGCH_VERSION); +PG_MODULE_MAGIC_EXT(.name = "pg_clickhouse", .version = PGCH_VERSION); #else PG_MODULE_MAGIC; #endif /* Default CPU cost to start up a foreign query. */ -#define DEFAULT_FDW_STARTUP_COST 100.0 +#define DEFAULT_FDW_STARTUP_COST 100.0 /* Default CPU cost to process 1 row (above and beyond cpu_tuple_cost). */ -#define DEFAULT_FDW_TUPLE_COST 0.01 +#define DEFAULT_FDW_TUPLE_COST 0.01 /* If no remote estimates, assume a sort costs 20% extra */ #define DEFAULT_FDW_SORT_MULTIPLIER 1.2 /* Approximate batch size in bytes for HTTP streaming (50 MB). */ -#define DEFAULT_FETCH_SIZE (50 * 1000 * 1000) +#define DEFAULT_FETCH_SIZE (50 * 1000 * 1000) /* * Indexes of FDW-private information stored in fdw_private lists. @@ -70,20 +70,19 @@ PG_MODULE_MAGIC; * can be fetched with list_nth(). For example, to get the SELECT statement: * sql = strVal(list_nth(fdw_private, FdwScanPrivateSelectSql)); */ -enum FdwScanPrivateIndex -{ - /* SQL statement to execute remotely (as a String node) */ - FdwScanPrivateSelectSql, - /* Integer list of attribute numbers retrieved by the SELECT */ - FdwScanPrivateRetrievedAttrs, - /* Approximate batch size in bytes for HTTP streaming */ - FdwScanPrivateFetchSize, - - /* - * String describing join i.e. names of relations being joined and types - * of join, added when the scan is join - */ - FdwScanPrivateRelations +enum FdwScanPrivateIndex { + /* SQL statement to execute remotely (as a String node) */ + FdwScanPrivateSelectSql, + /* Integer list of attribute numbers retrieved by the SELECT */ + FdwScanPrivateRetrievedAttrs, + /* Approximate batch size in bytes for HTTP streaming */ + FdwScanPrivateFetchSize, + + /* + * String describing join i.e. names of relations being joined and types + * of join, added when the scan is join + */ + FdwScanPrivateRelations }; /* @@ -94,68 +93,64 @@ enum FdwScanPrivateIndex * 1) Integer list of target attribute numbers for INSERT * 2) String table name. */ -enum FdwModifyPrivateIndex -{ - /* SQL statement to execute remotely (as a String node) */ - FdwModifyPrivateInsertSQL, - /* Integer list of target attribute numbers for INSERT/UPDATE */ - FdwModifyPrivateTargetAttnums, - /* Deparsed name of the result table */ - FdwModifyPrivateTableName, +enum FdwModifyPrivateIndex { + /* SQL statement to execute remotely (as a String node) */ + FdwModifyPrivateInsertSQL, + /* Integer list of target attribute numbers for INSERT/UPDATE */ + FdwModifyPrivateTargetAttnums, + /* Deparsed name of the result table */ + FdwModifyPrivateTableName, }; - /* * Execution state of a foreign scan using pg_clickhouse. */ -typedef struct ChFdwScanState -{ - Relation rel; /* relcache entry for the foreign table. NULL - * for a foreign join scan. */ - TupleDesc tupdesc; /* tuple descriptor of scan */ - AttInMetadata *attinmeta; /* attribute datatype conversion metadata */ - - /* extracted fdw_private data */ - char *query; /* text of SELECT command */ - List *retrieved_attrs; /* list of retrieved attribute numbers */ - - /* for remote query execution */ - ch_connection conn; /* connection for the scan */ - int numParams; /* number of parameters passed to query */ - Oid *param_oids; /* output parameter type OIDs them */ - List *param_exprs; /* executable expressions for param values */ - const char **param_values; /* textual values of query parameters */ - ch_cursor *ch_cursor; /* result of query from clickhouse */ - - /* for storing result tuple */ - HeapTuple tuple; /* array of currently-retrieved tuples */ - - /* working memory contexts */ - MemoryContext batch_cxt; /* context holding current batch of tuples */ - MemoryContext temp_cxt; /* context for per-tuple temporary data */ - - int32 fetch_size; /* approximate batch size in bytes */ - bool is_streaming; /* true when using HTTP streaming */ -} ChFdwScanState; +typedef struct ChFdwScanState { + Relation rel; /* relcache entry for the foreign table. NULL + * for a foreign join scan. */ + TupleDesc tupdesc; /* tuple descriptor of scan */ + AttInMetadata* attinmeta; /* attribute datatype conversion metadata */ + + /* extracted fdw_private data */ + char* query; /* text of SELECT command */ + List* retrieved_attrs; /* list of retrieved attribute numbers */ + + /* for remote query execution */ + ch_connection conn; /* connection for the scan */ + int numParams; /* number of parameters passed to query */ + Oid* param_oids; /* output parameter type OIDs them */ + List* param_exprs; /* executable expressions for param values */ + const char** param_values; /* textual values of query parameters */ + ch_cursor* ch_cursor; /* result of query from clickhouse */ + + /* for storing result tuple */ + HeapTuple tuple; /* array of currently-retrieved tuples */ + + /* working memory contexts */ + MemoryContext batch_cxt; /* context holding current batch of tuples */ + MemoryContext temp_cxt; /* context for per-tuple temporary data */ + + int32 fetch_size; /* approximate batch size in bytes */ + bool is_streaming; /* true when using HTTP streaming */ +} ChFdwScanState; /* * Execution state of a foreign insert. */ -typedef struct CHFdwModifyState -{ - Relation rel; /* relcache entry for the foreign table */ - AttInMetadata *attinmeta; /* attribute datatype conversion metadata */ +typedef struct CHFdwModifyState { + Relation rel; /* relcache entry for the foreign table */ + AttInMetadata* attinmeta; /* attribute datatype conversion metadata */ - /* for remote query execution */ - ch_connection conn; /* connection for the scan */ + /* for remote query execution */ + ch_connection conn; /* connection for the scan */ - /* extracted fdw_private data */ - char *query; /* text of INSERT/UPDATE/DELETE command */ - void *state; /* internal state for a connection */ + /* extracted fdw_private data */ + char* query; /* text of INSERT/UPDATE/DELETE command */ + void* state; /* internal state for a connection */ - /* working memory context */ - MemoryContext temp_cxt; /* context for per-tuple temporary data */ -} CHFdwModifyState; + /* working memory context */ + MemoryContext temp_cxt; /* context for per-tuple temporary data */ +} CHFdwModifyState; /* * This enum describes what's kept in the fdw_private list for a ForeignPath. @@ -164,25 +159,22 @@ typedef struct CHFdwModifyState * 1) Boolean flag showing if the remote query has the final sort * 2) Boolean flag showing if the remote query has the LIMIT clause */ -enum FdwPathPrivateIndex -{ - /* has-final-sort flag (as an integer Value node) */ - FdwPathPrivateHasFinalSort, - /* has-limit flag (as an integer Value node) */ - FdwPathPrivateHasLimit +enum FdwPathPrivateIndex { + /* has-final-sort flag (as an integer Value node) */ + FdwPathPrivateHasFinalSort, + /* has-limit flag (as an integer Value node) */ + FdwPathPrivateHasLimit }; /* Struct for extra information passed to estimate_path_cost_size() */ -typedef struct -{ - PathTarget *target; - bool has_final_sort; - bool has_limit; - double limit_tuples; - int64 count_est; - int64 offset_est; -} ChFdwPathExtraData; - +typedef struct { + PathTarget* target; + bool has_final_sort; + bool has_limit; + double limit_tuples; + int64 count_est; + int64 offset_est; +} ChFdwPathExtraData; /* * SQL functions @@ -198,152 +190,219 @@ static double time_used = 0; /* * FDW callback routines */ -static void clickhouseGetForeignRelSize(PlannerInfo * root, - RelOptInfo * baserel, - Oid foreigntableid); -static ForeignScan * clickhouseGetForeignPlan(PlannerInfo * root, - RelOptInfo * foreignrel, - Oid foreigntableid, - ForeignPath * best_path, - List * tlist, - List * scan_clauses, - Plan * outer_plan); +static void +clickhouseGetForeignRelSize(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid); +static ForeignScan* +clickhouseGetForeignPlan( + PlannerInfo* root, + RelOptInfo* foreignrel, + Oid foreigntableid, + ForeignPath* best_path, + List* tlist, + List* scan_clauses, + Plan* outer_plan +); static int - clickhouseAcquireSampleRowsFunc(Relation relation, int elevel, - HeapTuple * rows, int targrows, - double *totalrows, - double *totaldeadrows); -static void clickhouseBeginForeignScan(ForeignScanState * node, int eflags); -static TupleTableSlot * clickhouseIterateForeignScan(ForeignScanState * node); -static void clickhouseEndForeignScan(ForeignScanState * node); -static List * clickhousePlanForeignModify(PlannerInfo * root, - ModifyTable * plan, - Index resultRelation, - int subplan_index); -static void clickhouseBeginForeignModify(ModifyTableState * mtstate, - ResultRelInfo * resultRelInfo, - List * fdw_private, - int subplan_index, - int eflags); -static TupleTableSlot * clickhouseExecForeignInsert(EState * estate, - ResultRelInfo * resultRelInfo, - TupleTableSlot * slot, - TupleTableSlot * planSlot); -static void clickhouseBeginForeignInsert(ModifyTableState * mtstate, - ResultRelInfo * resultRelInfo); -static void clickhouseEndForeignInsert(EState * estate, - ResultRelInfo * resultRelInfo); -static void clickhouseExplainForeignScan(ForeignScanState * node, - ExplainState * es); -static void clickhouseGetForeignUpperPaths(PlannerInfo * root, - UpperRelationKind stage, - RelOptInfo * input_rel, RelOptInfo * output_rel, - void *extra); -static bool clickhouseAnalyzeForeignTable(Relation relation, - AcquireSampleRowsFunc * func, - BlockNumber * totalpages); -static bool clickhouseRecheckForeignScan(ForeignScanState * node, - TupleTableSlot * slot); +clickhouseAcquireSampleRowsFunc( + Relation relation, + int elevel, + HeapTuple* rows, + int targrows, + double* totalrows, + double* totaldeadrows +); +static void +clickhouseBeginForeignScan(ForeignScanState* node, int eflags); +static TupleTableSlot* +clickhouseIterateForeignScan(ForeignScanState* node); +static void +clickhouseEndForeignScan(ForeignScanState* node); +static List* +clickhousePlanForeignModify( + PlannerInfo* root, + ModifyTable* plan, + Index resultRelation, + int subplan_index +); +static void +clickhouseBeginForeignModify( + ModifyTableState* mtstate, + ResultRelInfo* resultRelInfo, + List* fdw_private, + int subplan_index, + int eflags +); +static TupleTableSlot* +clickhouseExecForeignInsert( + EState* estate, + ResultRelInfo* resultRelInfo, + TupleTableSlot* slot, + TupleTableSlot* planSlot +); +static void +clickhouseBeginForeignInsert(ModifyTableState* mtstate, ResultRelInfo* resultRelInfo); +static void +clickhouseEndForeignInsert(EState* estate, ResultRelInfo* resultRelInfo); +static void +clickhouseExplainForeignScan(ForeignScanState* node, ExplainState* es); +static void +clickhouseGetForeignUpperPaths( + PlannerInfo* root, + UpperRelationKind stage, + RelOptInfo* input_rel, + RelOptInfo* output_rel, + void* extra +); +static bool +clickhouseAnalyzeForeignTable( + Relation relation, + AcquireSampleRowsFunc* func, + BlockNumber* totalpages +); +static bool +clickhouseRecheckForeignScan(ForeignScanState* node, TupleTableSlot* slot); /* * Helper functions */ -static void estimate_path_cost_size(double *p_rows, int *p_width, - Cost * p_startup_cost, Cost * p_total_cost, - double coef); -static CHFdwModifyState * create_foreign_modify(EState * estate, - RangeTblEntry * rte, - ResultRelInfo * resultRelInfo, - CmdType operation, - Plan * subplan, - char *query, - List * target_attrs, - char *table_name); -static void finish_foreign_modify(CHFdwModifyState * fmstate); -static void prepare_query_params(PlanState * node, - List * fdw_exprs, - int numParams, - Oid * *param_oids, - List * *param_exprs, - const char ***param_values); -static void process_query_params(ExprContext * econtext, - Oid * param_oids, - List * param_exprs, - const char **param_values); -static bool foreign_join_ok(PlannerInfo * root, RelOptInfo * joinrel, - JoinType jointype, RelOptInfo * outerrel, RelOptInfo * innerrel, - JoinPathExtraData * extra); -static bool foreign_grouping_ok(PlannerInfo * root, RelOptInfo * grouped_rel, - Node * havingQual); -static List * get_useful_pathkeys_for_relation(PlannerInfo * root, - RelOptInfo * rel); -static void add_paths_with_pathkeys_for_rel(PlannerInfo * root, RelOptInfo * rel, - Path * epq_path); -static void add_foreign_grouping_paths(PlannerInfo * root, - RelOptInfo * input_rel, - RelOptInfo * grouped_rel, - GroupPathExtraData * extra); -static void add_foreign_window_paths(PlannerInfo * root, RelOptInfo * input_rel, - RelOptInfo * window_rel); -static bool foreign_window_ok(PlannerInfo * root, RelOptInfo * window_rel); -static void add_foreign_ordered_paths(PlannerInfo * root, RelOptInfo * input_rel, - RelOptInfo * ordered_rel); -static void add_foreign_final_paths(PlannerInfo * root, RelOptInfo * input_rel, - RelOptInfo * final_rel, - void *fextra); -static void merge_fdw_options(CHFdwRelationInfo * fpinfo, - const CHFdwRelationInfo * fpinfo_o, - const CHFdwRelationInfo * fpinfo_i); -static int get_fetch_size_option(DefElem * def); -static DefElem * ch_get_table_or_server_option(CHFdwRelationInfo * fpinfo, char *name); +static void +estimate_path_cost_size( + double* p_rows, + int* p_width, + Cost* p_startup_cost, + Cost* p_total_cost, + double coef +); +static CHFdwModifyState* +create_foreign_modify( + EState* estate, + RangeTblEntry* rte, + ResultRelInfo* resultRelInfo, + CmdType operation, + Plan* subplan, + char* query, + List* target_attrs, + char* table_name +); +static void +finish_foreign_modify(CHFdwModifyState* fmstate); +static void +prepare_query_params( + PlanState* node, + List* fdw_exprs, + int numParams, + Oid** param_oids, + List** param_exprs, + const char*** param_values +); +static void +process_query_params( + ExprContext* econtext, + Oid* param_oids, + List* param_exprs, + const char** param_values +); +static bool +foreign_join_ok( + PlannerInfo* root, + RelOptInfo* joinrel, + JoinType jointype, + RelOptInfo* outerrel, + RelOptInfo* innerrel, + JoinPathExtraData* extra +); +static bool +foreign_grouping_ok(PlannerInfo* root, RelOptInfo* grouped_rel, Node* havingQual); +static List* +get_useful_pathkeys_for_relation(PlannerInfo* root, RelOptInfo* rel); +static void +add_paths_with_pathkeys_for_rel(PlannerInfo* root, RelOptInfo* rel, Path* epq_path); +static void +add_foreign_grouping_paths( + PlannerInfo* root, + RelOptInfo* input_rel, + RelOptInfo* grouped_rel, + GroupPathExtraData* extra +); +static void +add_foreign_window_paths( + PlannerInfo* root, + RelOptInfo* input_rel, + RelOptInfo* window_rel +); +static bool +foreign_window_ok(PlannerInfo* root, RelOptInfo* window_rel); +static void +add_foreign_ordered_paths( + PlannerInfo* root, + RelOptInfo* input_rel, + RelOptInfo* ordered_rel +); +static void +add_foreign_final_paths( + PlannerInfo* root, + RelOptInfo* input_rel, + RelOptInfo* final_rel, + void* fextra +); +static void +merge_fdw_options( + CHFdwRelationInfo* fpinfo, + const CHFdwRelationInfo* fpinfo_o, + const CHFdwRelationInfo* fpinfo_i +); +static int +get_fetch_size_option(DefElem* def); +static DefElem* +ch_get_table_or_server_option(CHFdwRelationInfo* fpinfo, char* name); /* Make one query and close the connection */ Datum -clickhouse_raw_query(PG_FUNCTION_ARGS) -{ - char *connstring = text_to_cstring(PG_GETARG_TEXT_P(1)); - ch_query query = new_query(text_to_cstring(PG_GETARG_TEXT_P(0)), 0, NULL, NULL, NULL); +clickhouse_raw_query(PG_FUNCTION_ARGS) { + char* connstring = text_to_cstring(PG_GETARG_TEXT_P(1)); + ch_query query = + new_query(text_to_cstring(PG_GETARG_TEXT_P(0)), 0, NULL, NULL, NULL); - ch_connection_details *details = connstring_parse(connstring); - ch_connection conn = chfdw_http_connect(details); - ch_cursor *cursor = conn.methods->simple_query(conn.conn, &query); - text *res = chfdw_http_fetch_raw_data(cursor); + ch_connection_details* details = connstring_parse(connstring); + ch_connection conn = chfdw_http_connect(details); + ch_cursor* cursor = conn.methods->simple_query(conn.conn, &query); + text* res = chfdw_http_fetch_raw_data(cursor); - MemoryContextDelete(cursor->memcxt); - conn.methods->disconnect(conn.conn); + MemoryContextDelete(cursor->memcxt); + conn.methods->disconnect(conn.conn); - if (res) - PG_RETURN_TEXT_P(res); + if (res) { + PG_RETURN_TEXT_P(res); + } - PG_RETURN_NULL(); + PG_RETURN_NULL(); } /* calculate difference */ double -time_diff(struct timeval *prior, struct timeval *latter) -{ - double x = - (double) (latter->tv_usec - prior->tv_usec) / 1000.0L + - (double) (latter->tv_sec - prior->tv_sec) * 1000.0L; +time_diff(struct timeval* prior, struct timeval* latter) { + double x = (double)(latter->tv_usec - prior->tv_usec) / 1000.0L + + (double)(latter->tv_sec - prior->tv_sec) * 1000.0L; - return x; + return x; } static int -get_fetch_size_option(DefElem * def) -{ - int fetch_size = pg_strtoint32(defGetString(def)); - - if (fetch_size < 0) - { - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), - errmsg("invalid value for option \"%s\": %s", - def->defname, defGetString(def)), - errhint("fetch_size must be greater than or equal to 0"))); - } - - return fetch_size; +get_fetch_size_option(DefElem* def) { + int fetch_size = pg_strtoint32(defGetString(def)); + + if (fetch_size < 0) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg( + "invalid value for option \"%s\": %s", def->defname, defGetString(def) + ), + errhint("fetch_size must be greater than or equal to 0")) + ); + } + + return fetch_size; } /* @@ -351,29 +410,28 @@ get_fetch_size_option(DefElem * def) * Returns the first option that matches in `fpinfo->table->options` or, if * none, then the first found in `fpinfo->server->options`. Returns `NULL` * when none found. -*/ -static DefElem * -ch_get_table_or_server_option(CHFdwRelationInfo * fpinfo, char *name) -{ - ListCell *lc; + */ +static DefElem* +ch_get_table_or_server_option(CHFdwRelationInfo* fpinfo, char* name) { + ListCell* lc; - foreach(lc, fpinfo->table->options) - { - DefElem *def = (DefElem *) lfirst(lc); + foreach (lc, fpinfo->table->options) { + DefElem* def = (DefElem*)lfirst(lc); - if (strcmp(def->defname, name) == 0) - return def; - } + if (strcmp(def->defname, name) == 0) { + return def; + } + } - foreach(lc, fpinfo->server->options) - { - DefElem *def = (DefElem *) lfirst(lc); + foreach (lc, fpinfo->server->options) { + DefElem* def = (DefElem*)lfirst(lc); - if (strcmp(def->defname, name) == 0) - return def; - } + if (strcmp(def->defname, name) == 0) { + return def; + } + } - return NULL; + return NULL; } /* @@ -384,126 +442,128 @@ ch_get_table_or_server_option(CHFdwRelationInfo * fpinfo, char *name) * not any join clauses. */ static void -clickhouseGetForeignRelSize(PlannerInfo * root, - RelOptInfo * baserel, - Oid foreigntableid) -{ - CHFdwRelationInfo *fpinfo; - ListCell *lc; - RangeTblEntry *rte = planner_rt_fetch(baserel->relid, root); - char *relname, - *refname; - - /* - * We use CHFdwRelationInfo to pass various information to subsequent - * functions. - */ - fpinfo = (CHFdwRelationInfo *) palloc0(sizeof(CHFdwRelationInfo)); - baserel->fdw_private = (void *) fpinfo; - - /* Base foreign tables need to be pushed down always. */ - fpinfo->pushdown_safe = true; - - /* Look up foreign-table catalog info. */ - fpinfo->table = GetForeignTable(foreigntableid); - fpinfo->server = GetForeignServer(fpinfo->table->serverid); - - /* - * Extract user-settable option values. Note that per-table setting of - * use_remote_estimate overrides per-server setting. - */ - fpinfo->fdw_startup_cost = DEFAULT_FDW_STARTUP_COST; - fpinfo->fdw_tuple_cost = DEFAULT_FDW_TUPLE_COST; - fpinfo->shippable_extensions = NIL; - - /* - * Extract fetch_size: table option overrides server option, default - * DEFAULT_FETCH_SIZE. Value is approximate batch size in bytes; 0 means - * buffer entire response (disable HTTP streaming). - */ - DefElem *def = ch_get_table_or_server_option(fpinfo, "fetch_size"); - - fpinfo->fetch_size = def ? get_fetch_size_option(def) : DEFAULT_FETCH_SIZE; - - chfdw_apply_custom_table_options(fpinfo, foreigntableid); - - fpinfo->user = NULL; - - /* - * Identify which baserestrictinfo clauses can be sent to the remote - * server and which can't. - */ - chfdw_classify_conditions(root, baserel, baserel->baserestrictinfo, - &fpinfo->remote_conds, &fpinfo->local_conds); - - /* - * Identify which attributes will need to be retrieved from the remote - * server. These include all attrs needed for joins or final output, plus - * all attrs used in the local_conds. (Note: if we end up using a - * parameterized scan, it's possible that some of the join clauses will be - * sent to the remote and thus we wouldn't really need to retrieve the - * columns used in them. Doesn't seem worth detecting that case though.) - */ - fpinfo->attrs_used = NULL; - pull_varattnos((Node *) baserel->reltarget->exprs, baserel->relid, - &fpinfo->attrs_used); - foreach(lc, fpinfo->local_conds) - { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); - - pull_varattnos((Node *) rinfo->clause, baserel->relid, - &fpinfo->attrs_used); - } - - /* - * Compute the selectivity and cost of the local_conds, so we don't have - * to do it over again for each path. The best we can do for these - * conditions is to estimate selectivity on the basis of local statistics. - */ - fpinfo->local_conds_sel = clauselist_selectivity(root, - fpinfo->local_conds, - baserel->relid, - JOIN_INNER, - NULL); - - cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root); - - /* - * Set cached relation costs to some negative value, so that we can detect - * when they are set to some sensible costs during one (usually the first) - * of the calls to estimate_path_cost_size(). - */ - fpinfo->rel_startup_cost = -1; - fpinfo->rel_total_cost = -1; - - /* Make base scans more expensive than join pushdowns */ - fpinfo->rows = baserel->rows; - fpinfo->startup_cost = 10.0; - fpinfo->total_cost = 10.0 + baserel->rows * 0.01; - - /* - * Set the name of relation in fpinfo, while we are constructing it here. - * It will be used to build the string describing the join relation in - * EXPLAIN output. We can't know whether VERBOSE option is specified or - * not, so always schema-qualify the foreign table name. - */ - fpinfo->relation_name = makeStringInfo(); - relname = get_rel_name(foreigntableid); - refname = rte->eref->aliasname; - appendStringInfoString(fpinfo->relation_name, quote_identifier(relname)); - if (*refname && strcmp(refname, relname) != 0) - { - appendStringInfoChar(fpinfo->relation_name, ' '); - appendStringInfoString(fpinfo->relation_name, - quote_identifier(rte->eref->aliasname)); - } - - /* No outer and inner relations. */ - fpinfo->make_outerrel_subquery = false; - fpinfo->make_innerrel_subquery = false; - fpinfo->lower_subquery_rels = NULL; - /* Set the relation index. */ - fpinfo->relation_index = baserel->relid; +clickhouseGetForeignRelSize( + PlannerInfo* root, + RelOptInfo* baserel, + Oid foreigntableid +) { + CHFdwRelationInfo* fpinfo; + ListCell* lc; + RangeTblEntry* rte = planner_rt_fetch(baserel->relid, root); + char *relname, *refname; + + /* + * We use CHFdwRelationInfo to pass various information to subsequent + * functions. + */ + fpinfo = (CHFdwRelationInfo*)palloc0(sizeof(CHFdwRelationInfo)); + baserel->fdw_private = (void*)fpinfo; + + /* Base foreign tables need to be pushed down always. */ + fpinfo->pushdown_safe = true; + + /* Look up foreign-table catalog info. */ + fpinfo->table = GetForeignTable(foreigntableid); + fpinfo->server = GetForeignServer(fpinfo->table->serverid); + + /* + * Extract user-settable option values. Note that per-table setting of + * use_remote_estimate overrides per-server setting. + */ + fpinfo->fdw_startup_cost = DEFAULT_FDW_STARTUP_COST; + fpinfo->fdw_tuple_cost = DEFAULT_FDW_TUPLE_COST; + fpinfo->shippable_extensions = NIL; + + /* + * Extract fetch_size: table option overrides server option, default + * DEFAULT_FETCH_SIZE. Value is approximate batch size in bytes; 0 means + * buffer entire response (disable HTTP streaming). + */ + DefElem* def = ch_get_table_or_server_option(fpinfo, "fetch_size"); + + fpinfo->fetch_size = def ? get_fetch_size_option(def) : DEFAULT_FETCH_SIZE; + + chfdw_apply_custom_table_options(fpinfo, foreigntableid); + + fpinfo->user = NULL; + + /* + * Identify which baserestrictinfo clauses can be sent to the remote + * server and which can't. + */ + chfdw_classify_conditions( + root, + baserel, + baserel->baserestrictinfo, + &fpinfo->remote_conds, + &fpinfo->local_conds + ); + + /* + * Identify which attributes will need to be retrieved from the remote + * server. These include all attrs needed for joins or final output, plus + * all attrs used in the local_conds. (Note: if we end up using a + * parameterized scan, it's possible that some of the join clauses will be + * sent to the remote and thus we wouldn't really need to retrieve the + * columns used in them. Doesn't seem worth detecting that case though.) + */ + fpinfo->attrs_used = NULL; + pull_varattnos( + (Node*)baserel->reltarget->exprs, baserel->relid, &fpinfo->attrs_used + ); + foreach (lc, fpinfo->local_conds) { + RestrictInfo* rinfo = lfirst_node(RestrictInfo, lc); + + pull_varattnos((Node*)rinfo->clause, baserel->relid, &fpinfo->attrs_used); + } + + /* + * Compute the selectivity and cost of the local_conds, so we don't have + * to do it over again for each path. The best we can do for these + * conditions is to estimate selectivity on the basis of local statistics. + */ + fpinfo->local_conds_sel = clauselist_selectivity( + root, fpinfo->local_conds, baserel->relid, JOIN_INNER, NULL + ); + + cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root); + + /* + * Set cached relation costs to some negative value, so that we can detect + * when they are set to some sensible costs during one (usually the first) + * of the calls to estimate_path_cost_size(). + */ + fpinfo->rel_startup_cost = -1; + fpinfo->rel_total_cost = -1; + + /* Make base scans more expensive than join pushdowns */ + fpinfo->rows = baserel->rows; + fpinfo->startup_cost = 10.0; + fpinfo->total_cost = 10.0 + baserel->rows * 0.01; + + /* + * Set the name of relation in fpinfo, while we are constructing it here. + * It will be used to build the string describing the join relation in + * EXPLAIN output. We can't know whether VERBOSE option is specified or + * not, so always schema-qualify the foreign table name. + */ + fpinfo->relation_name = makeStringInfo(); + relname = get_rel_name(foreigntableid); + refname = rte->eref->aliasname; + appendStringInfoString(fpinfo->relation_name, quote_identifier(relname)); + if (*refname && strcmp(refname, relname) != 0) { + appendStringInfoChar(fpinfo->relation_name, ' '); + appendStringInfoString( + fpinfo->relation_name, quote_identifier(rte->eref->aliasname) + ); + } + + /* No outer and inner relations. */ + fpinfo->make_outerrel_subquery = false; + fpinfo->make_innerrel_subquery = false; + fpinfo->lower_subquery_rels = NULL; + /* Set the relation index. */ + fpinfo->relation_index = baserel->relid; } /* @@ -515,83 +575,83 @@ clickhouseGetForeignRelSize(PlannerInfo * root, * planning, or because it enables an efficient merge join. Here, we try * to figure out which pathkeys to consider. */ -static List * -get_useful_pathkeys_for_relation(PlannerInfo * root, RelOptInfo * rel) -{ - List *useful_pathkeys_list = NIL; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) rel->fdw_private; - ListCell *lc; - - /* - * Pushing the query_pathkeys to the remote server is always worth - * considering, because it might let us avoid a local sort. - */ - fpinfo->qp_is_pushdown_safe = false; - if (root->query_pathkeys) - { - bool query_pathkeys_ok = true; - - foreach(lc, root->query_pathkeys) - { - PathKey *pathkey = (PathKey *) lfirst(lc); - EquivalenceClass *pathkey_ec = pathkey->pk_eclass; - Expr *em_expr; - - /* - * The planner and executor don't have any clever strategy for - * taking data sorted by a prefix of the query's pathkeys and - * getting it to be sorted by all of those pathkeys. We'll just - * end up resorting the entire data set. So, unless we can push - * down all of the query pathkeys, forget it. - * - * chfdw_is_foreign_expr would detect volatile expressions as - * well, but checking ec_has_volatile here saves some cycles. - */ - if (pathkey_ec->ec_has_volatile || - !(em_expr = chfdw_find_em_expr_for_rel(pathkey_ec, rel)) || - !chfdw_is_foreign_expr(root, rel, em_expr)) - { - query_pathkeys_ok = false; - break; - } - } - - if (query_pathkeys_ok) - { - useful_pathkeys_list = list_make1(list_copy(root->query_pathkeys)); - fpinfo->qp_is_pushdown_safe = true; - } - } - - return useful_pathkeys_list; +static List* +get_useful_pathkeys_for_relation(PlannerInfo* root, RelOptInfo* rel) { + List* useful_pathkeys_list = NIL; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)rel->fdw_private; + ListCell* lc; + + /* + * Pushing the query_pathkeys to the remote server is always worth + * considering, because it might let us avoid a local sort. + */ + fpinfo->qp_is_pushdown_safe = false; + if (root->query_pathkeys) { + bool query_pathkeys_ok = true; + + foreach (lc, root->query_pathkeys) { + PathKey* pathkey = (PathKey*)lfirst(lc); + EquivalenceClass* pathkey_ec = pathkey->pk_eclass; + Expr* em_expr; + + /* + * The planner and executor don't have any clever strategy for + * taking data sorted by a prefix of the query's pathkeys and + * getting it to be sorted by all of those pathkeys. We'll just + * end up resorting the entire data set. So, unless we can push + * down all of the query pathkeys, forget it. + * + * chfdw_is_foreign_expr would detect volatile expressions as + * well, but checking ec_has_volatile here saves some cycles. + */ + if (pathkey_ec->ec_has_volatile || + !(em_expr = chfdw_find_em_expr_for_rel(pathkey_ec, rel)) || + !chfdw_is_foreign_expr(root, rel, em_expr)) { + query_pathkeys_ok = false; + break; + } + } + + if (query_pathkeys_ok) { + useful_pathkeys_list = list_make1(list_copy(root->query_pathkeys)); + fpinfo->qp_is_pushdown_safe = true; + } + } + + return useful_pathkeys_list; } /* -* clickhouseGetForeignPaths + * clickhouseGetForeignPaths * Create possible scan paths for a scan on the foreign table */ static void -clickhouseGetForeignPaths(PlannerInfo * root, - RelOptInfo * baserel, - Oid foreigntableid) -{ - ForeignPath *path; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) baserel->fdw_private; - - path = create_foreignscan_path(root, baserel, NULL, - fpinfo->rows, +clickhouseGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid) { + ForeignPath* path; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)baserel->fdw_private; + + path = create_foreignscan_path( + root, + baserel, + NULL, + fpinfo->rows, #if PG_VERSION_NUM >= 180000 - 0, + 0, #endif - fpinfo->startup_cost, fpinfo->total_cost, - NULL, NULL, NULL, NIL + fpinfo->startup_cost, + fpinfo->total_cost, + NULL, + NULL, + NULL, + NIL #if PG_VERSION_NUM >= 170000 - ,NIL + , + NIL #endif - ); + ); - add_path(baserel, (Path *) path); - add_paths_with_pathkeys_for_rel(root, baserel, NULL); + add_path(baserel, (Path*)path); + add_paths_with_pathkeys_for_rel(root, baserel, NULL); } /* @@ -619,286 +679,288 @@ clickhouseGetForeignPaths(PlannerInfo * root, * lives only as schema in fdw_scan_tlist or is rewritten to INDEX_VAR in * outer tlist. */ -typedef struct WindowFuncSubstState -{ - List *originals; /* WindowFunc nodes seen so far */ - List *placeholders; /* matching placeholder nodes (same index) */ - int counter; -} WindowFuncSubstState; - -static Node * replace_windowfuncs_mutator(Node * node, WindowFuncSubstState * state); - -static Node * -replace_windowfuncs_mutator_callback(Node * node, void *state) -{ - return replace_windowfuncs_mutator(node, state); +typedef struct WindowFuncSubstState { + List* originals; /* WindowFunc nodes seen so far */ + List* placeholders; /* matching placeholder nodes (same index) */ + int counter; +} WindowFuncSubstState; + +static Node* +replace_windowfuncs_mutator(Node* node, WindowFuncSubstState* state); + +static Node* +replace_windowfuncs_mutator_callback(Node* node, void* state) { + return replace_windowfuncs_mutator(node, state); } -static Node * -replace_windowfuncs_mutator(Node * node, WindowFuncSubstState * state) -{ - if (node == NULL) - return NULL; - if (IsA(node, WindowFunc)) - { - WindowFunc *wf = (WindowFunc *) node; - ListCell *lo, - *lp; - FuncExpr *ph; - - forboth(lo, state->originals, lp, state->placeholders) - { - if (equal(lfirst(lo), wf)) - return (Node *) copyObject(lfirst(lp)); - } - - ph = makeNode(FuncExpr); - ph->funcid = wf->winfnoid; - ph->funcresulttype = wf->wintype; - ph->funcretset = false; - ph->funcvariadic = false; - ph->funcformat = COERCE_EXPLICIT_CALL; - ph->funccollid = wf->wincollid; - ph->inputcollid = ++state->counter; - ph->args = (List *) copyObject(wf->args); - ph->location = -1; - - state->originals = lappend(state->originals, copyObject(wf)); - state->placeholders = lappend(state->placeholders, ph); - return (Node *) ph; - } - return expression_tree_mutator(node, replace_windowfuncs_mutator_callback, state); +static Node* +replace_windowfuncs_mutator(Node* node, WindowFuncSubstState* state) { + if (node == NULL) { + return NULL; + } + if (IsA(node, WindowFunc)) { + WindowFunc* wf = (WindowFunc*)node; + ListCell *lo, *lp; + FuncExpr* ph; + + forboth(lo, state->originals, lp, state->placeholders) { + if (equal(lfirst(lo), wf)) { + return (Node*)copyObject(lfirst(lp)); + } + } + + ph = makeNode(FuncExpr); + ph->funcid = wf->winfnoid; + ph->funcresulttype = wf->wintype; + ph->funcretset = false; + ph->funcvariadic = false; + ph->funcformat = COERCE_EXPLICIT_CALL; + ph->funccollid = wf->wincollid; + ph->inputcollid = ++state->counter; + ph->args = (List*)copyObject(wf->args); + ph->location = -1; + + state->originals = lappend(state->originals, copyObject(wf)); + state->placeholders = lappend(state->placeholders, ph); + return (Node*)ph; + } + return expression_tree_mutator(node, replace_windowfuncs_mutator_callback, state); } /* * clickhouseGetForeignPlan * Create ForeignScan plan node which implements selected best path */ -static ForeignScan * -clickhouseGetForeignPlan(PlannerInfo * root, - RelOptInfo * foreignrel, - Oid foreigntableid, - ForeignPath * best_path, - List * tlist, - List * scan_clauses, - Plan * outer_plan) -{ - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) foreignrel->fdw_private; - Index scan_relid; - List *fdw_private; - List *remote_exprs = NIL; - List *local_exprs = NIL; - List *params_list = NIL; - List *fdw_scan_tlist = NIL; - List *fdw_recheck_quals = NIL; - List *retrieved_attrs; - StringInfoData sql; - bool has_final_sort = false; - bool has_limit = false; - ListCell *lc; - struct timeval time1, - time2; - - gettimeofday(&time1, NULL); - - /* - * Get FDW private data created by clickhouseGetForeignUpperPaths(), if - * any. - */ - if (best_path->fdw_private) - { - has_final_sort = intVal(list_nth(best_path->fdw_private, - FdwPathPrivateHasFinalSort)); - has_limit = intVal(list_nth(best_path->fdw_private, - FdwPathPrivateHasLimit)); - } - - if (IS_SIMPLE_REL(foreignrel)) - { - /* - * For base relations, set scan_relid as the relid of the relation. - */ - scan_relid = foreignrel->relid; - - /* - * In a base-relation scan, we must apply the given scan_clauses. - * - * Separate the scan_clauses into those that can be executed remotely - * and those that can't. baserestrictinfo clauses that were previously - * determined to be safe or unsafe by chfdw_classify_conditions are - * found in fpinfo->remote_conds and fpinfo->local_conds. Anything - * else in the scan_clauses list will be a join clause, which we have - * to check for remote-safety. - * - * Note: the join clauses we see here should be the exact same ones - * previously examined by clickhouseGetForeignPaths. Possibly it'd be - * worth passing forward the classification work done then, rather - * than repeating it here. - * - * This code must match "extract_actual_clauses(scan_clauses, false)" - * except for the additional decision about remote versus local - * execution. - */ - foreach(lc, scan_clauses) - { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); - - /* Ignore any pseudoconstants, they're dealt with elsewhere */ - if (rinfo->pseudoconstant) - continue; - - if (list_member_ptr(fpinfo->remote_conds, rinfo)) - remote_exprs = lappend(remote_exprs, rinfo->clause); - else if (list_member_ptr(fpinfo->local_conds, rinfo)) - local_exprs = lappend(local_exprs, rinfo->clause); - else if (chfdw_is_foreign_expr(root, foreignrel, rinfo->clause)) - remote_exprs = lappend(remote_exprs, rinfo->clause); - else - local_exprs = lappend(local_exprs, rinfo->clause); - } - - /* - * For a base-relation scan, we have to support EPQ recheck, which - * should recheck all the remote quals. - */ - fdw_recheck_quals = remote_exprs; - } - else - { - /* - * Join relation or upper relation - set scan_relid to 0. - */ - scan_relid = 0; - - /* - * For a join rel, baserestrictinfo is NIL and we are not considering - * parameterization right now, so there should be no scan_clauses for - * a joinrel or an upper rel either. - */ - Assert(!scan_clauses); - - /* - * Instead we get the conditions to apply from the fdw_private - * structure. - */ - remote_exprs = extract_actual_clauses(fpinfo->remote_conds, false); - local_exprs = extract_actual_clauses(fpinfo->local_conds, false); - - /* - * We leave fdw_recheck_quals empty in this case, since we never need - * to apply EPQ recheck clauses. In the case of a joinrel, EPQ recheck - * is handled elsewhere --- see clickhouseGetForeignJoinPaths(). If - * we're planning an upperrel (ie, remote grouping or aggregation) - * then there's no EPQ to do because SELECT FOR UPDATE wouldn't be - * allowed, and indeed we *can't* put the remote clauses into - * fdw_recheck_quals because the unaggregated Vars won't be available - * locally. - */ - - /* Build the list of columns to be fetched from the foreign server. */ - fdw_scan_tlist = chfdw_build_tlist_to_deparse(foreignrel); - - /* - * Ensure that the outer plan produces a tuple whose descriptor - * matches our scan tuple slot. This is safe because all scans and - * joins support projection, so we never need to insert a Result node. - * Also, remove the local conditions from outer plan's quals, lest - * they be evaluated twice, once by the local plan and once by the - * scan. - */ - if (outer_plan) - { - ListCell *outer_lc; - - /* - * Right now, we only consider grouping and aggregation beyond - * joins. Queries involving aggregates or grouping do not require - * EPQ mechanism, hence should not have an outer plan here. - */ - Assert(!IS_UPPER_REL(foreignrel)); - - outer_plan->targetlist = fdw_scan_tlist; - - foreach(outer_lc, local_exprs) - { - Join *join_plan = (Join *) outer_plan; - Node *qual = lfirst(outer_lc); - - outer_plan->qual = list_delete(outer_plan->qual, qual); - - /* - * For an inner join the local conditions of foreign scan plan - * can be part of the joinquals as well. - */ - if (join_plan->jointype == JOIN_INNER) - join_plan->joinqual = list_delete(join_plan->joinqual, - qual); - } - } - } - - /* - * Build the query string to be sent for execution, and identify - * expressions to be sent as parameters. - */ - initStringInfo(&sql); - chfdw_deparse_select_stmt_for_rel(&sql, root, foreignrel, fdw_scan_tlist, - remote_exprs, best_path->path.pathkeys, - has_final_sort, has_limit, false, - &retrieved_attrs, ¶ms_list); - - /* Remember remote_exprs for possible use by clickhousePlanDirectModify */ - fpinfo->final_remote_exprs = remote_exprs; - - /* - * When window functions are pushed down, ForeignScan takes the place of - * what would otherwise be a WindowAgg plan node. Core EXPLAIN VERBOSE - * deparses WindowFunc via a WindowClause (query context) or WindowAgg in - * the plan; neither exists here, so deparse would error with "could not - * find window clause for winref N". Remote SQL has already been deparsed - * above, so swap WindowFunc for FuncExpr placeholders in both - * fdw_scan_tlist and the outer tlist. setrefs' equal() match rewrites the - * outer tlist to INDEX_VAR references against fdw_scan_tlist so execution - * reads real values from the scan tuple; VERBOSE deparse sees the - * placeholder instead of a WindowFunc. - */ - if (contain_window_function((Node *) fdw_scan_tlist)) - { - WindowFuncSubstState state = {NIL, NIL, 0}; - - fdw_scan_tlist = (List *) replace_windowfuncs_mutator((Node *) fdw_scan_tlist, &state); - tlist = (List *) replace_windowfuncs_mutator((Node *) tlist, &state); - } - - /* - * Build the fdw_private list that will be available to the executor. - * Items in the list must match order in enum FdwScanPrivateIndex. - */ - fdw_private = list_make3(makeString(sql.data), - retrieved_attrs, - makeInteger(fpinfo->fetch_size)); - if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel)) - fdw_private = lappend(fdw_private, - makeString(fpinfo->relation_name->data)); - - gettimeofday(&time2, NULL); - time_used += time_diff(&time1, &time2); - - /* - * Create the ForeignScan node for the given relation. - * - * Note that the remote parameter expressions are stored in the fdw_exprs - * field of the finished plan node; we can't keep them in private state - * because then they wouldn't be subject to later planner processing. - */ - return make_foreignscan(tlist, - local_exprs, - scan_relid, - params_list, - fdw_private, - fdw_scan_tlist, - fdw_recheck_quals, - outer_plan); +static ForeignScan* +clickhouseGetForeignPlan( + PlannerInfo* root, + RelOptInfo* foreignrel, + Oid foreigntableid, + ForeignPath* best_path, + List* tlist, + List* scan_clauses, + Plan* outer_plan +) { + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)foreignrel->fdw_private; + Index scan_relid; + List* fdw_private; + List* remote_exprs = NIL; + List* local_exprs = NIL; + List* params_list = NIL; + List* fdw_scan_tlist = NIL; + List* fdw_recheck_quals = NIL; + List* retrieved_attrs; + StringInfoData sql; + bool has_final_sort = false; + bool has_limit = false; + ListCell* lc; + struct timeval time1, time2; + + gettimeofday(&time1, NULL); + + /* + * Get FDW private data created by clickhouseGetForeignUpperPaths(), if + * any. + */ + if (best_path->fdw_private) { + has_final_sort = + intVal(list_nth(best_path->fdw_private, FdwPathPrivateHasFinalSort)); + has_limit = intVal(list_nth(best_path->fdw_private, FdwPathPrivateHasLimit)); + } + + if (IS_SIMPLE_REL(foreignrel)) { + /* + * For base relations, set scan_relid as the relid of the relation. + */ + scan_relid = foreignrel->relid; + + /* + * In a base-relation scan, we must apply the given scan_clauses. + * + * Separate the scan_clauses into those that can be executed remotely + * and those that can't. baserestrictinfo clauses that were previously + * determined to be safe or unsafe by chfdw_classify_conditions are + * found in fpinfo->remote_conds and fpinfo->local_conds. Anything + * else in the scan_clauses list will be a join clause, which we have + * to check for remote-safety. + * + * Note: the join clauses we see here should be the exact same ones + * previously examined by clickhouseGetForeignPaths. Possibly it'd be + * worth passing forward the classification work done then, rather + * than repeating it here. + * + * This code must match "extract_actual_clauses(scan_clauses, false)" + * except for the additional decision about remote versus local + * execution. + */ + foreach (lc, scan_clauses) { + RestrictInfo* rinfo = lfirst_node(RestrictInfo, lc); + + /* Ignore any pseudoconstants, they're dealt with elsewhere */ + if (rinfo->pseudoconstant) { + continue; + } + + if (list_member_ptr(fpinfo->remote_conds, rinfo)) { + remote_exprs = lappend(remote_exprs, rinfo->clause); + } else if (list_member_ptr(fpinfo->local_conds, rinfo)) { + local_exprs = lappend(local_exprs, rinfo->clause); + } else if (chfdw_is_foreign_expr(root, foreignrel, rinfo->clause)) { + remote_exprs = lappend(remote_exprs, rinfo->clause); + } else { + local_exprs = lappend(local_exprs, rinfo->clause); + } + } + + /* + * For a base-relation scan, we have to support EPQ recheck, which + * should recheck all the remote quals. + */ + fdw_recheck_quals = remote_exprs; + } else { + /* + * Join relation or upper relation - set scan_relid to 0. + */ + scan_relid = 0; + + /* + * For a join rel, baserestrictinfo is NIL and we are not considering + * parameterization right now, so there should be no scan_clauses for + * a joinrel or an upper rel either. + */ + Assert(!scan_clauses); + + /* + * Instead we get the conditions to apply from the fdw_private + * structure. + */ + remote_exprs = extract_actual_clauses(fpinfo->remote_conds, false); + local_exprs = extract_actual_clauses(fpinfo->local_conds, false); + + /* + * We leave fdw_recheck_quals empty in this case, since we never need + * to apply EPQ recheck clauses. In the case of a joinrel, EPQ recheck + * is handled elsewhere --- see clickhouseGetForeignJoinPaths(). If + * we're planning an upperrel (ie, remote grouping or aggregation) + * then there's no EPQ to do because SELECT FOR UPDATE wouldn't be + * allowed, and indeed we *can't* put the remote clauses into + * fdw_recheck_quals because the unaggregated Vars won't be available + * locally. + */ + + /* Build the list of columns to be fetched from the foreign server. */ + fdw_scan_tlist = chfdw_build_tlist_to_deparse(foreignrel); + + /* + * Ensure that the outer plan produces a tuple whose descriptor + * matches our scan tuple slot. This is safe because all scans and + * joins support projection, so we never need to insert a Result node. + * Also, remove the local conditions from outer plan's quals, lest + * they be evaluated twice, once by the local plan and once by the + * scan. + */ + if (outer_plan) { + ListCell* outer_lc; + + /* + * Right now, we only consider grouping and aggregation beyond + * joins. Queries involving aggregates or grouping do not require + * EPQ mechanism, hence should not have an outer plan here. + */ + Assert(!IS_UPPER_REL(foreignrel)); + + outer_plan->targetlist = fdw_scan_tlist; + + foreach (outer_lc, local_exprs) { + Join* join_plan = (Join*)outer_plan; + Node* qual = lfirst(outer_lc); + + outer_plan->qual = list_delete(outer_plan->qual, qual); + + /* + * For an inner join the local conditions of foreign scan plan + * can be part of the joinquals as well. + */ + if (join_plan->jointype == JOIN_INNER) { + join_plan->joinqual = list_delete(join_plan->joinqual, qual); + } + } + } + } + + /* + * Build the query string to be sent for execution, and identify + * expressions to be sent as parameters. + */ + initStringInfo(&sql); + chfdw_deparse_select_stmt_for_rel( + &sql, + root, + foreignrel, + fdw_scan_tlist, + remote_exprs, + best_path->path.pathkeys, + has_final_sort, + has_limit, + false, + &retrieved_attrs, + ¶ms_list + ); + + /* Remember remote_exprs for possible use by clickhousePlanDirectModify */ + fpinfo->final_remote_exprs = remote_exprs; + + /* + * When window functions are pushed down, ForeignScan takes the place of + * what would otherwise be a WindowAgg plan node. Core EXPLAIN VERBOSE + * deparses WindowFunc via a WindowClause (query context) or WindowAgg in + * the plan; neither exists here, so deparse would error with "could not + * find window clause for winref N". Remote SQL has already been deparsed + * above, so swap WindowFunc for FuncExpr placeholders in both + * fdw_scan_tlist and the outer tlist. setrefs' equal() match rewrites the + * outer tlist to INDEX_VAR references against fdw_scan_tlist so execution + * reads real values from the scan tuple; VERBOSE deparse sees the + * placeholder instead of a WindowFunc. + */ + if (contain_window_function((Node*)fdw_scan_tlist)) { + WindowFuncSubstState state = { NIL, NIL, 0 }; + + fdw_scan_tlist = + (List*)replace_windowfuncs_mutator((Node*)fdw_scan_tlist, &state); + tlist = (List*)replace_windowfuncs_mutator((Node*)tlist, &state); + } + + /* + * Build the fdw_private list that will be available to the executor. + * Items in the list must match order in enum FdwScanPrivateIndex. + */ + fdw_private = list_make3( + makeString(sql.data), retrieved_attrs, makeInteger(fpinfo->fetch_size) + ); + if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel)) { + fdw_private = lappend(fdw_private, makeString(fpinfo->relation_name->data)); + } + + gettimeofday(&time2, NULL); + time_used += time_diff(&time1, &time2); + + /* + * Create the ForeignScan node for the given relation. + * + * Note that the remote parameter expressions are stored in the fdw_exprs + * field of the finished plan node; we can't keep them in private state + * because then they wouldn't be subject to later planner processing. + */ + return make_foreignscan( + tlist, + local_exprs, + scan_relid, + params_list, + fdw_private, + fdw_scan_tlist, + fdw_recheck_quals, + outer_plan + ); } /* @@ -906,102 +968,102 @@ clickhouseGetForeignPlan(PlannerInfo * root, * Initiate an executor scan of a foreign PostgreSQL table. */ static void -clickhouseBeginForeignScan(ForeignScanState * node, int eflags) -{ - ForeignScan *fsplan = (ForeignScan *) node->ss.ps.plan; - EState *estate = node->ss.ps.state; - ChFdwScanState *fsstate; - RangeTblEntry *rte; - Oid userid; - ForeignTable *table; - UserMapping *user; - int rtindex; - int numParams; - - /* - * Do nothing in EXPLAIN (no ANALYZE) case. node->fdw_state stays NULL. - */ - if (eflags & EXEC_FLAG_EXPLAIN_ONLY) - return; - - /* - * We'll save private state in node->fdw_state. - */ - fsstate = (ChFdwScanState *) palloc0(sizeof(ChFdwScanState)); - node->fdw_state = (void *) fsstate; - - /* - * Identify which user to do the remote access as. This should match what - * ExecCheckRTEPerms() does. In case of a join or aggregate, use the - * lowest-numbered member RTE as a representative; we would get the same - * result from any. - */ - if (fsplan->scan.scanrelid > 0) - rtindex = fsplan->scan.scanrelid; - else - rtindex = bms_next_member(fsplan->fs_relids, -1); - rte = rt_fetch(rtindex, estate->es_range_table); +clickhouseBeginForeignScan(ForeignScanState* node, int eflags) { + ForeignScan* fsplan = (ForeignScan*)node->ss.ps.plan; + EState* estate = node->ss.ps.state; + ChFdwScanState* fsstate; + RangeTblEntry* rte; + Oid userid; + ForeignTable* table; + UserMapping* user; + int rtindex; + int numParams; + + /* + * Do nothing in EXPLAIN (no ANALYZE) case. node->fdw_state stays NULL. + */ + if (eflags & EXEC_FLAG_EXPLAIN_ONLY) { + return; + } + + /* + * We'll save private state in node->fdw_state. + */ + fsstate = (ChFdwScanState*)palloc0(sizeof(ChFdwScanState)); + node->fdw_state = (void*)fsstate; + + /* + * Identify which user to do the remote access as. This should match what + * ExecCheckRTEPerms() does. In case of a join or aggregate, use the + * lowest-numbered member RTE as a representative; we would get the same + * result from any. + */ + if (fsplan->scan.scanrelid > 0) { + rtindex = fsplan->scan.scanrelid; + } else { + rtindex = bms_next_member(fsplan->fs_relids, -1); + } + rte = rt_fetch(rtindex, estate->es_range_table); #if PG_VERSION_NUM >= 160000 - userid = OidIsValid(fsplan->checkAsUser) ? fsplan->checkAsUser : GetUserId(); + userid = OidIsValid(fsplan->checkAsUser) ? fsplan->checkAsUser : GetUserId(); #else - userid = rte->checkAsUser ? rte->checkAsUser : GetUserId(); + userid = rte->checkAsUser ? rte->checkAsUser : GetUserId(); #endif - /* Get info about foreign table. */ - table = GetForeignTable(rte->relid); - user = GetUserMapping(userid, table->serverid); - - /* - * Get connection to the foreign server. Connection manager will establish - * new connection if necessary. - */ - fsstate->conn = chfdw_get_connection(user); - - /* Get private info created by planner functions. */ - fsstate->query = strVal(list_nth(fsplan->fdw_private, - FdwScanPrivateSelectSql)); - fsstate->retrieved_attrs = (List *) list_nth(fsplan->fdw_private, - FdwScanPrivateRetrievedAttrs); - fsstate->fetch_size = intVal(list_nth(fsplan->fdw_private, - FdwScanPrivateFetchSize)); - - /* Create contexts for batches of tuples and per-tuple temp workspace. */ - fsstate->batch_cxt = AllocSetContextCreate(estate->es_query_cxt, - "pg_clickhouse tuple data", - ALLOCSET_DEFAULT_SIZES); - fsstate->temp_cxt = AllocSetContextCreate(estate->es_query_cxt, - "pg_clickhouse temporary data", - ALLOCSET_SMALL_SIZES); - - /* - * Get info we'll need for converting data fetched from the foreign server - * into local representation and error reporting during that process. - */ - if (fsplan->scan.scanrelid > 0) - { - fsstate->rel = node->ss.ss_currentRelation; - fsstate->tupdesc = RelationGetDescr(fsstate->rel); - } - else - { - fsstate->rel = NULL; - fsstate->tupdesc = node->ss.ss_ScanTupleSlot->tts_tupleDescriptor; - } - - fsstate->attinmeta = TupleDescGetAttInMetadata(fsstate->tupdesc); - - /* - * Prepare for processing of parameters used in remote query, if any. - */ - numParams = list_length(fsplan->fdw_exprs); - fsstate->numParams = numParams; - if (numParams > 0) - prepare_query_params((PlanState *) node, - fsplan->fdw_exprs, - numParams, - &fsstate->param_oids, - &fsstate->param_exprs, - &fsstate->param_values); + /* Get info about foreign table. */ + table = GetForeignTable(rte->relid); + user = GetUserMapping(userid, table->serverid); + + /* + * Get connection to the foreign server. Connection manager will establish + * new connection if necessary. + */ + fsstate->conn = chfdw_get_connection(user); + + /* Get private info created by planner functions. */ + fsstate->query = strVal(list_nth(fsplan->fdw_private, FdwScanPrivateSelectSql)); + fsstate->retrieved_attrs = + (List*)list_nth(fsplan->fdw_private, FdwScanPrivateRetrievedAttrs); + fsstate->fetch_size = + intVal(list_nth(fsplan->fdw_private, FdwScanPrivateFetchSize)); + + /* Create contexts for batches of tuples and per-tuple temp workspace. */ + fsstate->batch_cxt = AllocSetContextCreate( + estate->es_query_cxt, "pg_clickhouse tuple data", ALLOCSET_DEFAULT_SIZES + ); + fsstate->temp_cxt = AllocSetContextCreate( + estate->es_query_cxt, "pg_clickhouse temporary data", ALLOCSET_SMALL_SIZES + ); + + /* + * Get info we'll need for converting data fetched from the foreign server + * into local representation and error reporting during that process. + */ + if (fsplan->scan.scanrelid > 0) { + fsstate->rel = node->ss.ss_currentRelation; + fsstate->tupdesc = RelationGetDescr(fsstate->rel); + } else { + fsstate->rel = NULL; + fsstate->tupdesc = node->ss.ss_ScanTupleSlot->tts_tupleDescriptor; + } + + fsstate->attinmeta = TupleDescGetAttInMetadata(fsstate->tupdesc); + + /* + * Prepare for processing of parameters used in remote query, if any. + */ + numParams = list_length(fsplan->fdw_exprs); + fsstate->numParams = numParams; + if (numParams > 0) { + prepare_query_params( + (PlanState*)node, + fsplan->fdw_exprs, + numParams, + &fsstate->param_oids, + &fsstate->param_exprs, + &fsstate->param_values + ); + } } /* @@ -1013,81 +1075,77 @@ clickhouseBeginForeignScan(ForeignScanState * node, int eflags) * temp_context is a working context that can be reset after each tuple. */ static HeapTuple -fetch_tuple(ChFdwScanState * fsstate, TupleDesc tupdesc) -{ - Datum *values; - HeapTuple tuple = NULL; - ItemPointer ctid = NULL; - MemoryContext oldcontext; - bool *nulls; - - oldcontext = MemoryContextSwitchTo(fsstate->temp_cxt); - - /* - * Create enough slots for every attribute in the tuple definition - * (tupdesc->natts), without regard to how many we'll actually fetch - * (fsstate->retrieved_attrs), because we need all the slots to create the - * Tuple. Those we don't fetch will simply be zeros. - */ - values = (Datum *) palloc0(tupdesc->natts * sizeof(Datum)); - nulls = (bool *) palloc(tupdesc->natts * sizeof(bool)); - - /* Initialize to nulls for any columns not present in result */ - memset(nulls, true, tupdesc->natts * sizeof(bool)); - ChFdwScanRowContext ctx = { - tupdesc, - fsstate->retrieved_attrs, - fsstate->attinmeta, - fsstate->ch_cursor, - values, - nulls - }; - - /* In both cases (binary and non binary), NULL means end of tuples. */ - { - cursor_fetch_row_method fetch_fn = fsstate->conn.methods->fetch_row; - - if (fsstate->is_streaming) - { - Assert(fsstate->conn.methods->streaming_fetch_row != NULL); - fetch_fn = fsstate->conn.methods->streaming_fetch_row; - } - - if (fetch_fn(&ctx) == NULL) - { - MemoryContextSwitchTo(oldcontext); - MemoryContextReset(fsstate->temp_cxt); - return NULL; - } - } - - MemoryContextSwitchTo(oldcontext); - - tuple = heap_form_tuple(tupdesc, values, nulls); - - /* - * If we have a CTID to return, install it in both t_self and t_ctid. - * t_self is the normal place, but if the tuple is converted to a - * composite Datum, t_self will be lost; setting t_ctid allows CTID to be - * preserved during EvalPlanQual re-evaluations (see ROW_MARK_COPY code). - */ - if (ctid) - tuple->t_self = tuple->t_data->t_ctid = *ctid; - - /* - * Stomp on the xmin, xmax, and cmin fields from the tuple created by - * heap_form_tuple. heap_form_tuple actually creates the tuple with - * DatumTupleFields, not HeapTupleFields, but the executor expects - * HeapTupleFields and will happily extract system columns on that - * assumption. If we don't do this then, for example, the tuple length - * ends up in the xmin field, which isn't what we want. - */ - HeapTupleHeaderSetXmax(tuple->t_data, InvalidTransactionId); - HeapTupleHeaderSetXmin(tuple->t_data, InvalidTransactionId); - HeapTupleHeaderSetCmin(tuple->t_data, InvalidTransactionId); - - MemoryContextReset(fsstate->temp_cxt); - return tuple; +fetch_tuple(ChFdwScanState* fsstate, TupleDesc tupdesc) { + Datum* values; + HeapTuple tuple = NULL; + ItemPointer ctid = NULL; + MemoryContext oldcontext; + bool* nulls; + + oldcontext = MemoryContextSwitchTo(fsstate->temp_cxt); + + /* + * Create enough slots for every attribute in the tuple definition + * (tupdesc->natts), without regard to how many we'll actually fetch + * (fsstate->retrieved_attrs), because we need all the slots to create the + * Tuple. Those we don't fetch will simply be zeros. + */ + values = (Datum*)palloc0(tupdesc->natts * sizeof(Datum)); + nulls = (bool*)palloc(tupdesc->natts * sizeof(bool)); + + /* Initialize to nulls for any columns not present in result */ + memset(nulls, true, tupdesc->natts * sizeof(bool)); + ChFdwScanRowContext ctx = { tupdesc, + fsstate->retrieved_attrs, + fsstate->attinmeta, + fsstate->ch_cursor, + values, + nulls }; + + /* In both cases (binary and non binary), NULL means end of tuples. */ + { + cursor_fetch_row_method fetch_fn = fsstate->conn.methods->fetch_row; + + if (fsstate->is_streaming) { + Assert(fsstate->conn.methods->streaming_fetch_row != NULL); + fetch_fn = fsstate->conn.methods->streaming_fetch_row; + } + + if (fetch_fn(&ctx) == NULL) { + MemoryContextSwitchTo(oldcontext); + MemoryContextReset(fsstate->temp_cxt); + return NULL; + } + } + + MemoryContextSwitchTo(oldcontext); + + tuple = heap_form_tuple(tupdesc, values, nulls); + + /* + * If we have a CTID to return, install it in both t_self and t_ctid. + * t_self is the normal place, but if the tuple is converted to a + * composite Datum, t_self will be lost; setting t_ctid allows CTID to be + * preserved during EvalPlanQual re-evaluations (see ROW_MARK_COPY code). + */ + if (ctid) { + tuple->t_self = tuple->t_data->t_ctid = *ctid; + } + + /* + * Stomp on the xmin, xmax, and cmin fields from the tuple created by + * heap_form_tuple. heap_form_tuple actually creates the tuple with + * DatumTupleFields, not HeapTupleFields, but the executor expects + * HeapTupleFields and will happily extract system columns on that + * assumption. If we don't do this then, for example, the tuple length + * ends up in the xmin field, which isn't what we want. + */ + HeapTupleHeaderSetXmax(tuple->t_data, InvalidTransactionId); + HeapTupleHeaderSetXmin(tuple->t_data, InvalidTransactionId); + HeapTupleHeaderSetCmin(tuple->t_data, InvalidTransactionId); + + MemoryContextReset(fsstate->temp_cxt); + return tuple; } /* @@ -1095,76 +1153,72 @@ fetch_tuple(ChFdwScanState * fsstate, TupleDesc tupdesc) * Retrieve next row from the result set, or clear tuple slot to indicate * EOF. */ -static TupleTableSlot * -clickhouseIterateForeignScan(ForeignScanState * node) -{ - HeapTuple tup; - ChFdwScanState *fsstate = (ChFdwScanState *) node->fdw_state; - ExprContext *econtext = node->ss.ps.ps_ExprContext; - int numParams = fsstate->numParams; - const char **values = fsstate->param_values; - TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; - struct timeval time1, - time2; - - /* Allow query cancel (e.g. Ctrl+C) between tuple fetches. */ - CHECK_FOR_INTERRUPTS(); - - /* make query if needed */ - if (fsstate->ch_cursor == NULL) - { - MemoryContext old = MemoryContextSwitchTo(fsstate->batch_cxt); - ch_query query = new_query(fsstate->query, fsstate->numParams, fsstate->param_values, - fsstate->tupdesc, fsstate->retrieved_attrs); - - /* - * Construct array of query parameter values in text format. We do - * the conversions in the short-lived per-tuple context, so as not to - * cause a memory leak over repeated scans. - */ - if (numParams > 0) - { - process_query_params(econtext, - fsstate->param_oids, - fsstate->param_exprs, - values); - } - - fsstate->is_streaming = fsstate->fetch_size > 0 - && fsstate->conn.methods->streaming_query != NULL; - - if (fsstate->is_streaming) - { - fsstate->ch_cursor = - fsstate->conn.methods->streaming_query( - fsstate->conn.conn, &query, - fsstate->fetch_size); - } - else - { - /* Binary still falls back to the simple path for now. */ - fsstate->ch_cursor = - fsstate->conn.methods->simple_query( - fsstate->conn.conn, &query); - } - - time_used += fsstate->ch_cursor->request_time; - MemoryContextSwitchTo(old); - } - - gettimeofday(&time1, NULL); - tup = fetch_tuple(fsstate, fsstate->tupdesc); - gettimeofday(&time2, NULL); - time_used += time_diff(&time1, &time2); - - if (tup == NULL) - return ExecClearTuple(slot); - - /* - * Return the next tuple. - */ - ExecStoreHeapTuple(tup, slot, false); - return slot; +static TupleTableSlot* +clickhouseIterateForeignScan(ForeignScanState* node) { + HeapTuple tup; + ChFdwScanState* fsstate = (ChFdwScanState*)node->fdw_state; + ExprContext* econtext = node->ss.ps.ps_ExprContext; + int numParams = fsstate->numParams; + const char** values = fsstate->param_values; + TupleTableSlot* slot = node->ss.ss_ScanTupleSlot; + struct timeval time1, time2; + + /* Allow query cancel (e.g. Ctrl+C) between tuple fetches. */ + CHECK_FOR_INTERRUPTS(); + + /* make query if needed */ + if (fsstate->ch_cursor == NULL) { + MemoryContext old = MemoryContextSwitchTo(fsstate->batch_cxt); + ch_query query = new_query( + fsstate->query, + fsstate->numParams, + fsstate->param_values, + fsstate->tupdesc, + fsstate->retrieved_attrs + ); + + /* + * Construct array of query parameter values in text format. We do + * the conversions in the short-lived per-tuple context, so as not to + * cause a memory leak over repeated scans. + */ + if (numParams > 0) { + process_query_params( + econtext, fsstate->param_oids, fsstate->param_exprs, values + ); + } + + fsstate->is_streaming = + fsstate->fetch_size > 0 && fsstate->conn.methods->streaming_query != NULL; + + if (fsstate->is_streaming) { + fsstate->ch_cursor = fsstate->conn.methods->streaming_query( + fsstate->conn.conn, &query, fsstate->fetch_size + ); + } else { + /* Binary still falls back to the simple path for now. */ + fsstate->ch_cursor = + fsstate->conn.methods->simple_query(fsstate->conn.conn, &query); + } + + time_used += fsstate->ch_cursor->request_time; + MemoryContextSwitchTo(old); + } + + gettimeofday(&time1, NULL); + tup = fetch_tuple(fsstate, fsstate->tupdesc); + gettimeofday(&time2, NULL); + time_used += time_diff(&time1, &time2); + + if (tup == NULL) { + return ExecClearTuple(slot); + } + + /* + * Return the next tuple. + */ + ExecStoreHeapTuple(tup, slot, false); + return slot; } /* @@ -1172,92 +1226,94 @@ clickhouseIterateForeignScan(ForeignScanState * node) * Finish scanning foreign table and dispose objects used for this scan */ static void -clickhouseEndForeignScan(ForeignScanState * node) -{ - ChFdwScanState *fsstate = (ChFdwScanState *) node->fdw_state; - - time_used = 0; - if (fsstate && fsstate->ch_cursor) - { - MemoryContextDelete(fsstate->ch_cursor->memcxt); - fsstate->ch_cursor = NULL; - } +clickhouseEndForeignScan(ForeignScanState* node) { + ChFdwScanState* fsstate = (ChFdwScanState*)node->fdw_state; + + time_used = 0; + if (fsstate && fsstate->ch_cursor) { + MemoryContextDelete(fsstate->ch_cursor->memcxt); + fsstate->ch_cursor = NULL; + } } /* * clickhousePlanForeignModify * Plan an insert operation on a foreign table */ -static List * -clickhousePlanForeignModify(PlannerInfo * root, - ModifyTable * plan, - Index resultRelation, - int subplan_index) -{ - CmdType operation = plan->operation; - RangeTblEntry *rte = planner_rt_fetch(resultRelation, root); - Relation rel; - StringInfoData sql; - List *targetAttrs = NIL; - - initStringInfo(&sql); - - /* - * Core code already has some lock on each rel being planned, so we can - * use NoLock here. - */ - rel = table_open_compat(rte->relid, NoLock); - if (operation == CMD_INSERT) - { - TupleDesc tupdesc = RelationGetDescr(rel); - int attnum; - - for (attnum = 1; attnum <= tupdesc->natts; attnum++) - { - Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1); - - if (!attr->attisdropped) - targetAttrs = lappend_int(targetAttrs, attnum); - } - } - - /* - * Construct the SQL command string. - */ - char *table_name; - - switch (operation) - { - case CMD_INSERT: - - /* - * Write start of INSERT statement to & sql:INSERT INTO table(col - * list) - */ - table_name = chfdw_deparse_insert_sql(&sql, rte, resultRelation, rel, targetAttrs); - break; - case CMD_UPDATE: - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("ClickHouse does not support updates"))); - break; - case CMD_DELETE: - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("ClickHouse does not support deletes"))); - break; - default: - elog(ERROR, "unexpected operation: %d", (int) operation); - break; - } - - table_close_compat(rel, NoLock); - - /* - * Build the fdw_private list that will be available to the executor. - * Items in the list must match enum FdwModifyPrivateIndex, above. - */ - return list_make3(makeString(sql.data), targetAttrs, makeString(table_name)); +static List* +clickhousePlanForeignModify( + PlannerInfo* root, + ModifyTable* plan, + Index resultRelation, + int subplan_index +) { + CmdType operation = plan->operation; + RangeTblEntry* rte = planner_rt_fetch(resultRelation, root); + Relation rel; + StringInfoData sql; + List* targetAttrs = NIL; + + initStringInfo(&sql); + + /* + * Core code already has some lock on each rel being planned, so we can + * use NoLock here. + */ + rel = table_open_compat(rte->relid, NoLock); + if (operation == CMD_INSERT) { + TupleDesc tupdesc = RelationGetDescr(rel); + int attnum; + + for (attnum = 1; attnum <= tupdesc->natts; attnum++) { + Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1); + + if (!attr->attisdropped) { + targetAttrs = lappend_int(targetAttrs, attnum); + } + } + } + + /* + * Construct the SQL command string. + */ + char* table_name; + + switch (operation) { + case CMD_INSERT: + + /* + * Write start of INSERT statement to & sql:INSERT INTO table(col + * list) + */ + table_name = + chfdw_deparse_insert_sql(&sql, rte, resultRelation, rel, targetAttrs); + break; + case CMD_UPDATE: + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ClickHouse does not support updates")) + ); + break; + case CMD_DELETE: + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ClickHouse does not support deletes")) + ); + break; + default: + elog(ERROR, "unexpected operation: %d", (int)operation); + break; + } + + table_close_compat(rel, NoLock); + + /* + * Build the fdw_private list that will be available to the executor. + * Items in the list must match enum FdwModifyPrivateIndex, above. + */ + return list_make3(makeString(sql.data), targetAttrs, makeString(table_name)); } /* @@ -1265,60 +1321,63 @@ clickhousePlanForeignModify(PlannerInfo * root, * Begin an insertoperation on a foreign table */ static void -clickhouseBeginForeignModify(ModifyTableState * mtstate, - ResultRelInfo * resultRelInfo, - List * fdw_private, - int subplan_index, - int eflags) -{ - CHFdwModifyState *fmstate; - char *query; - List *target_attrs = NULL; - RangeTblEntry *rte; - char *table_name; - - /* - * Do nothing in EXPLAIN (no ANALYZE) case. resultRelInfo->ri_FdwState - * stays NULL. - */ - if (eflags & EXEC_FLAG_EXPLAIN_ONLY) - return; - - /* Deconstruct fdw_private data. */ - query = strVal(list_nth(fdw_private, FdwModifyPrivateInsertSQL)); - target_attrs = (List *) list_nth(fdw_private, FdwModifyPrivateTargetAttnums); - table_name = strVal(list_nth(fdw_private, FdwModifyPrivateTableName)); - - /* Find RTE. */ - rte = rt_fetch(resultRelInfo->ri_RangeTableIndex, - mtstate->ps.state->es_range_table); - - /* Construct an execution state. */ - fmstate = create_foreign_modify(mtstate->ps.state, - rte, - resultRelInfo, - mtstate->operation, +clickhouseBeginForeignModify( + ModifyTableState* mtstate, + ResultRelInfo* resultRelInfo, + List* fdw_private, + int subplan_index, + int eflags +) { + CHFdwModifyState* fmstate; + char* query; + List* target_attrs = NULL; + RangeTblEntry* rte; + char* table_name; + + /* + * Do nothing in EXPLAIN (no ANALYZE) case. resultRelInfo->ri_FdwState + * stays NULL. + */ + if (eflags & EXEC_FLAG_EXPLAIN_ONLY) { + return; + } + + /* Deconstruct fdw_private data. */ + query = strVal(list_nth(fdw_private, FdwModifyPrivateInsertSQL)); + target_attrs = (List*)list_nth(fdw_private, FdwModifyPrivateTargetAttnums); + table_name = strVal(list_nth(fdw_private, FdwModifyPrivateTableName)); + + /* Find RTE. */ + rte = + rt_fetch(resultRelInfo->ri_RangeTableIndex, mtstate->ps.state->es_range_table); + + /* Construct an execution state. */ + fmstate = create_foreign_modify( + mtstate->ps.state, + rte, + resultRelInfo, + mtstate->operation, #if PG_VERSION_NUM < 140000 - mtstate->mt_plans[subplan_index]->plan, + mtstate->mt_plans[subplan_index]->plan, #else - outerPlanState(mtstate)->plan, + outerPlanState(mtstate)->plan, #endif - query, - target_attrs, - table_name); + query, + target_attrs, + table_name + ); - resultRelInfo->ri_FdwState = fmstate; + resultRelInfo->ri_FdwState = fmstate; } -ForeignServer * -chfdw_get_foreign_server(Relation rel) -{ - ForeignServer *server; - ForeignTable *table; +ForeignServer* +chfdw_get_foreign_server(Relation rel) { + ForeignServer* server; + ForeignTable* table; - table = GetForeignTable(RelationGetRelid(rel)); - server = GetForeignServer(table->serverid); - return server; + table = GetForeignTable(RelationGetRelid(rel)); + server = GetForeignServer(table->serverid); + return server; } /* @@ -1326,84 +1385,84 @@ chfdw_get_foreign_server(Relation rel) * Begin an insert operation on a foreign table */ static void -clickhouseBeginForeignInsert(ModifyTableState * mtstate, - ResultRelInfo * resultRelInfo) -{ - CHFdwModifyState *fmstate; - EState *estate = mtstate->ps.state; - Index resultRelation = resultRelInfo->ri_RangeTableIndex; - Relation rel = resultRelInfo->ri_RelationDesc; - RangeTblEntry *rte; - TupleDesc tupdesc = RelationGetDescr(rel); - int attnum; - List *targetAttrs = NIL; - StringInfoData sql; - char *table_name; - - /* We transmit all columns that are defined in the foreign table. */ - for (attnum = 1; attnum <= tupdesc->natts; attnum++) - { - Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1); - - if (!attr->attisdropped) - targetAttrs = lappend_int(targetAttrs, attnum); - } - - /* - * If the foreign table is a partition, we need to create a new RTE - * describing the foreign table for use by chfdw_deparse_insert_sql and - * create_foreign_modify() below, after first copying the parent's RTE and - * modifying some fields to describe the foreign partition to work on. - * However, if this is invoked by UPDATE, the existing RTE may already - * correspond to this partition if it is one of the UPDATE subplan target - * rels; in that case, we can just use the existing RTE as-is. - */ - rte = list_nth(estate->es_range_table, resultRelation - 1); - if (rte->relid != RelationGetRelid(rel)) - { - rte = (RangeTblEntry *) copyObjectImpl(rte); - rte->relid = RelationGetRelid(rel); - rte->relkind = RELKIND_FOREIGN_TABLE; - } - - initStringInfo(&sql); - table_name = chfdw_deparse_insert_sql(&sql, rte, resultRelation, rel, targetAttrs); - - /* Construct an execution state. */ - fmstate = create_foreign_modify(mtstate->ps.state, - rte, - resultRelInfo, - CMD_INSERT, - NULL, - sql.data, - targetAttrs, - table_name); - - resultRelInfo->ri_FdwState = fmstate; +clickhouseBeginForeignInsert(ModifyTableState* mtstate, ResultRelInfo* resultRelInfo) { + CHFdwModifyState* fmstate; + EState* estate = mtstate->ps.state; + Index resultRelation = resultRelInfo->ri_RangeTableIndex; + Relation rel = resultRelInfo->ri_RelationDesc; + RangeTblEntry* rte; + TupleDesc tupdesc = RelationGetDescr(rel); + int attnum; + List* targetAttrs = NIL; + StringInfoData sql; + char* table_name; + + /* We transmit all columns that are defined in the foreign table. */ + for (attnum = 1; attnum <= tupdesc->natts; attnum++) { + Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1); + + if (!attr->attisdropped) { + targetAttrs = lappend_int(targetAttrs, attnum); + } + } + + /* + * If the foreign table is a partition, we need to create a new RTE + * describing the foreign table for use by chfdw_deparse_insert_sql and + * create_foreign_modify() below, after first copying the parent's RTE and + * modifying some fields to describe the foreign partition to work on. + * However, if this is invoked by UPDATE, the existing RTE may already + * correspond to this partition if it is one of the UPDATE subplan target + * rels; in that case, we can just use the existing RTE as-is. + */ + rte = list_nth(estate->es_range_table, resultRelation - 1); + if (rte->relid != RelationGetRelid(rel)) { + rte = (RangeTblEntry*)copyObjectImpl(rte); + rte->relid = RelationGetRelid(rel); + rte->relkind = RELKIND_FOREIGN_TABLE; + } + + initStringInfo(&sql); + table_name = chfdw_deparse_insert_sql(&sql, rte, resultRelation, rel, targetAttrs); + + /* Construct an execution state. */ + fmstate = create_foreign_modify( + mtstate->ps.state, + rte, + resultRelInfo, + CMD_INSERT, + NULL, + sql.data, + targetAttrs, + table_name + ); + + resultRelInfo->ri_FdwState = fmstate; } /* * clickhouseExecForeignInsert * Put one row to buffer, if buffer is big enough push it to ClickHouse */ -static TupleTableSlot * -clickhouseExecForeignInsert(EState * estate, - ResultRelInfo * resultRelInfo, - TupleTableSlot * slot, - TupleTableSlot * planSlot) -{ - MemoryContext oldcontext; +static TupleTableSlot* +clickhouseExecForeignInsert( + EState* estate, + ResultRelInfo* resultRelInfo, + TupleTableSlot* slot, + TupleTableSlot* planSlot +) { + MemoryContext oldcontext; - CHFdwModifyState *fmstate = (CHFdwModifyState *) resultRelInfo->ri_FdwState; + CHFdwModifyState* fmstate = (CHFdwModifyState*)resultRelInfo->ri_FdwState; - oldcontext = MemoryContextSwitchTo(fmstate->temp_cxt); + oldcontext = MemoryContextSwitchTo(fmstate->temp_cxt); - fmstate->conn.methods->insert_tuple(fmstate->state, slot); + fmstate->conn.methods->insert_tuple(fmstate->state, slot); - MemoryContextSwitchTo(oldcontext); - MemoryContextReset(fmstate->temp_cxt); + MemoryContextSwitchTo(oldcontext); + MemoryContextReset(fmstate->temp_cxt); - return slot; + return slot; } /* @@ -1411,33 +1470,31 @@ clickhouseExecForeignInsert(EState * estate, * Finish an insert operation on a foreign table */ static void -clickhouseEndForeignInsert(EState * estate, - ResultRelInfo * resultRelInfo) -{ - MemoryContext oldcontext; - - CHFdwModifyState *fmstate = (CHFdwModifyState *) resultRelInfo->ri_FdwState; - - if (fmstate) - { - /* flush */ - oldcontext = MemoryContextSwitchTo(fmstate->temp_cxt); - fmstate->conn.methods->insert_tuple(fmstate->state, NULL); - - /* - * Finalize on the happy path so the binary driver can ereport on a - * server-side INSERT exception without raising from inside the - * MemoryContext reset callback that fires during abort. - */ - if (fmstate->conn.methods->finalize_insert) - fmstate->conn.methods->finalize_insert(fmstate->state); - - MemoryContextSwitchTo(oldcontext); - MemoryContextReset(fmstate->temp_cxt); - - /* Destroy the execution state */ - finish_foreign_modify(fmstate); - } +clickhouseEndForeignInsert(EState* estate, ResultRelInfo* resultRelInfo) { + MemoryContext oldcontext; + + CHFdwModifyState* fmstate = (CHFdwModifyState*)resultRelInfo->ri_FdwState; + + if (fmstate) { + /* flush */ + oldcontext = MemoryContextSwitchTo(fmstate->temp_cxt); + fmstate->conn.methods->insert_tuple(fmstate->state, NULL); + + /* + * Finalize on the happy path so the binary driver can ereport on a + * server-side INSERT exception without raising from inside the + * MemoryContext reset callback that fires during abort. + */ + if (fmstate->conn.methods->finalize_insert) { + fmstate->conn.methods->finalize_insert(fmstate->state); + } + + MemoryContextSwitchTo(oldcontext); + MemoryContextReset(fmstate->temp_cxt); + + /* Destroy the execution state */ + finish_foreign_modify(fmstate); + } } /* @@ -1445,27 +1502,28 @@ clickhouseEndForeignInsert(EState * estate, * Execute a local join execution plan for a foreign join */ static bool -clickhouseRecheckForeignScan(ForeignScanState * node, TupleTableSlot * slot) -{ - Index scanrelid = ((Scan *) node->ss.ps.plan)->scanrelid; - PlanState *outerPlan = outerPlanState(node); - TupleTableSlot *result; +clickhouseRecheckForeignScan(ForeignScanState* node, TupleTableSlot* slot) { + Index scanrelid = ((Scan*)node->ss.ps.plan)->scanrelid; + PlanState* outerPlan = outerPlanState(node); + TupleTableSlot* result; - /* For base foreign relations, it suffices to set fdw_recheck_quals */ - if (scanrelid > 0) - return true; + /* For base foreign relations, it suffices to set fdw_recheck_quals */ + if (scanrelid > 0) { + return true; + } - Assert(outerPlan != NULL); + Assert(outerPlan != NULL); - /* Execute a local join execution plan */ - result = ExecProcNode(outerPlan); - if (TupIsNull(result)) - return false; + /* Execute a local join execution plan */ + result = ExecProcNode(outerPlan); + if (TupIsNull(result)) { + return false; + } - /* Store result in the given slot */ - ExecCopySlot(slot, result); + /* Store result in the given slot */ + ExecCopySlot(slot, result); - return true; + return true; } /* @@ -1473,35 +1531,33 @@ clickhouseRecheckForeignScan(ForeignScanState * node, TupleTableSlot * slot) * Produce extra output for EXPLAIN of a ForeignScan on a foreign table */ static void -clickhouseExplainForeignScan(ForeignScanState * node, ExplainState * es) -{ - List *fdw_private; - char *sql; - char *relations; - - fdw_private = ((ForeignScan *) node->ss.ps.plan)->fdw_private; - - /* - * Add names of relation handled by the foreign scan when the scan is a - * join - */ - if (list_length(fdw_private) > FdwScanPrivateRelations) - { - relations = strVal(list_nth(fdw_private, FdwScanPrivateRelations)); - ExplainPropertyText("Relations", relations, es); - } - - /* - * Add remote query, when VERBOSE option is specified. - */ - if (es->verbose) - { - sql = strVal(list_nth(fdw_private, FdwScanPrivateSelectSql)); - ExplainPropertyText("Remote SQL", sql, es); - } - - if (es->timing && time_used > 0) - ExplainPropertyFloat("FDW Time", "ms", time_used, 3, es); +clickhouseExplainForeignScan(ForeignScanState* node, ExplainState* es) { + List* fdw_private; + char* sql; + char* relations; + + fdw_private = ((ForeignScan*)node->ss.ps.plan)->fdw_private; + + /* + * Add names of relation handled by the foreign scan when the scan is a + * join + */ + if (list_length(fdw_private) > FdwScanPrivateRelations) { + relations = strVal(list_nth(fdw_private, FdwScanPrivateRelations)); + ExplainPropertyText("Relations", relations, es); + } + + /* + * Add remote query, when VERBOSE option is specified. + */ + if (es->verbose) { + sql = strVal(list_nth(fdw_private, FdwScanPrivateSelectSql)); + ExplainPropertyText("Remote SQL", sql, es); + } + + if (es->timing && time_used > 0) { + ExplainPropertyFloat("FDW Time", "ms", time_used, 3, es); + } } /* @@ -1517,14 +1573,18 @@ clickhouseExplainForeignScan(ForeignScanState * node, ExplainState * es) * p_startup_cost and p_total_cost variables. */ static void -estimate_path_cost_size(double *p_rows, int *p_width, - Cost * p_startup_cost, Cost * p_total_cost, double coef) -{ - /* Make pushdown paths attractive to the planner */ - *p_rows = 1000; - *p_width = 32; - *p_startup_cost = 1.0; - *p_total_cost = 0.1 + coef; +estimate_path_cost_size( + double* p_rows, + int* p_width, + Cost* p_startup_cost, + Cost* p_total_cost, + double coef +) { + /* Make pushdown paths attractive to the planner */ + *p_rows = 1000; + *p_width = 32; + *p_startup_cost = 1.0; + *p_total_cost = 0.1 + coef; } /* @@ -1532,58 +1592,60 @@ estimate_path_cost_size(double *p_rows, int *p_width, * Construct an execution state of a foreign insert * operation */ -static CHFdwModifyState * -create_foreign_modify(EState * estate, - RangeTblEntry * rte, - ResultRelInfo * rri, - CmdType operation, - Plan * subplan, - char *query, - List * target_attrs, - char *table_name) -{ - CHFdwModifyState *fmstate; - Oid userid; - ForeignTable *table; - UserMapping *user; - MemoryContext old_mcxt; - Relation rel = rri->ri_RelationDesc; - ch_query q = new_query(query, 0, NULL, RelationGetDescr(rel), target_attrs); - - /* Begin constructing CHFdwModifyState. */ - fmstate = (CHFdwModifyState *) palloc0(sizeof(CHFdwModifyState)); - fmstate->rel = rel; - - /* - * Identify which user to do the remote access as. This should match what - * ExecCheckRTEPerms() does. - */ +static CHFdwModifyState* +create_foreign_modify( + EState* estate, + RangeTblEntry* rte, + ResultRelInfo* rri, + CmdType operation, + Plan* subplan, + char* query, + List* target_attrs, + char* table_name +) { + CHFdwModifyState* fmstate; + Oid userid; + ForeignTable* table; + UserMapping* user; + MemoryContext old_mcxt; + Relation rel = rri->ri_RelationDesc; + ch_query q = new_query(query, 0, NULL, RelationGetDescr(rel), target_attrs); + + /* Begin constructing CHFdwModifyState. */ + fmstate = (CHFdwModifyState*)palloc0(sizeof(CHFdwModifyState)); + fmstate->rel = rel; + + /* + * Identify which user to do the remote access as. This should match what + * ExecCheckRTEPerms() does. + */ #if PG_VERSION_NUM >= 160000 - userid = ExecGetResultRelCheckAsUser(rri, estate); + userid = ExecGetResultRelCheckAsUser(rri, estate); #else - userid = rte->checkAsUser ? rte->checkAsUser : GetUserId(); + userid = rte->checkAsUser ? rte->checkAsUser : GetUserId(); #endif - /* Get info about foreign table. */ - table = GetForeignTable(RelationGetRelid(rel)); - user = GetUserMapping(userid, table->serverid); + /* Get info about foreign table. */ + table = GetForeignTable(RelationGetRelid(rel)); + user = GetUserMapping(userid, table->serverid); - /* make a connection and prepare an insertion state */ - fmstate->conn = chfdw_get_connection(user); + /* make a connection and prepare an insertion state */ + fmstate->conn = chfdw_get_connection(user); - old_mcxt = MemoryContextSwitchTo(PortalContext); - fmstate->state = fmstate->conn.methods->prepare_insert(fmstate->conn.conn, - rri, target_attrs, &q, table_name); - MemoryContextSwitchTo(old_mcxt); + old_mcxt = MemoryContextSwitchTo(PortalContext); + fmstate->state = fmstate->conn.methods->prepare_insert( + fmstate->conn.conn, rri, target_attrs, &q, table_name + ); + MemoryContextSwitchTo(old_mcxt); - /* Create context for per-query temp workspace. */ - fmstate->temp_cxt = AllocSetContextCreate(estate->es_query_cxt, - "pg_clickhouse temporary data", - ALLOCSET_SMALL_SIZES); + /* Create context for per-query temp workspace. */ + fmstate->temp_cxt = AllocSetContextCreate( + estate->es_query_cxt, "pg_clickhouse temporary data", ALLOCSET_SMALL_SIZES + ); - /* Set up remote query information. */ - fmstate->query = query; - return fmstate; + /* Set up remote query information. */ + fmstate->query = query; + return fmstate; } /* @@ -1591,87 +1653,87 @@ create_foreign_modify(EState * estate, * Release resources for a foreign insert/delete operation */ static void -finish_foreign_modify(CHFdwModifyState * fmstate) -{ - Assert(fmstate != NULL); - memset(&fmstate->conn, 0, sizeof(fmstate->conn)); +finish_foreign_modify(CHFdwModifyState* fmstate) { + Assert(fmstate != NULL); + memset(&fmstate->conn, 0, sizeof(fmstate->conn)); } /* * Prepare for processing of parameters used in remote query. */ static void -prepare_query_params(PlanState * node, - List * fdw_exprs, - int numParams, - Oid * *param_oids, - List * *param_exprs, - const char ***param_values) -{ - int i; - ListCell *lc; - - Assert(numParams > 0); - - /* Prepare for output conversion of parameters used in remote query. */ - *param_oids = (Oid *) palloc0(sizeof(Oid) * numParams); - - i = 0; - foreach(lc, fdw_exprs) - { - Node *param_expr = (Node *) lfirst(lc); - - (*param_oids)[i] = exprType(param_expr); - i++; - } - - /* - * Prepare remote-parameter expressions for evaluation. (Note: in - * practice, we expect that all these expressions will be just Params, so - * we could possibly do something more efficient than using the full - * expression-eval machinery for this. But probably there would be little - * benefit, and it'd require pg_clickhouse to know more than is desirable - * about Param evaluation.) - */ - *param_exprs = ExecInitExprList(fdw_exprs, node); - - /* Allocate buffer for text form of query parameters. */ - *param_values = (const char **) palloc0(numParams * sizeof(char *)); +prepare_query_params( + PlanState* node, + List* fdw_exprs, + int numParams, + Oid** param_oids, + List** param_exprs, + const char*** param_values +) { + int i; + ListCell* lc; + + Assert(numParams > 0); + + /* Prepare for output conversion of parameters used in remote query. */ + *param_oids = (Oid*)palloc0(sizeof(Oid) * numParams); + + i = 0; + foreach (lc, fdw_exprs) { + Node* param_expr = (Node*)lfirst(lc); + + (*param_oids)[i] = exprType(param_expr); + i++; + } + + /* + * Prepare remote-parameter expressions for evaluation. (Note: in + * practice, we expect that all these expressions will be just Params, so + * we could possibly do something more efficient than using the full + * expression-eval machinery for this. But probably there would be little + * benefit, and it'd require pg_clickhouse to know more than is desirable + * about Param evaluation.) + */ + *param_exprs = ExecInitExprList(fdw_exprs, node); + + /* Allocate buffer for text form of query parameters. */ + *param_values = (const char**)palloc0(numParams * sizeof(char*)); } /* * Construct array of query parameter values in text format. */ static void -process_query_params(ExprContext * econtext, - Oid * param_oids, - List * param_exprs, - const char **param_values) -{ - int i; - ListCell *lc; - - i = 0; - foreach(lc, param_exprs) - { - ExprState *expr_state = (ExprState *) lfirst(lc); - Datum expr_value; - bool isNull; - - /* Evaluate the parameter expression */ - expr_value = ExecEvalExpr(expr_state, econtext, &isNull); - - /* - * Get string representation of each parameter value by invoking - * type-specific output function, unless the value is null. - */ - if (isNull) - param_values[i] = NULL; - else - param_values[i] = chfdw_datum_to_ch_literal(expr_value, param_oids[i]); - - i++; - } +process_query_params( + ExprContext* econtext, + Oid* param_oids, + List* param_exprs, + const char** param_values +) { + int i; + ListCell* lc; + + i = 0; + foreach (lc, param_exprs) { + ExprState* expr_state = (ExprState*)lfirst(lc); + Datum expr_value; + bool isNull; + + /* Evaluate the parameter expression */ + expr_value = ExecEvalExpr(expr_state, econtext, &isNull); + + /* + * Get string representation of each parameter value by invoking + * type-specific output function, unless the value is null. + */ + if (isNull) { + param_values[i] = NULL; + } else { + param_values[i] = chfdw_datum_to_ch_literal(expr_value, param_oids[i]); + } + + i++; + } } /* @@ -1679,12 +1741,13 @@ process_query_params(ExprContext * econtext, * Test whether analyzing this foreign table is supported */ static bool -clickhouseAnalyzeForeignTable(Relation relation, - AcquireSampleRowsFunc * func, - BlockNumber * totalpages) -{ - *func = clickhouseAcquireSampleRowsFunc; - return true; +clickhouseAnalyzeForeignTable( + Relation relation, + AcquireSampleRowsFunc* func, + BlockNumber* totalpages +) { + *func = clickhouseAcquireSampleRowsFunc; + return true; } /* @@ -1692,55 +1755,54 @@ clickhouseAnalyzeForeignTable(Relation relation, * Currently a no-op. */ static int -clickhouseAcquireSampleRowsFunc(Relation relation, int elevel, - HeapTuple * rows, int targrows, - double *totalrows, - double *totaldeadrows) -{ - /* - * TODO: Consider using the SAMPLE clause: - * https://clickhouse.com/docs/sql-reference/statements/select/sample - */ - return 0; +clickhouseAcquireSampleRowsFunc( + Relation relation, + int elevel, + HeapTuple* rows, + int targrows, + double* totalrows, + double* totaldeadrows +) { + /* + * TODO: Consider using the SAMPLE clause: + * https://clickhouse.com/docs/sql-reference/statements/select/sample + */ + return 0; } static bool -is_simple_join_clause(Expr * expr) -{ - if (IsA(expr, RestrictInfo)) - { - expr = ((RestrictInfo *) expr)->clause; - } - - if (IsA(expr, OpExpr)) - { - OpExpr *opexpr = (OpExpr *) expr; - - if (chfdw_is_equal_op(opexpr->opno) == 1 - && list_length(opexpr->args) == 2 - && IsA(list_nth(opexpr->args, 0), Var) - && IsA(list_nth(opexpr->args, 1), Var)) - return true; - } - return false; +is_simple_join_clause(Expr* expr) { + if (IsA(expr, RestrictInfo)) { + expr = ((RestrictInfo*)expr)->clause; + } + + if (IsA(expr, OpExpr)) { + OpExpr* opexpr = (OpExpr*)expr; + + if (chfdw_is_equal_op(opexpr->opno) == 1 && list_length(opexpr->args) == 2 && + IsA(list_nth(opexpr->args, 0), Var) && + IsA(list_nth(opexpr->args, 1), Var)) { + return true; + } + } + return false; } -static List * -extract_join_equals(List * conds, List * *to) -{ - ListCell *lc; - List *res = NIL; - - foreach(lc, conds) - { - Expr *expr = (Expr *) lfirst(lc); - - if (is_simple_join_clause(expr)) - *to = lappend(*to, expr); - else - res = lappend(res, expr); - } - return res; +static List* +extract_join_equals(List* conds, List** to) { + ListCell* lc; + List* res = NIL; + + foreach (lc, conds) { + Expr* expr = (Expr*)lfirst(lc); + + if (is_simple_join_clause(expr)) { + *to = lappend(*to, expr); + } else { + res = lappend(res, expr); + } + } + return res; } /* @@ -1749,35 +1811,37 @@ extract_join_equals(List * conds, List * *to) * relation. */ static bool -semijoin_target_ok(PlannerInfo * root, RelOptInfo * joinrel, - RelOptInfo * outerrel, RelOptInfo * innerrel) -{ - List *vars; - ListCell *lc; - bool ok = true; - - vars = pull_var_clause((Node *) joinrel->reltarget->exprs, - PVC_RECURSE_AGGREGATES | - PVC_RECURSE_WINDOWFUNCS | - PVC_INCLUDE_PLACEHOLDERS); - - foreach(lc, vars) - { - Var *var = (Var *) lfirst(lc); - - if (!IsA(var, Var)) - continue; - - if (bms_is_member(var->varno, innerrel->relids) && - !bms_is_member(var->varno, outerrel->relids)) - { - ok = false; - break; - } - } - - list_free(vars); - return ok; +semijoin_target_ok( + PlannerInfo* root, + RelOptInfo* joinrel, + RelOptInfo* outerrel, + RelOptInfo* innerrel +) { + List* vars; + ListCell* lc; + bool ok = true; + + vars = pull_var_clause( + (Node*)joinrel->reltarget->exprs, + PVC_RECURSE_AGGREGATES | PVC_RECURSE_WINDOWFUNCS | PVC_INCLUDE_PLACEHOLDERS + ); + + foreach (lc, vars) { + Var* var = (Var*)lfirst(lc); + + if (!IsA(var, Var)) { + continue; + } + + if (bms_is_member(var->varno, innerrel->relids) && + !bms_is_member(var->varno, outerrel->relids)) { + ok = false; + break; + } + } + + list_free(vars); + return ok; } /* @@ -1786,371 +1850,371 @@ semijoin_target_ok(PlannerInfo * root, RelOptInfo * joinrel, * function to CHFdwRelationInfo passed in. */ static bool -foreign_join_ok(PlannerInfo * root, RelOptInfo * joinrel, JoinType jointype, - RelOptInfo * outerrel, RelOptInfo * innerrel, - JoinPathExtraData * extra) -{ - CHFdwRelationInfo *fpinfo; - CHFdwRelationInfo *fpinfo_o; - CHFdwRelationInfo *fpinfo_i; - ListCell *lc; - List *joinclauses; - - /* - * We support pushing down INNER, LEFT, RIGHT, FULL OUTER and SEMI joins. - * ANTI joins are not supported. - */ - if (jointype != JOIN_INNER && jointype != JOIN_LEFT && - jointype != JOIN_RIGHT && jointype != JOIN_FULL && - jointype != JOIN_SEMI) - return false; - - /* Semi-join target can only reference the outer relation */ - if (jointype == JOIN_SEMI && - !semijoin_target_ok(root, joinrel, outerrel, innerrel)) - return false; - - /* - * If either of the joining relations is marked as unsafe to pushdown, the - * join cannot be pushed down. - */ - fpinfo = (CHFdwRelationInfo *) joinrel->fdw_private; - fpinfo_o = (CHFdwRelationInfo *) outerrel->fdw_private; - fpinfo_i = (CHFdwRelationInfo *) innerrel->fdw_private; - if (!fpinfo_o || !fpinfo_o->pushdown_safe || - !fpinfo_i || !fpinfo_i->pushdown_safe) - { - return false; - } - - /* - * If joining relations have local conditions, those conditions are - * required to be applied before joining the relations. Hence the join can - * not be pushed down. - */ - if (fpinfo_o->local_conds || fpinfo_i->local_conds) - { - return false; - } - - /* - * Merge FDW options. We might be tempted to do this after we have deemed - * the foreign join to be OK. But we must do this beforehand so that we - * know which quals can be evaluated on the foreign server, which might - * depend on shippable_extensions. - */ - fpinfo->server = fpinfo_o->server; - merge_fdw_options(fpinfo, fpinfo_o, fpinfo_i); - - /* - * Separate restrict list into join quals and pushed-down (other) quals. - * - * Join quals belonging to an outer join must all be shippable, else we - * cannot execute the join remotely. Add such quals to 'joinclauses'. - * - * Add other quals to fpinfo->remote_conds if they are shippable, else to - * fpinfo->local_conds. In an inner join it's okay to execute conditions - * either locally or remotely; the same is true for pushed-down conditions - * at an outer join. - * - * Note we might return failure after having already scribbled on - * fpinfo->remote_conds and fpinfo->local_conds. That's okay because we - * won't consult those lists again if we deem the join unshippable. - */ - joinclauses = NIL; - foreach(lc, extra->restrictlist) - { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); - bool is_remote_clause = chfdw_is_foreign_expr(root, joinrel, - rinfo->clause); - - if (IS_OUTER_JOIN(jointype) && - !RINFO_IS_PUSHED_DOWN(rinfo, joinrel->relids)) - { - if (!is_remote_clause) - return false; - joinclauses = lappend(joinclauses, rinfo); - } - else - { - if (is_remote_clause) - fpinfo->remote_conds = lappend(fpinfo->remote_conds, rinfo); - else - fpinfo->local_conds = lappend(fpinfo->local_conds, rinfo); - } - } - - /* - * deparseExplicitTargetList() isn't smart enough to handle anything other - * than a Var. In particular, if there's some PlaceHolderVar that would - * need to be evaluated within this join tree (because there's an upper - * reference to a quantity that may go to NULL as a result of an outer - * join), then we can't try to push the join down because we'll fail when - * we get to deparseExplicitTargetList(). However, a PlaceHolderVar that - * needs to be evaluated *at the top* of this join tree is OK, because we - * can do that locally after fetching the results from the remote side. - */ - foreach(lc, root->placeholder_list) - { - PlaceHolderInfo *phinfo = lfirst(lc); - Relids relids; - - /* PlaceHolderInfo refers to parent relids, not child relids. */ - relids = IS_OTHER_REL(joinrel) ? - joinrel->top_parent_relids : joinrel->relids; - - if (bms_is_subset(phinfo->ph_eval_at, relids) && - bms_nonempty_difference(relids, phinfo->ph_eval_at)) - { - return false; - } - } - - /* Save the join clauses, for later use. */ - fpinfo->joinclauses = joinclauses; - - fpinfo->outerrel = outerrel; - fpinfo->innerrel = innerrel; - fpinfo->jointype = jointype; - - /* - * By default, both the input relations are not required to be deparsed as - * subqueries, but there might be some relations covered by the input - * relations that are required to be deparsed as subqueries, so save the - * relids of those relations for later use by the deparser. - */ - fpinfo->make_outerrel_subquery = false; - fpinfo->make_innerrel_subquery = false; - Assert(bms_is_subset(fpinfo_o->lower_subquery_rels, outerrel->relids)); - Assert(bms_is_subset(fpinfo_i->lower_subquery_rels, innerrel->relids)); - fpinfo->lower_subquery_rels = bms_union(fpinfo_o->lower_subquery_rels, - fpinfo_i->lower_subquery_rels); - - /* - * Pull the other remote conditions from the joining relations into join - * clauses or other remote clauses (remote_conds) of this relation - * wherever possible. This avoids building subqueries at every join step. - * - * For an inner join, clauses from both the relations are added to the - * other remote clauses. For LEFT and RIGHT OUTER join, the clauses from - * the outer side are added to remote_conds since those can be evaluated - * after the join is evaluated. The clauses from inner side are added to - * the joinclauses, since they need to be evaluated while constructing the - * join. - * - * For a FULL OUTER JOIN, the other clauses from either relation cannot be - * added to the joinclauses or remote_conds, since each relation acts as - * an outer relation for the other. - * - * The joining sides cannot have local conditions, thus no need to test - * shippability of the clauses being pulled up. - */ - switch (jointype) - { - case JOIN_INNER: - fpinfo->remote_conds = list_concat(fpinfo->remote_conds, - list_copy(fpinfo_i->remote_conds)); - fpinfo->remote_conds = list_concat(fpinfo->remote_conds, - list_copy(fpinfo_o->remote_conds)); - - /* - * For an inner join, some restrictions can be treated alike. - * Treating the pushed down conditions as join conditions allows a - * top level full outer join to be deparsed without requiring - * subqueries. - */ - Assert(!fpinfo->joinclauses); - fpinfo->remote_conds = extract_join_equals(fpinfo->remote_conds, - &fpinfo->joinclauses); - break; - - case JOIN_LEFT: - fpinfo->joinclauses = list_concat(fpinfo->joinclauses, - list_copy(fpinfo_i->remote_conds)); - fpinfo->remote_conds = list_concat(fpinfo->remote_conds, - list_copy(fpinfo_o->remote_conds)); - break; - - case JOIN_RIGHT: - fpinfo->joinclauses = list_concat(fpinfo->joinclauses, - list_copy(fpinfo_o->remote_conds)); - fpinfo->remote_conds = list_concat(fpinfo->remote_conds, - list_copy(fpinfo_i->remote_conds)); - break; - - case JOIN_SEMI: - - /* - * For semi-join, inner's conditions go to joinclauses (ON), - * outer's conditions go to remote_conds (WHERE). Extract join key - * equalities to joinclauses for the ON clause. - */ - fpinfo->joinclauses = list_concat(fpinfo->joinclauses, - list_copy(fpinfo_i->remote_conds)); - fpinfo->remote_conds = list_concat(fpinfo->remote_conds, - list_copy(fpinfo_o->remote_conds)); - fpinfo->remote_conds = extract_join_equals(fpinfo->remote_conds, - &fpinfo->joinclauses); - break; - - case JOIN_FULL: - - /* - * In this case, if any of the input relations has conditions, we - * need to deparse that relation as a subquery so that the - * conditions can be evaluated before the join. Remember it in the - * fpinfo of this relation so that the deparser can take - * appropriate action. Also, save the relids of base relations - * covered by that relation for later use by the deparser. - */ - if (fpinfo_o->remote_conds) - { - fpinfo->make_outerrel_subquery = true; - fpinfo->lower_subquery_rels = - bms_add_members(fpinfo->lower_subquery_rels, - outerrel->relids); - } - if (fpinfo_i->remote_conds) - { - fpinfo->make_innerrel_subquery = true; - fpinfo->lower_subquery_rels = - bms_add_members(fpinfo->lower_subquery_rels, - innerrel->relids); - } - break; - - default: - /* Should not happen, we have just checked this above */ - elog(ERROR, "unsupported join type %d", jointype); - } - - /* - * ClickHouse requires SEMI JOINs to have an ON clause with join - * conditions. Reject uncorrelated EXISTS subqueries that have no join - * keys. - * - * XXX Change to use ClickHouse EXISTS in this case? - * https://clickhouse.com/docs/sql-reference/operators/exists - */ - if (jointype == JOIN_SEMI && fpinfo->joinclauses == NIL) - return false; - - /* Mark that this join can be pushed down safely */ - fpinfo->pushdown_safe = true; - - /* Get user mapping */ - if (fpinfo->use_remote_estimate) - { - if (fpinfo_o->use_remote_estimate) - fpinfo->user = fpinfo_o->user; - else - fpinfo->user = fpinfo_i->user; - } - else - fpinfo->user = NULL; - - /* - * Set cached relation costs to some negative value, so that we can detect - * when they are set to some sensible costs, during one (usually the - * first) of the calls to estimate_path_cost_size(). - */ - fpinfo->rel_startup_cost = -1; - fpinfo->rel_total_cost = -1; - - /* - * Set the string describing this join relation to be used in EXPLAIN - * output of corresponding ForeignScan. - */ - fpinfo->relation_name = makeStringInfo(); - appendStringInfo(fpinfo->relation_name, "(%s) %s JOIN (%s)", - fpinfo_o->relation_name->data, - chfdw_get_jointype_name(fpinfo->jointype), - fpinfo_i->relation_name->data); - - /* - * Set the relation index. This is defined as the position of this joinrel - * in the join_rel_list list plus the length of the rtable list. Note that - * since this joinrel is at the end of the join_rel_list list when we are - * called, we can get the position by list_length. - */ - Assert(fpinfo->relation_index == 0); /* shouldn't be set yet */ - fpinfo->relation_index = - list_length(root->parse->rtable) + list_length(root->join_rel_list); - - return true; +foreign_join_ok( + PlannerInfo* root, + RelOptInfo* joinrel, + JoinType jointype, + RelOptInfo* outerrel, + RelOptInfo* innerrel, + JoinPathExtraData* extra +) { + CHFdwRelationInfo* fpinfo; + CHFdwRelationInfo* fpinfo_o; + CHFdwRelationInfo* fpinfo_i; + ListCell* lc; + List* joinclauses; + + /* + * We support pushing down INNER, LEFT, RIGHT, FULL OUTER and SEMI joins. + * ANTI joins are not supported. + */ + if (jointype != JOIN_INNER && jointype != JOIN_LEFT && jointype != JOIN_RIGHT && + jointype != JOIN_FULL && jointype != JOIN_SEMI) { + return false; + } + + /* Semi-join target can only reference the outer relation */ + if (jointype == JOIN_SEMI && + !semijoin_target_ok(root, joinrel, outerrel, innerrel)) { + return false; + } + + /* + * If either of the joining relations is marked as unsafe to pushdown, the + * join cannot be pushed down. + */ + fpinfo = (CHFdwRelationInfo*)joinrel->fdw_private; + fpinfo_o = (CHFdwRelationInfo*)outerrel->fdw_private; + fpinfo_i = (CHFdwRelationInfo*)innerrel->fdw_private; + if (!fpinfo_o || !fpinfo_o->pushdown_safe || !fpinfo_i || + !fpinfo_i->pushdown_safe) { + return false; + } + + /* + * If joining relations have local conditions, those conditions are + * required to be applied before joining the relations. Hence the join can + * not be pushed down. + */ + if (fpinfo_o->local_conds || fpinfo_i->local_conds) { + return false; + } + + /* + * Merge FDW options. We might be tempted to do this after we have deemed + * the foreign join to be OK. But we must do this beforehand so that we + * know which quals can be evaluated on the foreign server, which might + * depend on shippable_extensions. + */ + fpinfo->server = fpinfo_o->server; + merge_fdw_options(fpinfo, fpinfo_o, fpinfo_i); + + /* + * Separate restrict list into join quals and pushed-down (other) quals. + * + * Join quals belonging to an outer join must all be shippable, else we + * cannot execute the join remotely. Add such quals to 'joinclauses'. + * + * Add other quals to fpinfo->remote_conds if they are shippable, else to + * fpinfo->local_conds. In an inner join it's okay to execute conditions + * either locally or remotely; the same is true for pushed-down conditions + * at an outer join. + * + * Note we might return failure after having already scribbled on + * fpinfo->remote_conds and fpinfo->local_conds. That's okay because we + * won't consult those lists again if we deem the join unshippable. + */ + joinclauses = NIL; + foreach (lc, extra->restrictlist) { + RestrictInfo* rinfo = lfirst_node(RestrictInfo, lc); + bool is_remote_clause = chfdw_is_foreign_expr(root, joinrel, rinfo->clause); + + if (IS_OUTER_JOIN(jointype) && !RINFO_IS_PUSHED_DOWN(rinfo, joinrel->relids)) { + if (!is_remote_clause) { + return false; + } + joinclauses = lappend(joinclauses, rinfo); + } else { + if (is_remote_clause) { + fpinfo->remote_conds = lappend(fpinfo->remote_conds, rinfo); + } else { + fpinfo->local_conds = lappend(fpinfo->local_conds, rinfo); + } + } + } + + /* + * deparseExplicitTargetList() isn't smart enough to handle anything other + * than a Var. In particular, if there's some PlaceHolderVar that would + * need to be evaluated within this join tree (because there's an upper + * reference to a quantity that may go to NULL as a result of an outer + * join), then we can't try to push the join down because we'll fail when + * we get to deparseExplicitTargetList(). However, a PlaceHolderVar that + * needs to be evaluated *at the top* of this join tree is OK, because we + * can do that locally after fetching the results from the remote side. + */ + foreach (lc, root->placeholder_list) { + PlaceHolderInfo* phinfo = lfirst(lc); + Relids relids; + + /* PlaceHolderInfo refers to parent relids, not child relids. */ + relids = IS_OTHER_REL(joinrel) ? joinrel->top_parent_relids : joinrel->relids; + + if (bms_is_subset(phinfo->ph_eval_at, relids) && + bms_nonempty_difference(relids, phinfo->ph_eval_at)) { + return false; + } + } + + /* Save the join clauses, for later use. */ + fpinfo->joinclauses = joinclauses; + + fpinfo->outerrel = outerrel; + fpinfo->innerrel = innerrel; + fpinfo->jointype = jointype; + + /* + * By default, both the input relations are not required to be deparsed as + * subqueries, but there might be some relations covered by the input + * relations that are required to be deparsed as subqueries, so save the + * relids of those relations for later use by the deparser. + */ + fpinfo->make_outerrel_subquery = false; + fpinfo->make_innerrel_subquery = false; + Assert(bms_is_subset(fpinfo_o->lower_subquery_rels, outerrel->relids)); + Assert(bms_is_subset(fpinfo_i->lower_subquery_rels, innerrel->relids)); + fpinfo->lower_subquery_rels = + bms_union(fpinfo_o->lower_subquery_rels, fpinfo_i->lower_subquery_rels); + + /* + * Pull the other remote conditions from the joining relations into join + * clauses or other remote clauses (remote_conds) of this relation + * wherever possible. This avoids building subqueries at every join step. + * + * For an inner join, clauses from both the relations are added to the + * other remote clauses. For LEFT and RIGHT OUTER join, the clauses from + * the outer side are added to remote_conds since those can be evaluated + * after the join is evaluated. The clauses from inner side are added to + * the joinclauses, since they need to be evaluated while constructing the + * join. + * + * For a FULL OUTER JOIN, the other clauses from either relation cannot be + * added to the joinclauses or remote_conds, since each relation acts as + * an outer relation for the other. + * + * The joining sides cannot have local conditions, thus no need to test + * shippability of the clauses being pulled up. + */ + switch (jointype) { + case JOIN_INNER: + fpinfo->remote_conds = + list_concat(fpinfo->remote_conds, list_copy(fpinfo_i->remote_conds)); + fpinfo->remote_conds = + list_concat(fpinfo->remote_conds, list_copy(fpinfo_o->remote_conds)); + + /* + * For an inner join, some restrictions can be treated alike. + * Treating the pushed down conditions as join conditions allows a + * top level full outer join to be deparsed without requiring + * subqueries. + */ + Assert(!fpinfo->joinclauses); + fpinfo->remote_conds = + extract_join_equals(fpinfo->remote_conds, &fpinfo->joinclauses); + break; + + case JOIN_LEFT: + fpinfo->joinclauses = + list_concat(fpinfo->joinclauses, list_copy(fpinfo_i->remote_conds)); + fpinfo->remote_conds = + list_concat(fpinfo->remote_conds, list_copy(fpinfo_o->remote_conds)); + break; + + case JOIN_RIGHT: + fpinfo->joinclauses = + list_concat(fpinfo->joinclauses, list_copy(fpinfo_o->remote_conds)); + fpinfo->remote_conds = + list_concat(fpinfo->remote_conds, list_copy(fpinfo_i->remote_conds)); + break; + + case JOIN_SEMI: + + /* + * For semi-join, inner's conditions go to joinclauses (ON), + * outer's conditions go to remote_conds (WHERE). Extract join key + * equalities to joinclauses for the ON clause. + */ + fpinfo->joinclauses = + list_concat(fpinfo->joinclauses, list_copy(fpinfo_i->remote_conds)); + fpinfo->remote_conds = + list_concat(fpinfo->remote_conds, list_copy(fpinfo_o->remote_conds)); + fpinfo->remote_conds = + extract_join_equals(fpinfo->remote_conds, &fpinfo->joinclauses); + break; + + case JOIN_FULL: + + /* + * In this case, if any of the input relations has conditions, we + * need to deparse that relation as a subquery so that the + * conditions can be evaluated before the join. Remember it in the + * fpinfo of this relation so that the deparser can take + * appropriate action. Also, save the relids of base relations + * covered by that relation for later use by the deparser. + */ + if (fpinfo_o->remote_conds) { + fpinfo->make_outerrel_subquery = true; + fpinfo->lower_subquery_rels = + bms_add_members(fpinfo->lower_subquery_rels, outerrel->relids); + } + if (fpinfo_i->remote_conds) { + fpinfo->make_innerrel_subquery = true; + fpinfo->lower_subquery_rels = + bms_add_members(fpinfo->lower_subquery_rels, innerrel->relids); + } + break; + + default: + /* Should not happen, we have just checked this above */ + elog(ERROR, "unsupported join type %d", jointype); + } + + /* + * ClickHouse requires SEMI JOINs to have an ON clause with join + * conditions. Reject uncorrelated EXISTS subqueries that have no join + * keys. + * + * XXX Change to use ClickHouse EXISTS in this case? + * https://clickhouse.com/docs/sql-reference/operators/exists + */ + if (jointype == JOIN_SEMI && fpinfo->joinclauses == NIL) { + return false; + } + + /* Mark that this join can be pushed down safely */ + fpinfo->pushdown_safe = true; + + /* Get user mapping */ + if (fpinfo->use_remote_estimate) { + if (fpinfo_o->use_remote_estimate) { + fpinfo->user = fpinfo_o->user; + } else { + fpinfo->user = fpinfo_i->user; + } + } else { + fpinfo->user = NULL; + } + + /* + * Set cached relation costs to some negative value, so that we can detect + * when they are set to some sensible costs, during one (usually the + * first) of the calls to estimate_path_cost_size(). + */ + fpinfo->rel_startup_cost = -1; + fpinfo->rel_total_cost = -1; + + /* + * Set the string describing this join relation to be used in EXPLAIN + * output of corresponding ForeignScan. + */ + fpinfo->relation_name = makeStringInfo(); + appendStringInfo( + fpinfo->relation_name, + "(%s) %s JOIN (%s)", + fpinfo_o->relation_name->data, + chfdw_get_jointype_name(fpinfo->jointype), + fpinfo_i->relation_name->data + ); + + /* + * Set the relation index. This is defined as the position of this joinrel + * in the join_rel_list list plus the length of the rtable list. Note that + * since this joinrel is at the end of the join_rel_list list when we are + * called, we can get the position by list_length. + */ + Assert(fpinfo->relation_index == 0); /* shouldn't be set yet */ + fpinfo->relation_index = + list_length(root->parse->rtable) + list_length(root->join_rel_list); + + return true; } static void -add_paths_with_pathkeys_for_rel(PlannerInfo * root, RelOptInfo * rel, - Path * epq_path) -{ - List *useful_pathkeys_list = NIL; /* List of all pathkeys */ - ListCell *lc; - - useful_pathkeys_list = get_useful_pathkeys_for_relation(root, rel); - - /* Create one path for each set of pathkeys we found above. */ - foreach(lc, useful_pathkeys_list) - { - double rows; - int width; - Cost startup_cost; - Cost total_cost; - List *useful_pathkeys = lfirst(lc); - Path *sorted_epq_path; - - estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0.5); - - /* - * The EPQ path must be at least as well sorted as the path itself, in - * case it gets used as input to a mergejoin. - */ - sorted_epq_path = epq_path; - if (sorted_epq_path != NULL && - !pathkeys_contained_in(useful_pathkeys, - sorted_epq_path->pathkeys)) - sorted_epq_path = (Path *) - create_sort_path(root, - rel, - sorted_epq_path, - useful_pathkeys, - -1.0); - - if (IS_SIMPLE_REL(rel)) - add_path(rel, (Path *) - create_foreignscan_path(root, rel, - NULL, - rows, +add_paths_with_pathkeys_for_rel(PlannerInfo* root, RelOptInfo* rel, Path* epq_path) { + List* useful_pathkeys_list = NIL; /* List of all pathkeys */ + ListCell* lc; + + useful_pathkeys_list = get_useful_pathkeys_for_relation(root, rel); + + /* Create one path for each set of pathkeys we found above. */ + foreach (lc, useful_pathkeys_list) { + double rows; + int width; + Cost startup_cost; + Cost total_cost; + List* useful_pathkeys = lfirst(lc); + Path* sorted_epq_path; + + estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0.5); + + /* + * The EPQ path must be at least as well sorted as the path itself, in + * case it gets used as input to a mergejoin. + */ + sorted_epq_path = epq_path; + if (sorted_epq_path != NULL && + !pathkeys_contained_in(useful_pathkeys, sorted_epq_path->pathkeys)) { + sorted_epq_path = (Path*)create_sort_path( + root, rel, sorted_epq_path, useful_pathkeys, -1.0 + ); + } + + if (IS_SIMPLE_REL(rel)) { + add_path( + rel, + (Path*)create_foreignscan_path( + root, + rel, + NULL, + rows, #if PG_VERSION_NUM >= 180000 - 0, + 0, #endif - startup_cost, - total_cost, - useful_pathkeys, - NULL, - sorted_epq_path, + startup_cost, + total_cost, + useful_pathkeys, + NULL, + sorted_epq_path, #if PG_VERSION_NUM >= 170000 - NIL, + NIL, #endif - NIL)); - else - add_path(rel, (Path *) - create_foreign_join_path(root, rel, - NULL, - rows, + NIL + ) + ); + } else { + add_path( + rel, + (Path*)create_foreign_join_path( + root, + rel, + NULL, + rows, #if PG_VERSION_NUM >= 180000 - 0, + 0, #endif - startup_cost, - total_cost, - useful_pathkeys, - NULL, - sorted_epq_path, + startup_cost, + total_cost, + useful_pathkeys, + NULL, + sorted_epq_path, #if PG_VERSION_NUM >= 170000 - NIL, + NIL, #endif - NIL)); - } + NIL + ) + ); + } + } } /* @@ -2163,48 +2227,47 @@ add_paths_with_pathkeys_for_rel(PlannerInfo * root, RelOptInfo * rel, * expected to NULL. */ static void -merge_fdw_options(CHFdwRelationInfo * fpinfo, - const CHFdwRelationInfo * fpinfo_o, - const CHFdwRelationInfo * fpinfo_i) -{ - /* We must always have fpinfo_o. */ - Assert(fpinfo_o); - - /* fpinfo_i may be NULL, but if present the servers must both match. */ - Assert(!fpinfo_i || - fpinfo_i->server->serverid == fpinfo_o->server->serverid); - - /* - * Copy the server specific FDW options. (For a join, both relations come - * from the same server, so the server options should have the same value - * for both relations.) - */ - fpinfo->fdw_startup_cost = fpinfo_o->fdw_startup_cost; - fpinfo->fdw_tuple_cost = fpinfo_o->fdw_tuple_cost; - fpinfo->shippable_extensions = fpinfo_o->shippable_extensions; - fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate; - fpinfo->fetch_size = fpinfo_o->fetch_size; - - /* Merge the table level options from either side of the join. */ - if (fpinfo_i) - { - /* - * We'll prefer to use remote estimates for this join if any table - * from either side of the join is using remote estimates. This is - * most likely going to be preferred since they're already willing to - * pay the price of a round trip to get the remote EXPLAIN. In any - * case it's not entirely clear how we might otherwise handle this - * best. - */ - fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate || - fpinfo_i->use_remote_estimate; - - /* - * Set fetch size to maximum of the joining sides, since larger joins - * benefit from bigger batches. - */ - fpinfo->fetch_size = Max(fpinfo_o->fetch_size, fpinfo_i->fetch_size); - } +merge_fdw_options( + CHFdwRelationInfo* fpinfo, + const CHFdwRelationInfo* fpinfo_o, + const CHFdwRelationInfo* fpinfo_i +) { + /* We must always have fpinfo_o. */ + Assert(fpinfo_o); + + /* fpinfo_i may be NULL, but if present the servers must both match. */ + Assert(!fpinfo_i || fpinfo_i->server->serverid == fpinfo_o->server->serverid); + + /* + * Copy the server specific FDW options. (For a join, both relations come + * from the same server, so the server options should have the same value + * for both relations.) + */ + fpinfo->fdw_startup_cost = fpinfo_o->fdw_startup_cost; + fpinfo->fdw_tuple_cost = fpinfo_o->fdw_tuple_cost; + fpinfo->shippable_extensions = fpinfo_o->shippable_extensions; + fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate; + fpinfo->fetch_size = fpinfo_o->fetch_size; + + /* Merge the table level options from either side of the join. */ + if (fpinfo_i) { + /* + * We'll prefer to use remote estimates for this join if any table + * from either side of the join is using remote estimates. This is + * most likely going to be preferred since they're already willing to + * pay the price of a round trip to get the remote EXPLAIN. In any + * case it's not entirely clear how we might otherwise handle this + * best. + */ + fpinfo->use_remote_estimate = + fpinfo_o->use_remote_estimate || fpinfo_i->use_remote_estimate; + + /* + * Set fetch size to maximum of the joining sides, since larger joins + * benefit from bigger batches. + */ + fpinfo->fetch_size = Max(fpinfo_o->fetch_size, fpinfo_i->fetch_size); + } } /* @@ -2212,119 +2275,120 @@ merge_fdw_options(CHFdwRelationInfo * fpinfo, * Add possible ForeignPath to joinrel, if join is safe to push down. */ static void -clickhouseGetForeignJoinPaths(PlannerInfo * root, - RelOptInfo * joinrel, - RelOptInfo * outerrel, - RelOptInfo * innerrel, - JoinType jointype, - JoinPathExtraData * extra) -{ - CHFdwRelationInfo *fpinfo; - ForeignPath *joinpath; - double rows; - int width; - Cost startup_cost; - Cost total_cost; - Path *epq_path; /* Path to create plan to be executed when - * EvalPlanQual gets triggered. */ - - struct timeval time1, - time2; - - gettimeofday(&time1, NULL); - - /* - * Skip if this join combination has been considered already. - */ - if (joinrel->fdw_private) - return; - - /* - * Create unfinished CHFdwRelationInfo entry which is used to indicate - * that the join relation is already considered, so that we won't waste - * time in judging safety of join pushdown and adding the same paths again - * if found safe. Once we know that this join can be pushed down, we fill - * the entry. - */ - fpinfo = (CHFdwRelationInfo *) palloc0(sizeof(CHFdwRelationInfo)); - fpinfo->pushdown_safe = false; - joinrel->fdw_private = fpinfo; - /* attrs_used is only for base relations. */ - fpinfo->attrs_used = NULL; - epq_path = NULL; - - if (!foreign_join_ok(root, joinrel, jointype, outerrel, innerrel, extra)) - { - /* Free path required for EPQ if we copied one; we don't need it now */ - if (epq_path) - pfree(epq_path); - - return; - } - - /* - * Compute the selectivity and cost of the local_conds, so we don't have - * to do it over again for each path. The best we can do for these - * conditions is to estimate selectivity on the basis of local statistics. - * The local conditions are applied after the join has been computed on - * the remote side like quals in WHERE clause, so pass jointype as - * JOIN_INNER. - */ - fpinfo->local_conds_sel = clauselist_selectivity(root, - fpinfo->local_conds, - 0, - JOIN_INNER, - NULL); - cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root); - - /* - * If we are going to estimate costs locally, estimate the join clause - * selectivity here while we have special join info. - */ - if (!fpinfo->use_remote_estimate) - fpinfo->joinclause_sel = clauselist_selectivity(root, fpinfo->joinclauses, - 0, fpinfo->jointype, - extra->sjinfo); - - /* Estimate costs for bare join relation */ - estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0); - /* Now update this information in the joinrel */ - joinrel->rows = rows; - joinrel->reltarget->width = width; - fpinfo->rows = rows; - fpinfo->width = width; - fpinfo->startup_cost = startup_cost; - fpinfo->total_cost = total_cost; - - /* - * Create a new join path and add it to the joinrel which represents a - * join between foreign tables. - */ - joinpath = create_foreign_join_path(root, - joinrel, - NULL, /* default pathtarget */ - rows, +clickhouseGetForeignJoinPaths( + PlannerInfo* root, + RelOptInfo* joinrel, + RelOptInfo* outerrel, + RelOptInfo* innerrel, + JoinType jointype, + JoinPathExtraData* extra +) { + CHFdwRelationInfo* fpinfo; + ForeignPath* joinpath; + double rows; + int width; + Cost startup_cost; + Cost total_cost; + Path* epq_path; /* Path to create plan to be executed when + * EvalPlanQual gets triggered. */ + + struct timeval time1, time2; + + gettimeofday(&time1, NULL); + + /* + * Skip if this join combination has been considered already. + */ + if (joinrel->fdw_private) { + return; + } + + /* + * Create unfinished CHFdwRelationInfo entry which is used to indicate + * that the join relation is already considered, so that we won't waste + * time in judging safety of join pushdown and adding the same paths again + * if found safe. Once we know that this join can be pushed down, we fill + * the entry. + */ + fpinfo = (CHFdwRelationInfo*)palloc0(sizeof(CHFdwRelationInfo)); + fpinfo->pushdown_safe = false; + joinrel->fdw_private = fpinfo; + /* attrs_used is only for base relations. */ + fpinfo->attrs_used = NULL; + epq_path = NULL; + + if (!foreign_join_ok(root, joinrel, jointype, outerrel, innerrel, extra)) { + /* Free path required for EPQ if we copied one; we don't need it now */ + if (epq_path) { + pfree(epq_path); + } + + return; + } + + /* + * Compute the selectivity and cost of the local_conds, so we don't have + * to do it over again for each path. The best we can do for these + * conditions is to estimate selectivity on the basis of local statistics. + * The local conditions are applied after the join has been computed on + * the remote side like quals in WHERE clause, so pass jointype as + * JOIN_INNER. + */ + fpinfo->local_conds_sel = + clauselist_selectivity(root, fpinfo->local_conds, 0, JOIN_INNER, NULL); + cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root); + + /* + * If we are going to estimate costs locally, estimate the join clause + * selectivity here while we have special join info. + */ + if (!fpinfo->use_remote_estimate) { + fpinfo->joinclause_sel = clauselist_selectivity( + root, fpinfo->joinclauses, 0, fpinfo->jointype, extra->sjinfo + ); + } + + /* Estimate costs for bare join relation */ + estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0); + /* Now update this information in the joinrel */ + joinrel->rows = rows; + joinrel->reltarget->width = width; + fpinfo->rows = rows; + fpinfo->width = width; + fpinfo->startup_cost = startup_cost; + fpinfo->total_cost = total_cost; + + /* + * Create a new join path and add it to the joinrel which represents a + * join between foreign tables. + */ + joinpath = create_foreign_join_path( + root, + joinrel, + NULL, /* default pathtarget */ + rows, #if PG_VERSION_NUM >= 180000 - 0, + 0, #endif - startup_cost, - total_cost, - NIL, /* no pathkeys */ - NULL, - epq_path, + startup_cost, + total_cost, + NIL, /* no pathkeys */ + NULL, + epq_path, #if PG_VERSION_NUM >= 170000 - NIL, + NIL, #endif - NIL); /* no fdw_private */ + NIL + ); /* no fdw_private */ - /* Add generated path into joinrel by add_path(). */ - add_path(joinrel, (Path *) joinpath); + /* Add generated path into joinrel by add_path(). */ + add_path(joinrel, (Path*)joinpath); - /* Consider pathkeys for the join relation */ - add_paths_with_pathkeys_for_rel(root, joinrel, epq_path); + /* Consider pathkeys for the join relation */ + add_paths_with_pathkeys_for_rel(root, joinrel, epq_path); - gettimeofday(&time2, NULL); - time_used += time_diff(&time1, &time2); + gettimeofday(&time2, NULL); + time_used += time_diff(&time1, &time2); } /* @@ -2333,234 +2397,228 @@ clickhouseGetForeignJoinPaths(PlannerInfo * root, * this function to CHFdwRelationInfo of the input relation. */ static bool -foreign_grouping_ok(PlannerInfo * root, RelOptInfo * grouped_rel, - Node * havingQual) -{ - Query *query = root->parse; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) grouped_rel->fdw_private; - PathTarget *grouping_target = grouped_rel->reltarget; - CHFdwRelationInfo *ofpinfo; - List *aggvars; - ListCell *lc; - int i; - List *tlist = NIL; - - /* We currently don't support pushing Grouping Sets. */ - if (query->groupingSets) - return false; - - /* Get the fpinfo of the underlying scan relation. */ - ofpinfo = (CHFdwRelationInfo *) fpinfo->outerrel->fdw_private; - - /* - * If underlying scan relation has any local conditions, those conditions - * are required to be applied before performing aggregation. Hence the - * aggregate cannot be pushed down. - */ - if (ofpinfo->local_conds) - return false; - - /* - * Examine grouping expressions, as well as other expressions we'd need to - * compute, and check whether they are safe to push down to the foreign - * server. All GROUP BY expressions will be part of the grouping target - * and thus there is no need to search for them separately. Add grouping - * expressions into target list which will be passed to foreign server. - * - * A tricky fine point is that we must not put any expression into the - * target list that is just a foreign param (that is, something that - * deparse.c would conclude has to be sent to the foreign server). If we - * do, the expression will also appear in the fdw_exprs list of the plan - * node, and setrefs.c will get confused and decide that the fdw_exprs - * entry is actually a reference to the fdw_scan_tlist entry, resulting in - * a broken plan. Somewhat oddly, it's OK if the expression contains such - * a node, as long as it's not at top level; then no match is possible. - */ - i = 0; - foreach(lc, grouping_target->exprs) - { - Expr *expr = (Expr *) lfirst(lc); - Index sgref = get_pathtarget_sortgroupref(grouping_target, i); - ListCell *l; - - /* Check whether this expression is part of GROUP BY clause */ - if (sgref && get_sortgroupref_clause_noerr(sgref, query->groupClause)) - { - TargetEntry *tle; - - /* - * If any GROUP BY expression is not shippable, then we cannot - * push down aggregation to the foreign server. - */ - if (!chfdw_is_foreign_expr(root, grouped_rel, expr)) - return false; - - /* - * If it would be a foreign param, we can't put it into the tlist, - * so we have to fail. - */ - if (is_foreign_param(root, grouped_rel, expr)) - return false; - - /* - * Pushable, so add to tlist. We need to create a TLE for this - * expression and apply the sortgroupref to it. We cannot use - * add_to_flat_tlist() here because that avoids making duplicate - * entries in the tlist. If there are duplicate entries with - * distinct sortgrouprefs, we have to duplicate that situation in - * the output tlist. - */ - tle = makeTargetEntry(expr, list_length(tlist) + 1, NULL, false); - tle->ressortgroupref = sgref; - tlist = lappend(tlist, tle); - } - else - { - /* - * Non-grouping expression we need to compute. Is it shippable? - */ - if (chfdw_is_foreign_expr(root, grouped_rel, expr)) - { - /* Yes, so add to tlist as-is; OK to suppress duplicates */ - tlist = add_to_flat_tlist(tlist, list_make1(expr)); - } - else - { - /* Not pushable as a whole; extract its Vars and aggregates */ - aggvars = pull_var_clause((Node *) expr, - PVC_INCLUDE_AGGREGATES); - - /* - * If any aggregate expression is not shippable, then we - * cannot push down aggregation to the foreign server. - */ - if (!chfdw_is_foreign_expr(root, grouped_rel, (Expr *) aggvars)) - return false; - - /* - * Add aggregates, if any, into the targetlist. Plain Vars - * outside an aggregate can be ignored, because they should be - * either same as some GROUP BY column or part of some GROUP - * BY expression. In either case, they are already part of the - * targetlist and thus no need to add them again. In fact - * including plain Vars in the tlist when they do not match a - * GROUP BY column would cause the foreign server to complain - * that the shipped query is invalid. - */ - foreach(l, aggvars) - { - Expr *first_expr = (Expr *) lfirst(l); - - if (IsA(first_expr, Aggref)) - tlist = add_to_flat_tlist(tlist, list_make1(first_expr)); - } - } - } - - i++; - } - - /* - * Classify the pushable and non-pushable HAVING clauses and save them in - * remote_conds and local_conds of the grouped rel's fpinfo. - */ - if (havingQual) - { - ListCell *having_lc; - - foreach(having_lc, (List *) havingQual) - { - Expr *expr = (Expr *) lfirst(having_lc); - RestrictInfo *rinfo; - - /* - * Currently, the core code doesn't wrap havingQuals in - * RestrictInfos, so we must make our own. - */ - Assert(!IsA(expr, RestrictInfo)); - rinfo = make_restrictinfo( +foreign_grouping_ok(PlannerInfo* root, RelOptInfo* grouped_rel, Node* havingQual) { + Query* query = root->parse; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)grouped_rel->fdw_private; + PathTarget* grouping_target = grouped_rel->reltarget; + CHFdwRelationInfo* ofpinfo; + List* aggvars; + ListCell* lc; + int i; + List* tlist = NIL; + + /* We currently don't support pushing Grouping Sets. */ + if (query->groupingSets) { + return false; + } + + /* Get the fpinfo of the underlying scan relation. */ + ofpinfo = (CHFdwRelationInfo*)fpinfo->outerrel->fdw_private; + + /* + * If underlying scan relation has any local conditions, those conditions + * are required to be applied before performing aggregation. Hence the + * aggregate cannot be pushed down. + */ + if (ofpinfo->local_conds) { + return false; + } + + /* + * Examine grouping expressions, as well as other expressions we'd need to + * compute, and check whether they are safe to push down to the foreign + * server. All GROUP BY expressions will be part of the grouping target + * and thus there is no need to search for them separately. Add grouping + * expressions into target list which will be passed to foreign server. + * + * A tricky fine point is that we must not put any expression into the + * target list that is just a foreign param (that is, something that + * deparse.c would conclude has to be sent to the foreign server). If we + * do, the expression will also appear in the fdw_exprs list of the plan + * node, and setrefs.c will get confused and decide that the fdw_exprs + * entry is actually a reference to the fdw_scan_tlist entry, resulting in + * a broken plan. Somewhat oddly, it's OK if the expression contains such + * a node, as long as it's not at top level; then no match is possible. + */ + i = 0; + foreach (lc, grouping_target->exprs) { + Expr* expr = (Expr*)lfirst(lc); + Index sgref = get_pathtarget_sortgroupref(grouping_target, i); + ListCell* l; + + /* Check whether this expression is part of GROUP BY clause */ + if (sgref && get_sortgroupref_clause_noerr(sgref, query->groupClause)) { + TargetEntry* tle; + + /* + * If any GROUP BY expression is not shippable, then we cannot + * push down aggregation to the foreign server. + */ + if (!chfdw_is_foreign_expr(root, grouped_rel, expr)) { + return false; + } + + /* + * If it would be a foreign param, we can't put it into the tlist, + * so we have to fail. + */ + if (is_foreign_param(root, grouped_rel, expr)) { + return false; + } + + /* + * Pushable, so add to tlist. We need to create a TLE for this + * expression and apply the sortgroupref to it. We cannot use + * add_to_flat_tlist() here because that avoids making duplicate + * entries in the tlist. If there are duplicate entries with + * distinct sortgrouprefs, we have to duplicate that situation in + * the output tlist. + */ + tle = makeTargetEntry(expr, list_length(tlist) + 1, NULL, false); + tle->ressortgroupref = sgref; + tlist = lappend(tlist, tle); + } else { + /* + * Non-grouping expression we need to compute. Is it shippable? + */ + if (chfdw_is_foreign_expr(root, grouped_rel, expr)) { + /* Yes, so add to tlist as-is; OK to suppress duplicates */ + tlist = add_to_flat_tlist(tlist, list_make1(expr)); + } else { + /* Not pushable as a whole; extract its Vars and aggregates */ + aggvars = pull_var_clause((Node*)expr, PVC_INCLUDE_AGGREGATES); + + /* + * If any aggregate expression is not shippable, then we + * cannot push down aggregation to the foreign server. + */ + if (!chfdw_is_foreign_expr(root, grouped_rel, (Expr*)aggvars)) { + return false; + } + + /* + * Add aggregates, if any, into the targetlist. Plain Vars + * outside an aggregate can be ignored, because they should be + * either same as some GROUP BY column or part of some GROUP + * BY expression. In either case, they are already part of the + * targetlist and thus no need to add them again. In fact + * including plain Vars in the tlist when they do not match a + * GROUP BY column would cause the foreign server to complain + * that the shipped query is invalid. + */ + foreach (l, aggvars) { + Expr* first_expr = (Expr*)lfirst(l); + + if (IsA(first_expr, Aggref)) { + tlist = add_to_flat_tlist(tlist, list_make1(first_expr)); + } + } + } + } + + i++; + } + + /* + * Classify the pushable and non-pushable HAVING clauses and save them in + * remote_conds and local_conds of the grouped rel's fpinfo. + */ + if (havingQual) { + ListCell* having_lc; + + foreach (having_lc, (List*)havingQual) { + Expr* expr = (Expr*)lfirst(having_lc); + RestrictInfo* rinfo; + + /* + * Currently, the core code doesn't wrap havingQuals in + * RestrictInfos, so we must make our own. + */ + Assert(!IsA(expr, RestrictInfo)); + rinfo = make_restrictinfo( #if PG_VERSION_NUM >= 140000 - root, + root, #endif - expr, - true, - false, + expr, + true, + false, #if PG_VERSION_NUM >= 160000 - false, + false, #endif - false, - root->qual_security_level, - grouped_rel->relids, - NULL, - NULL); - if (chfdw_is_foreign_expr(root, grouped_rel, expr)) - fpinfo->remote_conds = lappend(fpinfo->remote_conds, rinfo); - else - fpinfo->local_conds = lappend(fpinfo->local_conds, rinfo); - } - } - - /* - * If there are any local conditions, pull Vars and aggregates from it and - * check whether they are safe to pushdown or not. - */ - if (fpinfo->local_conds) - { - List *local_aggvars = NIL; - ListCell *agg_lc; - - foreach(agg_lc, fpinfo->local_conds) - { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, agg_lc); - - local_aggvars = list_concat(local_aggvars, - pull_var_clause((Node *) rinfo->clause, - PVC_INCLUDE_AGGREGATES)); - } - - foreach(agg_lc, local_aggvars) - { - Expr *expr = (Expr *) lfirst(agg_lc); - - /* - * If aggregates within local conditions are not safe to push - * down, then we cannot push down the query. Vars are already part - * of GROUP BY clause which are checked above, so no need to - * access them again here. - */ - if (IsA(expr, Aggref)) - { - if (!chfdw_is_foreign_expr(root, grouped_rel, expr)) - return false; - - tlist = add_to_flat_tlist(tlist, list_make1(expr)); - } - } - } - - /* Store generated targetlist */ - fpinfo->grouped_tlist = tlist; - - /* Safe to pushdown */ - fpinfo->pushdown_safe = true; - - /* - * Set cached relation costs to some negative value, so that we can detect - * when they are set to some sensible costs, during one (usually the - * first) of the calls to estimate_path_cost_size(). - */ - fpinfo->rel_startup_cost = -1; - fpinfo->rel_total_cost = -1; - - /* - * Set the string describing this grouped relation to be used in EXPLAIN - * output of corresponding ForeignScan. - */ - fpinfo->relation_name = makeStringInfo(); - appendStringInfo(fpinfo->relation_name, "Aggregate on (%s)", - ofpinfo->relation_name->data); - - return true; + false, + root->qual_security_level, + grouped_rel->relids, + NULL, + NULL + ); + if (chfdw_is_foreign_expr(root, grouped_rel, expr)) { + fpinfo->remote_conds = lappend(fpinfo->remote_conds, rinfo); + } else { + fpinfo->local_conds = lappend(fpinfo->local_conds, rinfo); + } + } + } + + /* + * If there are any local conditions, pull Vars and aggregates from it and + * check whether they are safe to pushdown or not. + */ + if (fpinfo->local_conds) { + List* local_aggvars = NIL; + ListCell* agg_lc; + + foreach (agg_lc, fpinfo->local_conds) { + RestrictInfo* rinfo = lfirst_node(RestrictInfo, agg_lc); + + local_aggvars = list_concat( + local_aggvars, + pull_var_clause((Node*)rinfo->clause, PVC_INCLUDE_AGGREGATES) + ); + } + + foreach (agg_lc, local_aggvars) { + Expr* expr = (Expr*)lfirst(agg_lc); + + /* + * If aggregates within local conditions are not safe to push + * down, then we cannot push down the query. Vars are already part + * of GROUP BY clause which are checked above, so no need to + * access them again here. + */ + if (IsA(expr, Aggref)) { + if (!chfdw_is_foreign_expr(root, grouped_rel, expr)) { + return false; + } + + tlist = add_to_flat_tlist(tlist, list_make1(expr)); + } + } + } + + /* Store generated targetlist */ + fpinfo->grouped_tlist = tlist; + + /* Safe to pushdown */ + fpinfo->pushdown_safe = true; + + /* + * Set cached relation costs to some negative value, so that we can detect + * when they are set to some sensible costs, during one (usually the + * first) of the calls to estimate_path_cost_size(). + */ + fpinfo->rel_startup_cost = -1; + fpinfo->rel_total_cost = -1; + + /* + * Set the string describing this grouped relation to be used in EXPLAIN + * output of corresponding ForeignScan. + */ + fpinfo->relation_name = makeStringInfo(); + appendStringInfo( + fpinfo->relation_name, "Aggregate on (%s)", ofpinfo->relation_name->data + ); + + return true; } /* @@ -2571,59 +2629,61 @@ foreign_grouping_ok(PlannerInfo * root, RelOptInfo * grouped_rel, * Right now, we only support aggregate, grouping and having clause pushdown. */ static void -clickhouseGetForeignUpperPaths(PlannerInfo * root, UpperRelationKind stage, - RelOptInfo * input_rel, RelOptInfo * output_rel, - void *extra) -{ - CHFdwRelationInfo *fpinfo; - struct timeval time1, - time2; - - gettimeofday(&time1, NULL); - - /* - * If input rel is not safe to pushdown, then simply return as we cannot - * perform any post-join operations on the foreign server. - */ - if (!input_rel->fdw_private || - !((CHFdwRelationInfo *) input_rel->fdw_private)->pushdown_safe) - return; - - /* Ignore stages we don't support; and skip any duplicate calls. */ - if ((stage != UPPERREL_GROUP_AGG && - stage != UPPERREL_WINDOW && - stage != UPPERREL_ORDERED && - stage != UPPERREL_FINAL) || - output_rel->fdw_private) - return; - - fpinfo = (CHFdwRelationInfo *) palloc0(sizeof(CHFdwRelationInfo)); - fpinfo->pushdown_safe = false; - fpinfo->stage = stage; - output_rel->fdw_private = fpinfo; - - switch (stage) - { - case UPPERREL_GROUP_AGG: - add_foreign_grouping_paths(root, input_rel, output_rel, - (GroupPathExtraData *) extra); - break; - case UPPERREL_WINDOW: - add_foreign_window_paths(root, input_rel, output_rel); - break; - case UPPERREL_ORDERED: - add_foreign_ordered_paths(root, input_rel, output_rel); - break; - case UPPERREL_FINAL: - add_foreign_final_paths(root, input_rel, output_rel, extra); - break; - default: - elog(ERROR, "unexpected upper relation: %d", (int) stage); - break; - } - - gettimeofday(&time2, NULL); - time_used += time_diff(&time1, &time2); +clickhouseGetForeignUpperPaths( + PlannerInfo* root, + UpperRelationKind stage, + RelOptInfo* input_rel, + RelOptInfo* output_rel, + void* extra +) { + CHFdwRelationInfo* fpinfo; + struct timeval time1, time2; + + gettimeofday(&time1, NULL); + + /* + * If input rel is not safe to pushdown, then simply return as we cannot + * perform any post-join operations on the foreign server. + */ + if (!input_rel->fdw_private || + !((CHFdwRelationInfo*)input_rel->fdw_private)->pushdown_safe) { + return; + } + + /* Ignore stages we don't support; and skip any duplicate calls. */ + if ((stage != UPPERREL_GROUP_AGG && stage != UPPERREL_WINDOW && + stage != UPPERREL_ORDERED && stage != UPPERREL_FINAL) || + output_rel->fdw_private) { + return; + } + + fpinfo = (CHFdwRelationInfo*)palloc0(sizeof(CHFdwRelationInfo)); + fpinfo->pushdown_safe = false; + fpinfo->stage = stage; + output_rel->fdw_private = fpinfo; + + switch (stage) { + case UPPERREL_GROUP_AGG: + add_foreign_grouping_paths( + root, input_rel, output_rel, (GroupPathExtraData*)extra + ); + break; + case UPPERREL_WINDOW: + add_foreign_window_paths(root, input_rel, output_rel); + break; + case UPPERREL_ORDERED: + add_foreign_ordered_paths(root, input_rel, output_rel); + break; + case UPPERREL_FINAL: + add_foreign_final_paths(root, input_rel, output_rel, extra); + break; + default: + elog(ERROR, "unexpected upper relation: %d", (int)stage); + break; + } + + gettimeofday(&time2, NULL); + time_used += time_diff(&time1, &time2); } /* @@ -2634,89 +2694,99 @@ clickhouseGetForeignUpperPaths(PlannerInfo * root, UpperRelationKind stage, * given grouped_rel. */ static void -add_foreign_grouping_paths(PlannerInfo * root, RelOptInfo * input_rel, - RelOptInfo * grouped_rel, - GroupPathExtraData * extra) -{ - Query *parse = root->parse; - CHFdwRelationInfo *ifpinfo = input_rel->fdw_private; - CHFdwRelationInfo *fpinfo = grouped_rel->fdw_private; - ForeignPath *grouppath; - double rows; - int width; - Cost startup_cost; - Cost total_cost; - - /* Nothing to be done, if there is no grouping or aggregation required. */ - if (!parse->groupClause && !parse->groupingSets && !parse->hasAggs && - !root->hasHavingQual) - return; - - Assert(extra->patype == PARTITIONWISE_AGGREGATE_NONE || - extra->patype == PARTITIONWISE_AGGREGATE_FULL); - - /* save the input_rel as outerrel in fpinfo */ - fpinfo->outerrel = input_rel; - - /* - * Copy foreign table, foreign server, user mapping, FDW options etc. - * details from the input relation's fpinfo. - */ - fpinfo->table = ifpinfo->table; - fpinfo->server = ifpinfo->server; - fpinfo->user = ifpinfo->user; - merge_fdw_options(fpinfo, ifpinfo, NULL); - - /* - * Assess if it is safe to push down aggregation and grouping. - * - * Use HAVING qual from extra. In case of child partition, it will have - * translated Vars. - */ - if (!foreign_grouping_ok(root, grouped_rel, extra->havingQual)) - return; - - /* Estimate the cost of push down */ - estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0.1); - - /* Now update this information in the fpinfo */ - fpinfo->rows = rows; - fpinfo->width = width; - fpinfo->startup_cost = startup_cost; - fpinfo->total_cost = total_cost; - - /* Create and add foreign path to the grouping relation. */ +add_foreign_grouping_paths( + PlannerInfo* root, + RelOptInfo* input_rel, + RelOptInfo* grouped_rel, + GroupPathExtraData* extra +) { + Query* parse = root->parse; + CHFdwRelationInfo* ifpinfo = input_rel->fdw_private; + CHFdwRelationInfo* fpinfo = grouped_rel->fdw_private; + ForeignPath* grouppath; + double rows; + int width; + Cost startup_cost; + Cost total_cost; + + /* Nothing to be done, if there is no grouping or aggregation required. */ + if (!parse->groupClause && !parse->groupingSets && !parse->hasAggs && + !root->hasHavingQual) { + return; + } + + Assert( + extra->patype == PARTITIONWISE_AGGREGATE_NONE || + extra->patype == PARTITIONWISE_AGGREGATE_FULL + ); + + /* save the input_rel as outerrel in fpinfo */ + fpinfo->outerrel = input_rel; + + /* + * Copy foreign table, foreign server, user mapping, FDW options etc. + * details from the input relation's fpinfo. + */ + fpinfo->table = ifpinfo->table; + fpinfo->server = ifpinfo->server; + fpinfo->user = ifpinfo->user; + merge_fdw_options(fpinfo, ifpinfo, NULL); + + /* + * Assess if it is safe to push down aggregation and grouping. + * + * Use HAVING qual from extra. In case of child partition, it will have + * translated Vars. + */ + if (!foreign_grouping_ok(root, grouped_rel, extra->havingQual)) { + return; + } + + /* Estimate the cost of push down */ + estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0.1); + + /* Now update this information in the fpinfo */ + fpinfo->rows = rows; + fpinfo->width = width; + fpinfo->startup_cost = startup_cost; + fpinfo->total_cost = total_cost; + + /* Create and add foreign path to the grouping relation. */ #if (PG_VERSION_NUM < 120000) - grouppath = create_foreignscan_path(root, - grouped_rel, - grouped_rel->reltarget, - rows, - startup_cost, - total_cost, - NIL, /* no pathkeys */ - NULL, /* no required_outer */ - NULL, - NIL); /* no fdw_private */ + grouppath = create_foreignscan_path( + root, + grouped_rel, + grouped_rel->reltarget, + rows, + startup_cost, + total_cost, + NIL, /* no pathkeys */ + NULL, /* no required_outer */ + NULL, + NIL + ); /* no fdw_private */ #else - grouppath = create_foreign_upper_path(root, - grouped_rel, - grouped_rel->reltarget, - rows, + grouppath = create_foreign_upper_path( + root, + grouped_rel, + grouped_rel->reltarget, + rows, #if PG_VERSION_NUM >= 180000 - 0, + 0, #endif - startup_cost, - total_cost, - NIL, /* no pathkeys */ - NULL, + startup_cost, + total_cost, + NIL, /* no pathkeys */ + NULL, #if PG_VERSION_NUM >= 170000 - NIL, + NIL, #endif - NIL); /* no fdw_private */ + NIL + ); /* no fdw_private */ #endif - /* Add generated path into grouped_rel by add_path(). */ - add_path(grouped_rel, (Path *) grouppath); + /* Add generated path into grouped_rel by add_path(). */ + add_path(grouped_rel, (Path*)grouppath); } /* @@ -2727,77 +2797,79 @@ add_foreign_grouping_paths(PlannerInfo * root, RelOptInfo * input_rel, * shippable, and builds a target list suitable for the remote query. */ static bool -foreign_window_ok(PlannerInfo * root, RelOptInfo * window_rel) -{ - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) window_rel->fdw_private; - CHFdwRelationInfo *ofpinfo; - PathTarget *window_target = root->upper_targets[UPPERREL_WINDOW]; - ListCell *lc; - List *tlist = NIL; - int i; - - /* Get the fpinfo of the underlying scan relation. */ - ofpinfo = (CHFdwRelationInfo *) fpinfo->outerrel->fdw_private; - - /* - * If underlying scan relation has any local conditions, those conditions - * must be applied before performing windowing. Hence the window functions - * cannot be pushed down. - */ - if (ofpinfo->local_conds) - return false; - - /* - * Examine each expression in the window target list and check whether it - * is safe to push down. Build a tlist that matches the window target - * exactly (same length, same order, same sortgrouprefs) so that - * apply_pathtarget_labeling_to_tlist can match them up later. - */ - i = 0; - foreach(lc, window_target->exprs) - { - Expr *expr = (Expr *) lfirst(lc); - Index sgref = get_pathtarget_sortgroupref(window_target, i); - TargetEntry *tle; - - /* - * Check whether the expression is safe to push down. We need to check - * the expression against the window_rel so that WindowFunc nodes are - * recognized as being in an upper relation context. - */ - if (!chfdw_is_foreign_expr(root, window_rel, expr)) - return false; - - /* - * If it would be a foreign param, we can't put it into the tlist. - */ - if (is_foreign_param(root, window_rel, expr)) - return false; - - /* - * Build a TargetEntry preserving sortgroupref, without deduplication, - * to exactly mirror the PathTarget. - */ - tle = makeTargetEntry(expr, list_length(tlist) + 1, NULL, false); - tle->ressortgroupref = sgref; - tlist = lappend(tlist, tle); - i++; - } - - /* Store generated targetlist */ - fpinfo->grouped_tlist = tlist; - - /* Safe to pushdown */ - fpinfo->pushdown_safe = true; - - fpinfo->rel_startup_cost = -1; - fpinfo->rel_total_cost = -1; - - fpinfo->relation_name = makeStringInfo(); - appendStringInfo(fpinfo->relation_name, "Window on (%s)", - ofpinfo->relation_name->data); - - return true; +foreign_window_ok(PlannerInfo* root, RelOptInfo* window_rel) { + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)window_rel->fdw_private; + CHFdwRelationInfo* ofpinfo; + PathTarget* window_target = root->upper_targets[UPPERREL_WINDOW]; + ListCell* lc; + List* tlist = NIL; + int i; + + /* Get the fpinfo of the underlying scan relation. */ + ofpinfo = (CHFdwRelationInfo*)fpinfo->outerrel->fdw_private; + + /* + * If underlying scan relation has any local conditions, those conditions + * must be applied before performing windowing. Hence the window functions + * cannot be pushed down. + */ + if (ofpinfo->local_conds) { + return false; + } + + /* + * Examine each expression in the window target list and check whether it + * is safe to push down. Build a tlist that matches the window target + * exactly (same length, same order, same sortgrouprefs) so that + * apply_pathtarget_labeling_to_tlist can match them up later. + */ + i = 0; + foreach (lc, window_target->exprs) { + Expr* expr = (Expr*)lfirst(lc); + Index sgref = get_pathtarget_sortgroupref(window_target, i); + TargetEntry* tle; + + /* + * Check whether the expression is safe to push down. We need to check + * the expression against the window_rel so that WindowFunc nodes are + * recognized as being in an upper relation context. + */ + if (!chfdw_is_foreign_expr(root, window_rel, expr)) { + return false; + } + + /* + * If it would be a foreign param, we can't put it into the tlist. + */ + if (is_foreign_param(root, window_rel, expr)) { + return false; + } + + /* + * Build a TargetEntry preserving sortgroupref, without deduplication, + * to exactly mirror the PathTarget. + */ + tle = makeTargetEntry(expr, list_length(tlist) + 1, NULL, false); + tle->ressortgroupref = sgref; + tlist = lappend(tlist, tle); + i++; + } + + /* Store generated targetlist */ + fpinfo->grouped_tlist = tlist; + + /* Safe to pushdown */ + fpinfo->pushdown_safe = true; + + fpinfo->rel_startup_cost = -1; + fpinfo->rel_total_cost = -1; + + fpinfo->relation_name = makeStringInfo(); + appendStringInfo( + fpinfo->relation_name, "Window on (%s)", ofpinfo->relation_name->data + ); + + return true; } /* @@ -2808,74 +2880,81 @@ foreign_window_ok(PlannerInfo * root, RelOptInfo * window_rel) * The paths are added to the given window_rel. */ static void -add_foreign_window_paths(PlannerInfo * root, RelOptInfo * input_rel, - RelOptInfo * window_rel) -{ - CHFdwRelationInfo *ifpinfo = input_rel->fdw_private; - CHFdwRelationInfo *fpinfo = window_rel->fdw_private; - ForeignPath *window_path; - double rows; - int width; - Cost startup_cost; - Cost total_cost; - - /* Save the input_rel as outerrel in fpinfo */ - fpinfo->outerrel = input_rel; - - /* - * Copy foreign table, foreign server, user mapping, FDW options etc. - * details from the input relation's fpinfo. - */ - fpinfo->table = ifpinfo->table; - fpinfo->server = ifpinfo->server; - fpinfo->user = ifpinfo->user; - merge_fdw_options(fpinfo, ifpinfo, NULL); - - /* Assess if it is safe to push down the window functions */ - if (!foreign_window_ok(root, window_rel)) - return; - - /* Estimate the cost of push down */ - estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0.1); - - /* Now update this information in the fpinfo */ - fpinfo->rows = rows; - fpinfo->width = width; - fpinfo->startup_cost = startup_cost; - fpinfo->total_cost = total_cost; - - /* Create and add foreign path to the window relation. */ +add_foreign_window_paths( + PlannerInfo* root, + RelOptInfo* input_rel, + RelOptInfo* window_rel +) { + CHFdwRelationInfo* ifpinfo = input_rel->fdw_private; + CHFdwRelationInfo* fpinfo = window_rel->fdw_private; + ForeignPath* window_path; + double rows; + int width; + Cost startup_cost; + Cost total_cost; + + /* Save the input_rel as outerrel in fpinfo */ + fpinfo->outerrel = input_rel; + + /* + * Copy foreign table, foreign server, user mapping, FDW options etc. + * details from the input relation's fpinfo. + */ + fpinfo->table = ifpinfo->table; + fpinfo->server = ifpinfo->server; + fpinfo->user = ifpinfo->user; + merge_fdw_options(fpinfo, ifpinfo, NULL); + + /* Assess if it is safe to push down the window functions */ + if (!foreign_window_ok(root, window_rel)) { + return; + } + + /* Estimate the cost of push down */ + estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0.1); + + /* Now update this information in the fpinfo */ + fpinfo->rows = rows; + fpinfo->width = width; + fpinfo->startup_cost = startup_cost; + fpinfo->total_cost = total_cost; + + /* Create and add foreign path to the window relation. */ #if (PG_VERSION_NUM < 120000) - window_path = create_foreignscan_path(root, - window_rel, - root->upper_targets[UPPERREL_WINDOW], - rows, - startup_cost, - total_cost, - NIL, /* no pathkeys */ - NULL, /* no required_outer */ - NULL, - NIL); /* no fdw_private */ + window_path = create_foreignscan_path( + root, + window_rel, + root->upper_targets[UPPERREL_WINDOW], + rows, + startup_cost, + total_cost, + NIL, /* no pathkeys */ + NULL, /* no required_outer */ + NULL, + NIL + ); /* no fdw_private */ #else - window_path = create_foreign_upper_path(root, - window_rel, - root->upper_targets[UPPERREL_WINDOW], - rows, + window_path = create_foreign_upper_path( + root, + window_rel, + root->upper_targets[UPPERREL_WINDOW], + rows, #if PG_VERSION_NUM >= 180000 - 0, + 0, #endif - startup_cost, - total_cost, - NIL, /* no pathkeys */ - NULL, + startup_cost, + total_cost, + NIL, /* no pathkeys */ + NULL, #if PG_VERSION_NUM >= 170000 - NIL, + NIL, #endif - NIL); /* no fdw_private */ + NIL + ); /* no fdw_private */ #endif - /* Add generated path into window_rel by add_path(). */ - add_path(window_rel, (Path *) window_path); + /* Add generated path into window_rel by add_path(). */ + add_path(window_rel, (Path*)window_path); } /* @@ -2886,148 +2965,156 @@ add_foreign_window_paths(PlannerInfo * root, RelOptInfo * input_rel, * given ordered_rel. */ static void -add_foreign_ordered_paths(PlannerInfo * root, RelOptInfo * input_rel, - RelOptInfo * ordered_rel) -{ - Query *parse = root->parse; - CHFdwRelationInfo *ifpinfo = input_rel->fdw_private; - CHFdwRelationInfo *fpinfo = ordered_rel->fdw_private; - ChFdwPathExtraData *fpextra; - double rows; - int width; - Cost startup_cost; - Cost total_cost; - List *fdw_private; - ForeignPath *ordered_path; - ListCell *lc; - - /* Shouldn't get here unless the query has ORDER BY */ - Assert(parse->sortClause); - - /* We don't support cases where there are any SRFs in the targetlist */ - if (parse->hasTargetSRFs) - return; - - /* Save the input_rel as outerrel in fpinfo */ - fpinfo->outerrel = input_rel; - - /* - * Copy foreign table, foreign server, user mapping, FDW options etc. - * details from the input relation's fpinfo. - */ - fpinfo->table = ifpinfo->table; - fpinfo->server = ifpinfo->server; - fpinfo->user = ifpinfo->user; - merge_fdw_options(fpinfo, ifpinfo, NULL); - - /* - * If the input_rel is a base or join relation, we would already have - * considered pushing down the final sort to the remote server when - * creating pre-sorted foreign paths for that relation, because the - * query_pathkeys is set to the root->sort_pathkeys in that case (see - * standard_qp_callback()). - */ - if (input_rel->reloptkind == RELOPT_BASEREL || - input_rel->reloptkind == RELOPT_JOINREL) - { - Assert(root->query_pathkeys == root->sort_pathkeys); - - /* Safe to push down if the query_pathkeys is safe to push down */ - fpinfo->pushdown_safe = ifpinfo->qp_is_pushdown_safe; - - return; - } - - /* The input_rel should be a grouping or window relation */ - Assert(input_rel->reloptkind == RELOPT_UPPER_REL && - (ifpinfo->stage == UPPERREL_GROUP_AGG || - ifpinfo->stage == UPPERREL_WINDOW)); - - /* - * We try to create a path below by extending a simple foreign path for - * the underlying grouping relation to perform the final sort remotely, - * which is stored into the fdw_private list of the resulting path. - */ - - /* Assess if it is safe to push down the final sort */ - foreach(lc, root->sort_pathkeys) - { - PathKey *pathkey = (PathKey *) lfirst(lc); - EquivalenceClass *pathkey_ec = pathkey->pk_eclass; - Expr *sort_expr; - - /* - * chfdw_is_foreign_expr would detect volatile expressions as well, - * but checking ec_has_volatile here saves some cycles. - */ - if (pathkey_ec->ec_has_volatile) - return; - - /* - * Get the sort expression for the pathkey_ec. Upper rels may have an - * empty reltarget; use the planner's upper target for that stage - * instead. - */ - sort_expr = chfdw_find_em_expr_for_input_target(root, - pathkey_ec, - input_rel->reltarget->exprs != NIL - ? input_rel->reltarget - : root->upper_targets[ifpinfo->stage]); - - if (sort_expr == NULL || - !chfdw_is_foreign_expr(root, input_rel, sort_expr)) - return; - } - - /* Safe to push down */ - fpinfo->pushdown_safe = true; - - /* Construct ChFdwPathExtraData */ - fpextra = (ChFdwPathExtraData *) palloc0(sizeof(ChFdwPathExtraData)); - fpextra->target = root->upper_targets[UPPERREL_ORDERED]; - fpextra->has_final_sort = true; - - estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0.1); - - /* - * Build the fdw_private list that will be used by - * clickhouseGetForeignPlan. Items in the list must match order in enum - * FdwPathPrivateIndex. - */ - fdw_private = list_make2(makeInteger(true), makeInteger(false)); - - /* Create foreign ordering path */ +add_foreign_ordered_paths( + PlannerInfo* root, + RelOptInfo* input_rel, + RelOptInfo* ordered_rel +) { + Query* parse = root->parse; + CHFdwRelationInfo* ifpinfo = input_rel->fdw_private; + CHFdwRelationInfo* fpinfo = ordered_rel->fdw_private; + ChFdwPathExtraData* fpextra; + double rows; + int width; + Cost startup_cost; + Cost total_cost; + List* fdw_private; + ForeignPath* ordered_path; + ListCell* lc; + + /* Shouldn't get here unless the query has ORDER BY */ + Assert(parse->sortClause); + + /* We don't support cases where there are any SRFs in the targetlist */ + if (parse->hasTargetSRFs) { + return; + } + + /* Save the input_rel as outerrel in fpinfo */ + fpinfo->outerrel = input_rel; + + /* + * Copy foreign table, foreign server, user mapping, FDW options etc. + * details from the input relation's fpinfo. + */ + fpinfo->table = ifpinfo->table; + fpinfo->server = ifpinfo->server; + fpinfo->user = ifpinfo->user; + merge_fdw_options(fpinfo, ifpinfo, NULL); + + /* + * If the input_rel is a base or join relation, we would already have + * considered pushing down the final sort to the remote server when + * creating pre-sorted foreign paths for that relation, because the + * query_pathkeys is set to the root->sort_pathkeys in that case (see + * standard_qp_callback()). + */ + if (input_rel->reloptkind == RELOPT_BASEREL || + input_rel->reloptkind == RELOPT_JOINREL) { + Assert(root->query_pathkeys == root->sort_pathkeys); + + /* Safe to push down if the query_pathkeys is safe to push down */ + fpinfo->pushdown_safe = ifpinfo->qp_is_pushdown_safe; + + return; + } + + /* The input_rel should be a grouping or window relation */ + Assert( + input_rel->reloptkind == RELOPT_UPPER_REL && + (ifpinfo->stage == UPPERREL_GROUP_AGG || ifpinfo->stage == UPPERREL_WINDOW) + ); + + /* + * We try to create a path below by extending a simple foreign path for + * the underlying grouping relation to perform the final sort remotely, + * which is stored into the fdw_private list of the resulting path. + */ + + /* Assess if it is safe to push down the final sort */ + foreach (lc, root->sort_pathkeys) { + PathKey* pathkey = (PathKey*)lfirst(lc); + EquivalenceClass* pathkey_ec = pathkey->pk_eclass; + Expr* sort_expr; + + /* + * chfdw_is_foreign_expr would detect volatile expressions as well, + * but checking ec_has_volatile here saves some cycles. + */ + if (pathkey_ec->ec_has_volatile) { + return; + } + + /* + * Get the sort expression for the pathkey_ec. Upper rels may have an + * empty reltarget; use the planner's upper target for that stage + * instead. + */ + sort_expr = chfdw_find_em_expr_for_input_target( + root, + pathkey_ec, + input_rel->reltarget->exprs != NIL ? input_rel->reltarget + : root->upper_targets[ifpinfo->stage] + ); + + if (sort_expr == NULL || !chfdw_is_foreign_expr(root, input_rel, sort_expr)) { + return; + } + } + + /* Safe to push down */ + fpinfo->pushdown_safe = true; + + /* Construct ChFdwPathExtraData */ + fpextra = (ChFdwPathExtraData*)palloc0(sizeof(ChFdwPathExtraData)); + fpextra->target = root->upper_targets[UPPERREL_ORDERED]; + fpextra->has_final_sort = true; + + estimate_path_cost_size(&rows, &width, &startup_cost, &total_cost, 0.1); + + /* + * Build the fdw_private list that will be used by + * clickhouseGetForeignPlan. Items in the list must match order in enum + * FdwPathPrivateIndex. + */ + fdw_private = list_make2(makeInteger(true), makeInteger(false)); + + /* Create foreign ordering path */ #if (PG_VERSION_NUM < 120000) - ordered_path = create_foreignscan_path(root, - input_rel, - root->upper_targets[UPPERREL_ORDERED], - rows, - startup_cost, - total_cost, - root->sort_pathkeys, - NULL, /* no required_outer */ - NULL, - fdw_private); + ordered_path = create_foreignscan_path( + root, + input_rel, + root->upper_targets[UPPERREL_ORDERED], + rows, + startup_cost, + total_cost, + root->sort_pathkeys, + NULL, /* no required_outer */ + NULL, + fdw_private + ); #else - ordered_path = create_foreign_upper_path(root, - input_rel, - root->upper_targets[UPPERREL_ORDERED], - rows, + ordered_path = create_foreign_upper_path( + root, + input_rel, + root->upper_targets[UPPERREL_ORDERED], + rows, #if PG_VERSION_NUM >= 180000 - 0, + 0, #endif - startup_cost, - total_cost, - root->sort_pathkeys, - NULL, /* no extra plan */ + startup_cost, + total_cost, + root->sort_pathkeys, + NULL, /* no extra plan */ #if PG_VERSION_NUM >= 170000 - NIL, + NIL, #endif - fdw_private); + fdw_private + ); #endif - /* and add it to the ordered_rel */ - add_path(ordered_rel, (Path *) ordered_path); + /* and add it to the ordered_rel */ + add_path(ordered_rel, (Path*)ordered_path); } /* @@ -3038,146 +3125,155 @@ add_foreign_ordered_paths(PlannerInfo * root, RelOptInfo * input_rel, * given final_rel. */ static void -add_foreign_final_paths(PlannerInfo * root, RelOptInfo * input_rel, - RelOptInfo * final_rel, - void *fextra) -{ +add_foreign_final_paths( + PlannerInfo* root, + RelOptInfo* input_rel, + RelOptInfo* final_rel, + void* fextra +) { #if (PG_VERSION_NUM < 120000) - /* final paths supported only on pg >= v12 */ - return; + /* final paths supported only on pg >= v12 */ + return; #else - Query *parse = root->parse; - CHFdwRelationInfo *ifpinfo = (CHFdwRelationInfo *) input_rel->fdw_private; - CHFdwRelationInfo *fpinfo = (CHFdwRelationInfo *) final_rel->fdw_private; - bool has_final_sort = false; - List *pathkeys = NIL; - ChFdwPathExtraData *fpextra; - List *fdw_private; - ForeignPath *final_path; - FinalPathExtraData *extra = (FinalPathExtraData *) fextra; - - /* - * Currently, we only support this for SELECT commands - */ - if (parse->commandType != CMD_SELECT) - return; - - /* - * No work if there is no FOR UPDATE/SHARE clause and if there is no need - * to add a LIMIT node - */ - if (!extra->limit_needed) - return; - - /* We don't support cases where there are any SRFs in the targetlist */ - if (parse->hasTargetSRFs) - return; - - /* Save the input_rel as outerrel in fpinfo */ - fpinfo->outerrel = input_rel; - - /* - * Copy foreign table, foreign server, user mapping, FDW options etc. - * details from the input relation's fpinfo. - */ - fpinfo->table = ifpinfo->table; - fpinfo->server = ifpinfo->server; - fpinfo->user = ifpinfo->user; - merge_fdw_options(fpinfo, ifpinfo, NULL); - - Assert(extra->limit_needed); - - /* - * If the input_rel is an ordered relation, replace the input_rel with its - * input relation - */ - if (input_rel->reloptkind == RELOPT_UPPER_REL && - ifpinfo->stage == UPPERREL_ORDERED) - { - input_rel = ifpinfo->outerrel; - ifpinfo = (CHFdwRelationInfo *) input_rel->fdw_private; - has_final_sort = true; - pathkeys = root->sort_pathkeys; - } - - /* The input_rel should be a base, join, grouping, or window relation */ - Assert(input_rel->reloptkind == RELOPT_BASEREL || - input_rel->reloptkind == RELOPT_JOINREL || - (input_rel->reloptkind == RELOPT_UPPER_REL && - (ifpinfo->stage == UPPERREL_GROUP_AGG || - ifpinfo->stage == UPPERREL_WINDOW))); - - /* - * We try to create a path below by extending a simple foreign path for - * the underlying base, join, or grouping relation to perform the final - * sort (if has_final_sort) and the LIMIT restriction remotely, which is - * stored into the fdw_private list of the resulting path. (We re-estimate - * the costs of sorting the underlying relation, if has_final_sort.) - */ - - /* - * Assess if it is safe to push down the LIMIT and OFFSET to the remote - * server - */ - - /* - * If the underlying relation has any local conditions, the LIMIT/OFFSET - * cannot be pushed down. - */ - if (ifpinfo->local_conds) - return; - - /* - * Also, the LIMIT/OFFSET cannot be pushed down, if their expressions are - * not safe to remote. - */ - if (!chfdw_is_foreign_expr(root, input_rel, (Expr *) parse->limitOffset) || - !chfdw_is_foreign_expr(root, input_rel, (Expr *) parse->limitCount)) - return; - - /* Safe to push down */ - fpinfo->pushdown_safe = true; - - /* Construct ChFdwPathExtraData */ - fpextra = (ChFdwPathExtraData *) palloc0(sizeof(ChFdwPathExtraData)); - fpextra->target = root->upper_targets[UPPERREL_FINAL]; - fpextra->has_final_sort = has_final_sort; - fpextra->has_limit = extra->limit_needed; - fpextra->limit_tuples = extra->limit_tuples; - fpextra->count_est = extra->count_est; - fpextra->offset_est = extra->offset_est; - ifpinfo->use_remote_estimate = false; - - /* - * Build the fdw_private list that will be used by - * clickhouseGetForeignPlan. Items in the list must match order in enum - * FdwPathPrivateIndex. - */ - fdw_private = list_make2(makeInteger(has_final_sort), - makeInteger(extra->limit_needed)); - - /* - * Create foreign final path; this gets rid of a no-longer-needed outer - * plan (if any), which makes the EXPLAIN output look cleaner - */ - final_path = create_foreign_upper_path(root, - input_rel, - root->upper_targets[UPPERREL_FINAL], - 1, + Query* parse = root->parse; + CHFdwRelationInfo* ifpinfo = (CHFdwRelationInfo*)input_rel->fdw_private; + CHFdwRelationInfo* fpinfo = (CHFdwRelationInfo*)final_rel->fdw_private; + bool has_final_sort = false; + List* pathkeys = NIL; + ChFdwPathExtraData* fpextra; + List* fdw_private; + ForeignPath* final_path; + FinalPathExtraData* extra = (FinalPathExtraData*)fextra; + + /* + * Currently, we only support this for SELECT commands + */ + if (parse->commandType != CMD_SELECT) { + return; + } + + /* + * No work if there is no FOR UPDATE/SHARE clause and if there is no need + * to add a LIMIT node + */ + if (!extra->limit_needed) { + return; + } + + /* We don't support cases where there are any SRFs in the targetlist */ + if (parse->hasTargetSRFs) { + return; + } + + /* Save the input_rel as outerrel in fpinfo */ + fpinfo->outerrel = input_rel; + + /* + * Copy foreign table, foreign server, user mapping, FDW options etc. + * details from the input relation's fpinfo. + */ + fpinfo->table = ifpinfo->table; + fpinfo->server = ifpinfo->server; + fpinfo->user = ifpinfo->user; + merge_fdw_options(fpinfo, ifpinfo, NULL); + + Assert(extra->limit_needed); + + /* + * If the input_rel is an ordered relation, replace the input_rel with its + * input relation + */ + if (input_rel->reloptkind == RELOPT_UPPER_REL && + ifpinfo->stage == UPPERREL_ORDERED) { + input_rel = ifpinfo->outerrel; + ifpinfo = (CHFdwRelationInfo*)input_rel->fdw_private; + has_final_sort = true; + pathkeys = root->sort_pathkeys; + } + + /* The input_rel should be a base, join, grouping, or window relation */ + Assert( + input_rel->reloptkind == RELOPT_BASEREL || + input_rel->reloptkind == RELOPT_JOINREL || + (input_rel->reloptkind == RELOPT_UPPER_REL && + (ifpinfo->stage == UPPERREL_GROUP_AGG || ifpinfo->stage == UPPERREL_WINDOW)) + ); + + /* + * We try to create a path below by extending a simple foreign path for + * the underlying base, join, or grouping relation to perform the final + * sort (if has_final_sort) and the LIMIT restriction remotely, which is + * stored into the fdw_private list of the resulting path. (We re-estimate + * the costs of sorting the underlying relation, if has_final_sort.) + */ + + /* + * Assess if it is safe to push down the LIMIT and OFFSET to the remote + * server + */ + + /* + * If the underlying relation has any local conditions, the LIMIT/OFFSET + * cannot be pushed down. + */ + if (ifpinfo->local_conds) { + return; + } + + /* + * Also, the LIMIT/OFFSET cannot be pushed down, if their expressions are + * not safe to remote. + */ + if (!chfdw_is_foreign_expr(root, input_rel, (Expr*)parse->limitOffset) || + !chfdw_is_foreign_expr(root, input_rel, (Expr*)parse->limitCount)) { + return; + } + + /* Safe to push down */ + fpinfo->pushdown_safe = true; + + /* Construct ChFdwPathExtraData */ + fpextra = (ChFdwPathExtraData*)palloc0(sizeof(ChFdwPathExtraData)); + fpextra->target = root->upper_targets[UPPERREL_FINAL]; + fpextra->has_final_sort = has_final_sort; + fpextra->has_limit = extra->limit_needed; + fpextra->limit_tuples = extra->limit_tuples; + fpextra->count_est = extra->count_est; + fpextra->offset_est = extra->offset_est; + ifpinfo->use_remote_estimate = false; + + /* + * Build the fdw_private list that will be used by + * clickhouseGetForeignPlan. Items in the list must match order in enum + * FdwPathPrivateIndex. + */ + fdw_private = + list_make2(makeInteger(has_final_sort), makeInteger(extra->limit_needed)); + + /* + * Create foreign final path; this gets rid of a no-longer-needed outer + * plan (if any), which makes the EXPLAIN output look cleaner + */ + final_path = create_foreign_upper_path( + root, + input_rel, + root->upper_targets[UPPERREL_FINAL], + 1, #if PG_VERSION_NUM >= 180000 - 0, + 0, #endif - 0, - -10, - pathkeys, - NULL, /* no extra plan */ + 0, + -10, + pathkeys, + NULL, /* no extra plan */ #if PG_VERSION_NUM >= 170000 - NIL, + NIL, #endif - fdw_private); + fdw_private + ); - /* and add it to the final_rel */ - add_path(final_rel, (Path *) final_path); + /* and add it to the final_rel */ + add_path(final_rel, (Path*)final_path); #endif } @@ -3185,205 +3281,198 @@ add_foreign_final_paths(PlannerInfo * root, RelOptInfo * input_rel, * Find an equivalence class member expression, all of whose Vars come from * the indicated relation. */ -Expr * -chfdw_find_em_expr_for_rel(EquivalenceClass * ec, RelOptInfo * rel) -{ - ListCell *lc_em; - - foreach(lc_em, ec->ec_members) - { - EquivalenceMember *em = lfirst(lc_em); - - if (bms_is_subset(em->em_relids, rel->relids) && - !bms_is_empty(em->em_relids)) - { - /* - * If there is more than one equivalence member whose Vars are - * taken entirely from this relation, we'll be content to choose - * any one of those. - */ - return em->em_expr; - } - } - - /* We didn't find any suitable equivalence class expression */ - return NULL; +Expr* +chfdw_find_em_expr_for_rel(EquivalenceClass* ec, RelOptInfo* rel) { + ListCell* lc_em; + + foreach (lc_em, ec->ec_members) { + EquivalenceMember* em = lfirst(lc_em); + + if (bms_is_subset(em->em_relids, rel->relids) && !bms_is_empty(em->em_relids)) { + /* + * If there is more than one equivalence member whose Vars are + * taken entirely from this relation, we'll be content to choose + * any one of those. + */ + return em->em_expr; + } + } + + /* We didn't find any suitable equivalence class expression */ + return NULL; } /* * Find an equivalence class member expression to be computed as a sort column * in the given target. */ -Expr * -chfdw_find_em_expr_for_input_target(PlannerInfo * root, - EquivalenceClass * ec, - PathTarget * target) -{ - ListCell *lc1; - int i; - - i = 0; - foreach(lc1, target->exprs) - { - Expr *expr = (Expr *) lfirst(lc1); - Index sgref = get_pathtarget_sortgroupref(target, i); - ListCell *lc2; - - /* Ignore non-sort expressions */ - if (sgref == 0 || - get_sortgroupref_clause_noerr(sgref, - root->parse->sortClause) == NULL) - { - i++; - continue; - } - - /* We ignore binary-compatible relabeling on both ends */ - while (expr && IsA(expr, RelabelType)) - expr = ((RelabelType *) expr)->arg; - - /* Locate an EquivalenceClass member matching this expr, if any */ - foreach(lc2, ec->ec_members) - { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); - Expr *em_expr; - - /* Don't match constants */ - if (em->em_is_const) - continue; - - /* Ignore child members */ - if (em->em_is_child) - continue; - - /* Match if same expression (after stripping relabel) */ - em_expr = em->em_expr; - while (em_expr && IsA(em_expr, RelabelType)) - em_expr = ((RelabelType *) em_expr)->arg; - - if (equal(em_expr, expr)) - return em->em_expr; - } - - i++; - } - - return NULL; +Expr* +chfdw_find_em_expr_for_input_target( + PlannerInfo* root, + EquivalenceClass* ec, + PathTarget* target +) { + ListCell* lc1; + int i; + + i = 0; + foreach (lc1, target->exprs) { + Expr* expr = (Expr*)lfirst(lc1); + Index sgref = get_pathtarget_sortgroupref(target, i); + ListCell* lc2; + + /* Ignore non-sort expressions */ + if (sgref == 0 || + get_sortgroupref_clause_noerr(sgref, root->parse->sortClause) == NULL) { + i++; + continue; + } + + /* We ignore binary-compatible relabeling on both ends */ + while (expr && IsA(expr, RelabelType)) { + expr = ((RelabelType*)expr)->arg; + } + + /* Locate an EquivalenceClass member matching this expr, if any */ + foreach (lc2, ec->ec_members) { + EquivalenceMember* em = (EquivalenceMember*)lfirst(lc2); + Expr* em_expr; + + /* Don't match constants */ + if (em->em_is_const) { + continue; + } + + /* Ignore child members */ + if (em->em_is_child) { + continue; + } + + /* Match if same expression (after stripping relabel) */ + em_expr = em->em_expr; + while (em_expr && IsA(em_expr, RelabelType)) { + em_expr = ((RelabelType*)em_expr)->arg; + } + + if (equal(em_expr, expr)) { + return em->em_expr; + } + } + + i++; + } + + return NULL; } -static List * -clickhouseImportForeignSchema(ImportForeignSchemaStmt * stmt, Oid serverOid) -{ - ForeignServer *server; +static List* +clickhouseImportForeignSchema(ImportForeignSchemaStmt* stmt, Oid serverOid) { + ForeignServer* server; - server = GetForeignServer(serverOid); - return chfdw_construct_create_tables(stmt, server); + server = GetForeignServer(serverOid); + return chfdw_construct_create_tables(stmt, server); } - /* * Foreign-data wrapper handler function: return a struct with pointers * to my callback routines. */ Datum -clickhouse_fdw_handler(PG_FUNCTION_ARGS) -{ - FdwRoutine *routine = makeNode(FdwRoutine); - - /* Functions for scanning foreign tables */ - routine->GetForeignRelSize = clickhouseGetForeignRelSize; - routine->GetForeignPaths = clickhouseGetForeignPaths; - routine->GetForeignPlan = clickhouseGetForeignPlan; - routine->BeginForeignScan = clickhouseBeginForeignScan; - routine->IterateForeignScan = clickhouseIterateForeignScan; - routine->ReScanForeignScan = clickhouseEndForeignScan; - routine->EndForeignScan = clickhouseEndForeignScan; - - /* Functions for updating foreign tables */ - routine->PlanForeignModify = clickhousePlanForeignModify; - routine->BeginForeignModify = clickhouseBeginForeignModify; - routine->BeginForeignInsert = clickhouseBeginForeignInsert; - routine->ExecForeignInsert = clickhouseExecForeignInsert; - - /* - * TODO:Add support for ClickHouse 25.8 and later. - * - * routine->ExecForeignBatchInsert = XXX; - * - * routine->ExecForeignUpdate = XXX; - * - * routine->ExecForeignDelete = XXX; - */ - - routine->EndForeignInsert = clickhouseEndForeignInsert; - routine->EndForeignModify = clickhouseEndForeignInsert; - - /* Function for EvalPlanQual rechecks */ - routine->RecheckForeignScan = clickhouseRecheckForeignScan; - - /* Support functions for EXPLAIN */ - routine->ExplainForeignScan = clickhouseExplainForeignScan; - - /* Support functions for ANALYZE */ - routine->AnalyzeForeignTable = clickhouseAnalyzeForeignTable; - - /* Support functions for join push-down */ - routine->GetForeignJoinPaths = clickhouseGetForeignJoinPaths; - - /* Support functions for upper relation push-down */ - routine->GetForeignUpperPaths = clickhouseGetForeignUpperPaths; - - /* IMPORT FOREIGN SCHEMA */ - routine->ImportForeignSchema = clickhouseImportForeignSchema; - - PG_RETURN_POINTER(routine); +clickhouse_fdw_handler(PG_FUNCTION_ARGS) { + FdwRoutine* routine = makeNode(FdwRoutine); + + /* Functions for scanning foreign tables */ + routine->GetForeignRelSize = clickhouseGetForeignRelSize; + routine->GetForeignPaths = clickhouseGetForeignPaths; + routine->GetForeignPlan = clickhouseGetForeignPlan; + routine->BeginForeignScan = clickhouseBeginForeignScan; + routine->IterateForeignScan = clickhouseIterateForeignScan; + routine->ReScanForeignScan = clickhouseEndForeignScan; + routine->EndForeignScan = clickhouseEndForeignScan; + + /* Functions for updating foreign tables */ + routine->PlanForeignModify = clickhousePlanForeignModify; + routine->BeginForeignModify = clickhouseBeginForeignModify; + routine->BeginForeignInsert = clickhouseBeginForeignInsert; + routine->ExecForeignInsert = clickhouseExecForeignInsert; + + /* + * TODO:Add support for ClickHouse 25.8 and later. + * + * routine->ExecForeignBatchInsert = XXX; + * + * routine->ExecForeignUpdate = XXX; + * + * routine->ExecForeignDelete = XXX; + */ + + routine->EndForeignInsert = clickhouseEndForeignInsert; + routine->EndForeignModify = clickhouseEndForeignInsert; + + /* Function for EvalPlanQual rechecks */ + routine->RecheckForeignScan = clickhouseRecheckForeignScan; + + /* Support functions for EXPLAIN */ + routine->ExplainForeignScan = clickhouseExplainForeignScan; + + /* Support functions for ANALYZE */ + routine->AnalyzeForeignTable = clickhouseAnalyzeForeignTable; + + /* Support functions for join push-down */ + routine->GetForeignJoinPaths = clickhouseGetForeignJoinPaths; + + /* Support functions for upper relation push-down */ + routine->GetForeignUpperPaths = clickhouseGetForeignUpperPaths; + + /* IMPORT FOREIGN SCHEMA */ + routine->ImportForeignSchema = clickhouseImportForeignSchema; + + PG_RETURN_POINTER(routine); } /* * No-op function to use for the final function of a variadic function that * takes any number of values of any type. -*/ + */ Datum -clickhouse_noop(PG_FUNCTION_ARGS) -{ - PG_RETURN_NULL(); +clickhouse_noop(PG_FUNCTION_ARGS) { + PG_RETURN_NULL(); } /* * Function that simply raises an exception reporting that the operation * described by the first argument should have been pushed down. -*/ + */ Datum -clickhouse_op_push_fail(PG_FUNCTION_ARGS) -{ - text *name = PG_GETARG_TEXT_PP(0); - - ereport(ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: failed to push down %s", text_to_cstring(name)) - ); +clickhouse_op_push_fail(PG_FUNCTION_ARGS) { + text* name = PG_GETARG_TEXT_PP(0); + + ereport( + ERROR, + errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: failed to push down %s", text_to_cstring(name)) + ); } /* * Function that simply raises an exception reporting that the function should * have been pushed down. -*/ + */ Datum -clickhouse_push_fail(PG_FUNCTION_ARGS) -{ - char *name = get_func_name(fcinfo->flinfo->fn_oid); - - ereport(ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: failed to push down %s()", name) - ); +clickhouse_push_fail(PG_FUNCTION_ARGS) { + char* name = get_func_name(fcinfo->flinfo->fn_oid); + + ereport( + ERROR, + errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: failed to push down %s()", name) + ); } /* * Function returns the full pg_clickhouse library version. -*/ + */ Datum -pgch_version(PG_FUNCTION_ARGS) -{ - PG_RETURN_TEXT_P(cstring_to_text(PGCH_VERSION)); +pgch_version(PG_FUNCTION_ARGS) { + PG_RETURN_TEXT_P(cstring_to_text(PGCH_VERSION)); } diff --git a/src/http.c b/src/http.c index cbaa664..cdfcdba 100644 --- a/src/http.c +++ b/src/http.c @@ -1,7 +1,7 @@ -#include #include -#include #include +#include +#include #include #include @@ -10,40 +10,35 @@ static char curl_error_buffer[CURL_ERROR_SIZE]; static bool curl_error_happened = false; -static long curl_verbose = 0; -static void *curl_progressfunc = NULL; -static bool curl_initialized = false; +static long curl_verbose = 0; +static void* curl_progressfunc = NULL; +static bool curl_initialized = false; static char ch_query_id_prefix[5]; void -ch_http_init(int verbose, uint32_t query_id_prefix) -{ - curl_verbose = verbose; - snprintf(ch_query_id_prefix, 5, "%x", query_id_prefix); - - if (!curl_initialized) - { - curl_initialized = true; - curl_global_init(CURL_GLOBAL_ALL); - } +ch_http_init(int verbose, uint32_t query_id_prefix) { + curl_verbose = verbose; + snprintf(ch_query_id_prefix, 5, "%x", query_id_prefix); + + if (!curl_initialized) { + curl_initialized = true; + curl_global_init(CURL_GLOBAL_ALL); + } } void -ch_http_set_progress_func(void *progressfunc) -{ - curl_progressfunc = progressfunc; +ch_http_set_progress_func(void* progressfunc) { + curl_progressfunc = progressfunc; } -void * -ch_http_get_progress_func(void) -{ - return curl_progressfunc; +void* +ch_http_get_progress_func(void) { + return curl_progressfunc; } long -ch_http_get_verbose(void) -{ - return curl_verbose; +ch_http_get_verbose(void) { + return curl_verbose; } #define CLICKHOUSE_PORT 8123 @@ -56,133 +51,133 @@ ch_http_get_verbose(void) * leave curl's default (no minimum forced). */ static long -curl_min_tls_version(tls_version v) -{ - switch (v) - { - case CH_TLS_V1_0: - return CURL_SSLVERSION_TLSv1_0; - case CH_TLS_V1_1: - return CURL_SSLVERSION_TLSv1_1; - case CH_TLS_V1_2: - return CURL_SSLVERSION_TLSv1_2; - case CH_TLS_V1_3: - return CURL_SSLVERSION_TLSv1_3; - default: - return CURL_SSLVERSION_DEFAULT; - } +curl_min_tls_version(tls_version v) { + switch (v) { + case CH_TLS_V1_0: + return CURL_SSLVERSION_TLSv1_0; + case CH_TLS_V1_1: + return CURL_SSLVERSION_TLSv1_1; + case CH_TLS_V1_2: + return CURL_SSLVERSION_TLSv1_2; + case CH_TLS_V1_3: + return CURL_SSLVERSION_TLSv1_3; + default: + return CURL_SSLVERSION_DEFAULT; + } } -ch_http_connection_t * -ch_http_connection(ch_connection_details * details) -{ - int n; - char *connstring = NULL; - size_t len = 20; /* all symbols from url string + some extra */ - char *host = details->host, - *username = details->username, - *password = details->password; - int port = details->port; - - curl_error_happened = false; - ch_http_connection_t *conn = calloc(sizeof(ch_http_connection_t), 1); - - if (!conn) - goto cleanup; - - conn->curl = curl_easy_init(); - if (!conn->curl) - goto cleanup; - - conn->ssl_version = curl_min_tls_version(details->min_tls_version); - - if (details->dbname) - { - conn->dbname = strdup(details->dbname); - if (conn->dbname == NULL) - goto cleanup; - } - - - if (!host || !*host) - host = "localhost"; - - bool use_tls; - - switch (details->tls) - { - case CH_TLS_ON: - if (!port) - port = CLICKHOUSE_TLS_PORT; - use_tls = true; - break; - case CH_TLS_OFF: - if (!port) - port = CLICKHOUSE_PORT; - use_tls = false; - break; - default: /* CH_TLS_AUTO */ - if (!port) - port = ch_is_cloud_host(host) ? CLICKHOUSE_TLS_PORT : CLICKHOUSE_PORT; - use_tls = (port == CLICKHOUSE_TLS_PORT || port == HTTP_TLS_PORT); - break; - } - - len += strlen(host) + snprintf(NULL, 0, "%d", port); - - if (username) - { - username = curl_easy_escape(conn->curl, username, 0); - len += strlen(username); - } - - if (password) - { - password = curl_easy_escape(conn->curl, password, 0); - len += strlen(password); - } - - connstring = calloc(len, 1); - if (!connstring) - goto cleanup; - - char *scheme = use_tls ? "https" : "http"; - - if (username && password) - { - n = snprintf(connstring, len, "%s://%s:%s@%s:%d/", scheme, username, password, host, port); - curl_free(username); - curl_free(password); - } - else if (username) - { - n = snprintf(connstring, len, "%s://%s@%s:%d/", scheme, username, host, port); - curl_free(username); - } - else - n = snprintf(connstring, len, "%s://%s:%d/", scheme, host, port); - - if (n < 0) - goto cleanup; - - conn->base_url = connstring; - - return conn; +ch_http_connection_t* +ch_http_connection(ch_connection_details* details) { + int n; + char* connstring = NULL; + size_t len = 20; /* all symbols from url string + some extra */ + char *host = details->host, *username = details->username, + *password = details->password; + int port = details->port; + + curl_error_happened = false; + ch_http_connection_t* conn = calloc(sizeof(ch_http_connection_t), 1); + + if (!conn) { + goto cleanup; + } + + conn->curl = curl_easy_init(); + if (!conn->curl) { + goto cleanup; + } + + conn->ssl_version = curl_min_tls_version(details->min_tls_version); + + if (details->dbname) { + conn->dbname = strdup(details->dbname); + if (conn->dbname == NULL) { + goto cleanup; + } + } + + if (!host || !*host) { + host = "localhost"; + } + + bool use_tls; + + switch (details->tls) { + case CH_TLS_ON: + if (!port) { + port = CLICKHOUSE_TLS_PORT; + } + use_tls = true; + break; + case CH_TLS_OFF: + if (!port) { + port = CLICKHOUSE_PORT; + } + use_tls = false; + break; + default: /* CH_TLS_AUTO */ + if (!port) { + port = ch_is_cloud_host(host) ? CLICKHOUSE_TLS_PORT : CLICKHOUSE_PORT; + } + use_tls = (port == CLICKHOUSE_TLS_PORT || port == HTTP_TLS_PORT); + break; + } + + len += strlen(host) + snprintf(NULL, 0, "%d", port); + + if (username) { + username = curl_easy_escape(conn->curl, username, 0); + len += strlen(username); + } + + if (password) { + password = curl_easy_escape(conn->curl, password, 0); + len += strlen(password); + } + + connstring = calloc(len, 1); + if (!connstring) { + goto cleanup; + } + + char* scheme = use_tls ? "https" : "http"; + + if (username && password) { + n = snprintf( + connstring, len, "%s://%s:%s@%s:%d/", scheme, username, password, host, port + ); + curl_free(username); + curl_free(password); + } else if (username) { + n = snprintf(connstring, len, "%s://%s@%s:%d/", scheme, username, host, port); + curl_free(username); + } else { + n = snprintf(connstring, len, "%s://%s:%d/", scheme, host, port); + } + + if (n < 0) { + goto cleanup; + } + + conn->base_url = connstring; + + return conn; cleanup: - snprintf(curl_error_buffer, CURL_ERROR_SIZE, "OOM"); - curl_error_happened = true; - if (connstring) - free(connstring); - - if (conn) - { - if (conn->dbname) - free(conn->dbname); - free(conn); - } - - return NULL; + snprintf(curl_error_buffer, CURL_ERROR_SIZE, "OOM"); + curl_error_happened = true; + if (connstring) { + free(connstring); + } + + if (conn) { + if (conn->dbname) { + free(conn->dbname); + } + free(conn); + } + + return NULL; } /* @@ -192,59 +187,59 @@ ch_http_connection(ch_connection_details * details) * fetch_size, so the whole response lands in one batch that we then hand off * to the caller as a ch_http_response_t. */ -ch_http_response_t * -ch_http_simple_query(ch_http_connection_t * conn, const ch_query * query) -{ - HttpStream *stream; - ch_http_response_t *resp; - - stream = ch_http_stream_begin(conn, query, INT32_MAX); - if (stream == NULL) - return NULL; - - resp = calloc(1, sizeof(*resp)); - if (resp == NULL) - { - ch_http_stream_end(stream); - return NULL; - } - - resp->http_status = ch_http_stream_status(stream); - resp->pretransfer_time = ch_http_stream_request_time(stream) / 1000.0; - resp->total_time = ch_http_stream_total_time(stream) / 1000.0; - memcpy(resp->query_id, ch_http_stream_query_id(stream), CH_HTTP_QUERY_ID_LEN); - ch_http_stream_take_body(stream, &resp->data, &resp->datasize); - - if (curl_verbose && resp->http_status != CH_HTTP_STATUS_OK && resp->data) - fprintf(stderr, "%s", resp->data); - - ch_http_stream_end(stream); - return resp; +ch_http_response_t* +ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query) { + HttpStream* stream; + ch_http_response_t* resp; + + stream = ch_http_stream_begin(conn, query, INT32_MAX); + if (stream == NULL) { + return NULL; + } + + resp = calloc(1, sizeof(*resp)); + if (resp == NULL) { + ch_http_stream_end(stream); + return NULL; + } + + resp->http_status = ch_http_stream_status(stream); + resp->pretransfer_time = ch_http_stream_request_time(stream) / 1000.0; + resp->total_time = ch_http_stream_total_time(stream) / 1000.0; + memcpy(resp->query_id, ch_http_stream_query_id(stream), CH_HTTP_QUERY_ID_LEN); + ch_http_stream_take_body(stream, &resp->data, &resp->datasize); + + if (curl_verbose && resp->http_status != CH_HTTP_STATUS_OK && resp->data) { + fprintf(stderr, "%s", resp->data); + } + + ch_http_stream_end(stream); + return resp; } void -ch_http_close(ch_http_connection_t * conn) -{ - free(conn->base_url); - if (conn->dbname) - free(conn->dbname); - curl_easy_cleanup(conn->curl); +ch_http_close(ch_http_connection_t* conn) { + free(conn->base_url); + if (conn->dbname) { + free(conn->dbname); + } + curl_easy_cleanup(conn->curl); } -char * -ch_http_last_error(void) -{ - if (curl_error_happened) - return curl_error_buffer; +char* +ch_http_last_error(void) { + if (curl_error_happened) { + return curl_error_buffer; + } - return NULL; + return NULL; } void -ch_http_response_free(ch_http_response_t * resp) -{ - if (resp->data) - free(resp->data); +ch_http_response_free(ch_http_response_t* resp) { + if (resp->data) { + free(resp->data); + } - free(resp); + free(resp); } diff --git a/src/http_streaming.c b/src/http_streaming.c index 80143d0..d9337d5 100644 --- a/src/http_streaming.c +++ b/src/http_streaming.c @@ -11,17 +11,18 @@ * *------------------------------------------------------------------------- */ -#include #include +#include #include #include #include #include "postgres.h" + #include "http.h" -#include "internal.h" #include "http_streaming.h" +#include "internal.h" #include "kv_list.h" #ifndef CURL_WRITEFUNC_ERROR @@ -35,46 +36,49 @@ * HttpStream — opaque struct. * ---------------------------------------------------------------- */ -struct HttpStream -{ - /* Connection (borrowed, not owned) */ - ch_http_connection_t *conn; - - /* Owned CURL resources */ - CURL *curl; - CURLM *multi; - struct curl_slist *headers; - curl_mime *form; - char *url; /* allocated by curl_url_get, freed with - * curl_free */ - - /* Stream buffer */ - char *buf; - size_t buf_allocated; - size_t write_pos; - size_t parse_pos; - size_t batch_end; - int32 fetch_size; /* approximate batch size in bytes */ - bool paused; - bool started; - bool transfer_done; - char error_buffer[CURL_ERROR_SIZE]; - - /* Public state readable via C accessors */ - long http_status; - char query_id[CH_HTTP_QUERY_ID_LEN]; - double pretransfer_time; - double total_time; - char *error_msg; /* strdup'd, freed with free() */ +struct HttpStream { + /* Connection (borrowed, not owned) */ + ch_http_connection_t* conn; + + /* Owned CURL resources */ + CURL* curl; + CURLM* multi; + struct curl_slist* headers; + curl_mime* form; + char* url; /* allocated by curl_url_get, freed with + * curl_free */ + + /* Stream buffer */ + char* buf; + size_t buf_allocated; + size_t write_pos; + size_t parse_pos; + size_t batch_end; + int32 fetch_size; /* approximate batch size in bytes */ + bool paused; + bool started; + bool transfer_done; + char error_buffer[CURL_ERROR_SIZE]; + + /* Public state readable via C accessors */ + long http_status; + char query_id[CH_HTTP_QUERY_ID_LEN]; + double pretransfer_time; + double total_time; + char* error_msg; /* strdup'd, freed with free() */ }; /* Forward declarations of static helpers */ -static void setup_curl(HttpStream * stream, const ch_query * query); -static void capture_transfer_info(HttpStream * stream); -static void compact_buffer(HttpStream * stream); -static size_t find_batch_end(const HttpStream * stream); -static size_t write_callback(void *contents, size_t size, size_t nmemb, - void *userp); +static void +setup_curl(HttpStream* stream, const ch_query* query); +static void +capture_transfer_info(HttpStream* stream); +static void +compact_buffer(HttpStream* stream); +static size_t +find_batch_end(const HttpStream* stream); +static size_t +write_callback(void* contents, size_t size, size_t nmemb, void* userp); /* ---------------------------------------------------------------- * setup_curl — configure the CURL easy handle for this query. @@ -82,97 +86,100 @@ static size_t write_callback(void *contents, size_t size, size_t nmemb, * ---------------------------------------------------------------- */ static void -setup_curl(HttpStream * stream, const ch_query * query) -{ - CURLU *cu = curl_url(); - char temp_buf[512]; - - /* Build URL with query_id and settings */ - curl_url_set(cu, CURLUPART_URL, stream->conn->base_url, 0); - - snprintf(temp_buf, sizeof(temp_buf), "query_id=%s", stream->query_id); - curl_url_set(cu, CURLUPART_QUERY, temp_buf, - CURLU_APPENDQUERY | CURLU_URLENCODE); - - for (kv_iter iter = new_kv_iter(query->settings); - !kv_iter_done(&iter); kv_iter_next(&iter)) - { - if (strcmp(iter.name, "date_time_output_format") == 0 || - strcmp(iter.name, "format_tsv_null_representation") == 0 || - strcmp(iter.name, "output_format_tsv_crlf_end_of_line") == 0) - continue; - snprintf(temp_buf, sizeof(temp_buf), "%s=%s", - iter.name, iter.value); - curl_url_set(cu, CURLUPART_QUERY, temp_buf, - CURLU_APPENDQUERY | CURLU_URLENCODE); - } - - curl_url_set(cu, CURLUPART_QUERY, - "date_time_output_format=iso", - CURLU_APPENDQUERY | CURLU_URLENCODE); - curl_url_set(cu, CURLUPART_QUERY, - "format_tsv_null_representation=\\N", - CURLU_APPENDQUERY | CURLU_URLENCODE); - curl_url_set(cu, CURLUPART_QUERY, - "output_format_tsv_crlf_end_of_line=0", - CURLU_APPENDQUERY | CURLU_URLENCODE); - curl_url_get(cu, CURLUPART_URL, &stream->url, 0); - curl_url_cleanup(cu); - - /* Configure CURL easy handle */ - curl_easy_setopt(stream->curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(stream->curl, CURLOPT_WRITEDATA, stream); - curl_easy_setopt(stream->curl, CURLOPT_ERRORBUFFER, stream->error_buffer); - curl_easy_setopt(stream->curl, CURLOPT_PATH_AS_IS, 1L); - curl_easy_setopt(stream->curl, CURLOPT_URL, stream->url); - curl_easy_setopt(stream->curl, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(stream->curl, CURLOPT_VERBOSE, ch_http_get_verbose()); - - if (stream->conn->ssl_version != CURL_SSLVERSION_DEFAULT) - curl_easy_setopt(stream->curl, CURLOPT_SSLVERSION, - stream->conn->ssl_version); - - if (ch_http_get_progress_func()) - { - curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(stream->curl, CURLOPT_XFERINFOFUNCTION, - ch_http_get_progress_func()); - curl_easy_setopt(stream->curl, CURLOPT_XFERINFODATA, stream->conn); - } - else - curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 1L); - - if (stream->conn->dbname) - { - snprintf(temp_buf, sizeof(temp_buf), "%s: %s", - DATABASE_HEADER, stream->conn->dbname); - stream->headers = curl_slist_append(NULL, temp_buf); - curl_easy_setopt(stream->curl, CURLOPT_HTTPHEADER, stream->headers); - } - - /* POST body or MIME form */ - if (query->num_params == 0) - { - curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->sql); - } - else - { - curl_mimepart *part; - - stream->form = curl_mime_init(stream->curl); - part = curl_mime_addpart(stream->form); - curl_mime_name(part, "query"); - curl_mime_data(part, query->sql, CURL_ZERO_TERMINATED); - - for (int i = 0; i < query->num_params; i++) - { - part = curl_mime_addpart(stream->form); - snprintf(temp_buf, sizeof(temp_buf), "param_p%d", i + 1); - curl_mime_name(part, temp_buf); - curl_mime_data(part, query->param_values[i], CURL_ZERO_TERMINATED); - } - curl_easy_setopt(stream->curl, CURLOPT_MIMEPOST, stream->form); - } +setup_curl(HttpStream* stream, const ch_query* query) { + CURLU* cu = curl_url(); + char temp_buf[512]; + + /* Build URL with query_id and settings */ + curl_url_set(cu, CURLUPART_URL, stream->conn->base_url, 0); + + snprintf(temp_buf, sizeof(temp_buf), "query_id=%s", stream->query_id); + curl_url_set(cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE); + + for (kv_iter iter = new_kv_iter(query->settings); !kv_iter_done(&iter); + kv_iter_next(&iter)) { + if (strcmp(iter.name, "date_time_output_format") == 0 || + strcmp(iter.name, "format_tsv_null_representation") == 0 || + strcmp(iter.name, "output_format_tsv_crlf_end_of_line") == 0) { + continue; + } + snprintf(temp_buf, sizeof(temp_buf), "%s=%s", iter.name, iter.value); + curl_url_set( + cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE + ); + } + + curl_url_set( + cu, + CURLUPART_QUERY, + "date_time_output_format=iso", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + curl_url_set( + cu, + CURLUPART_QUERY, + "format_tsv_null_representation=\\N", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + curl_url_set( + cu, + CURLUPART_QUERY, + "output_format_tsv_crlf_end_of_line=0", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + curl_url_get(cu, CURLUPART_URL, &stream->url, 0); + curl_url_cleanup(cu); + + /* Configure CURL easy handle */ + curl_easy_setopt(stream->curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(stream->curl, CURLOPT_WRITEDATA, stream); + curl_easy_setopt(stream->curl, CURLOPT_ERRORBUFFER, stream->error_buffer); + curl_easy_setopt(stream->curl, CURLOPT_PATH_AS_IS, 1L); + curl_easy_setopt(stream->curl, CURLOPT_URL, stream->url); + curl_easy_setopt(stream->curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(stream->curl, CURLOPT_VERBOSE, ch_http_get_verbose()); + + if (stream->conn->ssl_version != CURL_SSLVERSION_DEFAULT) { + curl_easy_setopt(stream->curl, CURLOPT_SSLVERSION, stream->conn->ssl_version); + } + + if (ch_http_get_progress_func()) { + curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt( + stream->curl, CURLOPT_XFERINFOFUNCTION, ch_http_get_progress_func() + ); + curl_easy_setopt(stream->curl, CURLOPT_XFERINFODATA, stream->conn); + } else { + curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 1L); + } + + if (stream->conn->dbname) { + snprintf( + temp_buf, sizeof(temp_buf), "%s: %s", DATABASE_HEADER, stream->conn->dbname + ); + stream->headers = curl_slist_append(NULL, temp_buf); + curl_easy_setopt(stream->curl, CURLOPT_HTTPHEADER, stream->headers); + } + + /* POST body or MIME form */ + if (query->num_params == 0) { + curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->sql); + } else { + curl_mimepart* part; + + stream->form = curl_mime_init(stream->curl); + part = curl_mime_addpart(stream->form); + curl_mime_name(part, "query"); + curl_mime_data(part, query->sql, CURL_ZERO_TERMINATED); + + for (int i = 0; i < query->num_params; i++) { + part = curl_mime_addpart(stream->form); + snprintf(temp_buf, sizeof(temp_buf), "param_p%d", i + 1); + curl_mime_name(part, temp_buf); + curl_mime_data(part, query->param_values[i], CURL_ZERO_TERMINATED); + } + curl_easy_setopt(stream->curl, CURLOPT_MIMEPOST, stream->form); + } } /* ---------------------------------------------------------------- @@ -182,48 +189,47 @@ setup_curl(HttpStream * stream, const ch_query * query) * ---------------------------------------------------------------- */ static size_t -write_callback(void *contents, size_t size, size_t nmemb, void *userp) -{ - size_t realsize = size * nmemb; - HttpStream *self = (HttpStream *) userp; - size_t needed = self->write_pos + realsize + 1; - - /* Grow buffer if needed */ - if (needed > self->buf_allocated) - { - size_t newsize = self->buf_allocated * 2; - char *newbuf; - - if (newsize < needed) - newsize = needed; - - newbuf = (char *) realloc(self->buf, newsize); - if (!newbuf) - return CURL_WRITEFUNC_ERROR; - - self->buf = newbuf; - self->buf_allocated = newsize; - } - - memcpy(self->buf + self->write_pos, contents, realsize); - self->write_pos += realsize; - self->buf[self->write_pos] = '\0'; - - /* - * Once we have buffered at least fetch_size bytes AND at least one - * newline (so a row-aligned batch is ready), pause receipt. Pausing on - * byte-count alone can starve the parser of the newline it needs when - * fetch_size is small. We accept this chunk first and pause afterward via - * curl_easy_pause so CURL does not redeliver bytes we already hold. - */ - if (self->write_pos >= (size_t) self->fetch_size - && memchr(self->buf, '\n', self->write_pos) != NULL) - { - self->paused = true; - curl_easy_pause(self->curl, CURLPAUSE_RECV); - } - - return realsize; +write_callback(void* contents, size_t size, size_t nmemb, void* userp) { + size_t realsize = size * nmemb; + HttpStream* self = (HttpStream*)userp; + size_t needed = self->write_pos + realsize + 1; + + /* Grow buffer if needed */ + if (needed > self->buf_allocated) { + size_t newsize = self->buf_allocated * 2; + char* newbuf; + + if (newsize < needed) { + newsize = needed; + } + + newbuf = (char*)realloc(self->buf, newsize); + if (!newbuf) { + return CURL_WRITEFUNC_ERROR; + } + + self->buf = newbuf; + self->buf_allocated = newsize; + } + + memcpy(self->buf + self->write_pos, contents, realsize); + self->write_pos += realsize; + self->buf[self->write_pos] = '\0'; + + /* + * Once we have buffered at least fetch_size bytes AND at least one + * newline (so a row-aligned batch is ready), pause receipt. Pausing on + * byte-count alone can starve the parser of the newline it needs when + * fetch_size is small. We accept this chunk first and pause afterward via + * curl_easy_pause so CURL does not redeliver bytes we already hold. + */ + if (self->write_pos >= (size_t)self->fetch_size && + memchr(self->buf, '\n', self->write_pos) != NULL) { + self->paused = true; + curl_easy_pause(self->curl, CURLPAUSE_RECV); + } + + return realsize; } /* ---------------------------------------------------------------- @@ -231,15 +237,13 @@ write_callback(void *contents, size_t size, size_t nmemb, void *userp) * ---------------------------------------------------------------- */ static void -capture_transfer_info(HttpStream * stream) -{ - curl_easy_getinfo(stream->curl, CURLINFO_RESPONSE_CODE, - &stream->http_status); - curl_easy_getinfo(stream->curl, CURLINFO_PRETRANSFER_TIME, - &stream->pretransfer_time); - curl_easy_getinfo(stream->curl, CURLINFO_TOTAL_TIME, - &stream->total_time); - stream->started = true; +capture_transfer_info(HttpStream* stream) { + curl_easy_getinfo(stream->curl, CURLINFO_RESPONSE_CODE, &stream->http_status); + curl_easy_getinfo( + stream->curl, CURLINFO_PRETRANSFER_TIME, &stream->pretransfer_time + ); + curl_easy_getinfo(stream->curl, CURLINFO_TOTAL_TIME, &stream->total_time); + stream->started = true; } /* ---------------------------------------------------------------- @@ -250,33 +254,36 @@ capture_transfer_info(HttpStream * stream) * ---------------------------------------------------------------- */ static size_t -find_batch_end(const HttpStream * stream) -{ - const char *base = stream->buf; - const char *found; - - if (stream->fetch_size <= 0) - return stream->write_pos; - - if (stream->write_pos >= (size_t) stream->fetch_size) - { - /* Look forward first */ - found = memchr(base + stream->fetch_size, - '\n', - stream->write_pos - stream->fetch_size); - if (found) - return (found - base) + 1; - - /* Look backward */ - for (size_t i = stream->fetch_size; i > 0; i--) - if (base[i - 1] == '\n') - return i; - } - - if (stream->transfer_done) - return stream->write_pos; - - return 0; +find_batch_end(const HttpStream* stream) { + const char* base = stream->buf; + const char* found; + + if (stream->fetch_size <= 0) { + return stream->write_pos; + } + + if (stream->write_pos >= (size_t)stream->fetch_size) { + /* Look forward first */ + found = memchr( + base + stream->fetch_size, '\n', stream->write_pos - stream->fetch_size + ); + if (found) { + return (found - base) + 1; + } + + /* Look backward */ + for (size_t i = stream->fetch_size; i > 0; i--) { + if (base[i - 1] == '\n') { + return i; + } + } + } + + if (stream->transfer_done) { + return stream->write_pos; + } + + return 0; } /* ---------------------------------------------------------------- @@ -284,18 +291,16 @@ find_batch_end(const HttpStream * stream) * ---------------------------------------------------------------- */ static void -compact_buffer(HttpStream * stream) -{ - if (stream->parse_pos > 0) - { - size_t remaining = stream->write_pos - stream->parse_pos; - - memmove(stream->buf, stream->buf + stream->parse_pos, remaining); - stream->write_pos = remaining; - stream->parse_pos = 0; - stream->batch_end = 0; - stream->buf[stream->write_pos] = '\0'; - } +compact_buffer(HttpStream* stream) { + if (stream->parse_pos > 0) { + size_t remaining = stream->write_pos - stream->parse_pos; + + memmove(stream->buf, stream->buf + stream->parse_pos, remaining); + stream->write_pos = remaining; + stream->parse_pos = 0; + stream->batch_end = 0; + stream->buf[stream->write_pos] = '\0'; + } } /* ---------------------------------------------------------------- @@ -304,82 +309,79 @@ compact_buffer(HttpStream * stream) * ---------------------------------------------------------------- */ int -ch_http_stream_pump(HttpStream * stream) -{ - int running_handles; - CURLMcode mc; - CURLMsg *msg; - int msgs_left; - - /* - * Drop the already-consumed batch and see if there is enough buffered - * data for the next one before touching the network again. - */ - if (stream->parse_pos > 0) - compact_buffer(stream); - - stream->batch_end = find_batch_end(stream); - if (stream->batch_end > 0 || - (stream->transfer_done && stream->write_pos <= stream->parse_pos)) - { - if (!stream->started) - capture_transfer_info(stream); - return 0; - } - - /* Resume if paused from a previous batch */ - if (stream->paused) - { - stream->paused = false; - curl_easy_pause(stream->curl, CURLPAUSE_CONT); - } - - /* Drive the transfer */ - for (;;) - { - mc = curl_multi_perform(stream->multi, &running_handles); - if (mc != CURLM_OK) - { - stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; - free(stream->error_msg); - stream->error_msg = strdup(curl_multi_strerror(mc)); - return -1; - } - - if (running_handles == 0) - stream->transfer_done = true; - - stream->batch_end = find_batch_end(stream); - if (stream->batch_end > 0 || stream->paused || stream->transfer_done) - break; - - curl_multi_wait(stream->multi, NULL, 0, 100, NULL); - } - - capture_transfer_info(stream); - stream->batch_end = find_batch_end(stream); - - /* Check for transfer errors */ - while ((msg = curl_multi_info_read(stream->multi, &msgs_left))) - { - if (msg->msg == CURLMSG_DONE && msg->data.result != CURLE_OK) - { - if (msg->data.result == CURLE_ABORTED_BY_CALLBACK) - stream->http_status = CH_HTTP_STATUS_CANCELED; - else - { - stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; - free(stream->error_msg); - stream->error_msg = - strdup(stream->error_buffer[0] != '\0' - ? stream->error_buffer - : curl_easy_strerror(msg->data.result)); - } - return -1; - } - } - - return 0; +ch_http_stream_pump(HttpStream* stream) { + int running_handles; + CURLMcode mc; + CURLMsg* msg; + int msgs_left; + + /* + * Drop the already-consumed batch and see if there is enough buffered + * data for the next one before touching the network again. + */ + if (stream->parse_pos > 0) { + compact_buffer(stream); + } + + stream->batch_end = find_batch_end(stream); + if (stream->batch_end > 0 || + (stream->transfer_done && stream->write_pos <= stream->parse_pos)) { + if (!stream->started) { + capture_transfer_info(stream); + } + return 0; + } + + /* Resume if paused from a previous batch */ + if (stream->paused) { + stream->paused = false; + curl_easy_pause(stream->curl, CURLPAUSE_CONT); + } + + /* Drive the transfer */ + for (;;) { + mc = curl_multi_perform(stream->multi, &running_handles); + if (mc != CURLM_OK) { + stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; + free(stream->error_msg); + stream->error_msg = strdup(curl_multi_strerror(mc)); + return -1; + } + + if (running_handles == 0) { + stream->transfer_done = true; + } + + stream->batch_end = find_batch_end(stream); + if (stream->batch_end > 0 || stream->paused || stream->transfer_done) { + break; + } + + curl_multi_wait(stream->multi, NULL, 0, 100, NULL); + } + + capture_transfer_info(stream); + stream->batch_end = find_batch_end(stream); + + /* Check for transfer errors */ + while ((msg = curl_multi_info_read(stream->multi, &msgs_left))) { + if (msg->msg == CURLMSG_DONE && msg->data.result != CURLE_OK) { + if (msg->data.result == CURLE_ABORTED_BY_CALLBACK) { + stream->http_status = CH_HTTP_STATUS_CANCELED; + } else { + stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; + free(stream->error_msg); + stream->error_msg = strdup( + stream->error_buffer[0] != '\0' + ? stream->error_buffer + : curl_easy_strerror(msg->data.result) + ); + } + return -1; + } + } + + return 0; } /* ---------------------------------------------------------------- @@ -391,152 +393,154 @@ ch_http_stream_pump(HttpStream * stream) * ch_http_stream_begin — allocate and initialize a streaming HTTP query. * Returns NULL on failure. */ -HttpStream * -ch_http_stream_begin(ch_http_connection_t * conn, const ch_query * query, - int32 fetch_size) -{ - HttpStream *stream; - uuid_t id; - - stream = calloc(1, sizeof(HttpStream)); - if (!stream) - return NULL; - - stream->conn = conn; - stream->fetch_size = fetch_size; - - /* Generate query ID */ - uuid_generate(id); - uuid_unparse(id, stream->query_id); - - /* - * Create our own CURL easy handle so that multiple HttpStream instances - * (e.g. concurrent foreign scans in subqueries or joins) do not fight - * over the single handle in conn->curl. - */ - stream->curl = curl_easy_init(); - if (!stream->curl) - goto fail; - - /* Allocate stream buffer */ - stream->buf = (char *) malloc(INITIAL_BUF_SIZE); - if (!stream->buf) - goto fail; - stream->buf_allocated = INITIAL_BUF_SIZE; - stream->buf[0] = '\0'; - - setup_curl(stream, query); - - /* Create multi handle and kick off the transfer */ - stream->multi = curl_multi_init(); - if (!stream->multi) - goto fail; - curl_multi_add_handle(stream->multi, stream->curl); - - /* Pump until first batch is ready or transfer completes */ - ch_http_stream_pump(stream); - - return stream; +HttpStream* +ch_http_stream_begin( + ch_http_connection_t* conn, + const ch_query* query, + int32 fetch_size +) { + HttpStream* stream; + uuid_t id; + + stream = calloc(1, sizeof(HttpStream)); + if (!stream) { + return NULL; + } + + stream->conn = conn; + stream->fetch_size = fetch_size; + + /* Generate query ID */ + uuid_generate(id); + uuid_unparse(id, stream->query_id); + + /* + * Create our own CURL easy handle so that multiple HttpStream instances + * (e.g. concurrent foreign scans in subqueries or joins) do not fight + * over the single handle in conn->curl. + */ + stream->curl = curl_easy_init(); + if (!stream->curl) { + goto fail; + } + + /* Allocate stream buffer */ + stream->buf = (char*)malloc(INITIAL_BUF_SIZE); + if (!stream->buf) { + goto fail; + } + stream->buf_allocated = INITIAL_BUF_SIZE; + stream->buf[0] = '\0'; + + setup_curl(stream, query); + + /* Create multi handle and kick off the transfer */ + stream->multi = curl_multi_init(); + if (!stream->multi) { + goto fail; + } + curl_multi_add_handle(stream->multi, stream->curl); + + /* Pump until first batch is ready or transfer completes */ + ch_http_stream_pump(stream); + + return stream; fail: - ch_http_stream_end(stream); - return NULL; + ch_http_stream_end(stream); + return NULL; } /* * ch_http_stream_end — clean up all owned resources. */ void -ch_http_stream_end(HttpStream * stream) -{ - if (!stream) - return; - - if (stream->multi) - { - if (stream->curl) - curl_multi_remove_handle(stream->multi, stream->curl); - curl_multi_cleanup(stream->multi); - } - - if (stream->curl) - curl_easy_cleanup(stream->curl); - - if (stream->headers) - curl_slist_free_all(stream->headers); - if (stream->form) - curl_mime_free(stream->form); - if (stream->url) - curl_free(stream->url); - if (stream->buf) - free(stream->buf); - if (stream->error_msg) - free(stream->error_msg); - - free(stream); +ch_http_stream_end(HttpStream* stream) { + if (!stream) { + return; + } + + if (stream->multi) { + if (stream->curl) { + curl_multi_remove_handle(stream->multi, stream->curl); + } + curl_multi_cleanup(stream->multi); + } + + if (stream->curl) { + curl_easy_cleanup(stream->curl); + } + + if (stream->headers) { + curl_slist_free_all(stream->headers); + } + if (stream->form) { + curl_mime_free(stream->form); + } + if (stream->url) { + curl_free(stream->url); + } + if (stream->buf) { + free(stream->buf); + } + if (stream->error_msg) { + free(stream->error_msg); + } + + free(stream); } /* ---------------------------------------------------------------- * Public API — accessors * ---------------------------------------------------------------- */ -char * -ch_http_stream_buffer(HttpStream * stream) -{ - return stream->buf + stream->parse_pos; +char* +ch_http_stream_buffer(HttpStream* stream) { + return stream->buf + stream->parse_pos; } size_t -ch_http_stream_available(HttpStream * stream) -{ - return stream->batch_end > stream->parse_pos - ? stream->batch_end - stream->parse_pos - : 0; +ch_http_stream_available(HttpStream* stream) { + return stream->batch_end > stream->parse_pos ? stream->batch_end - stream->parse_pos + : 0; } void -ch_http_stream_advance(HttpStream * stream, size_t n) -{ - stream->parse_pos += n; - if (stream->parse_pos > stream->batch_end) - stream->parse_pos = stream->batch_end; +ch_http_stream_advance(HttpStream* stream, size_t n) { + stream->parse_pos += n; + if (stream->parse_pos > stream->batch_end) { + stream->parse_pos = stream->batch_end; + } } bool -ch_http_stream_transfer_done(HttpStream * stream) -{ - return stream->transfer_done - && (stream->write_pos <= stream->parse_pos); +ch_http_stream_transfer_done(HttpStream* stream) { + return stream->transfer_done && (stream->write_pos <= stream->parse_pos); } long -ch_http_stream_status(HttpStream * stream) -{ - return stream->http_status; +ch_http_stream_status(HttpStream* stream) { + return stream->http_status; } -const char * -ch_http_stream_query_id(HttpStream * stream) -{ - return stream->query_id; +const char* +ch_http_stream_query_id(HttpStream* stream) { + return stream->query_id; } -const char * -ch_http_stream_error(HttpStream * stream) -{ - return stream->error_msg; +const char* +ch_http_stream_error(HttpStream* stream) { + return stream->error_msg; } double -ch_http_stream_request_time(HttpStream * stream) -{ - return stream->pretransfer_time * 1000; +ch_http_stream_request_time(HttpStream* stream) { + return stream->pretransfer_time * 1000; } double -ch_http_stream_total_time(HttpStream * stream) -{ - return stream->total_time * 1000; +ch_http_stream_total_time(HttpStream* stream) { + return stream->total_time * 1000; } /* @@ -551,31 +555,29 @@ ch_http_stream_total_time(HttpStream * stream) * itself should still be released with ch_http_stream_end(). */ void -ch_http_stream_take_body(HttpStream * stream, char **out_data, size_t * out_size) -{ - size_t avail; - - if (stream->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR && stream->error_msg) - { - *out_data = stream->error_msg; - *out_size = strlen(stream->error_msg); - stream->error_msg = NULL; - return; - } - - avail = ch_http_stream_available(stream); - if (avail == 0 || !stream->buf) - { - *out_data = NULL; - *out_size = 0; - return; - } - - if (stream->parse_pos > 0) - memmove(stream->buf, stream->buf + stream->parse_pos, avail); - stream->buf[avail] = '\0'; - - *out_data = stream->buf; - *out_size = avail; - stream->buf = NULL; +ch_http_stream_take_body(HttpStream* stream, char** out_data, size_t* out_size) { + size_t avail; + + if (stream->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR && stream->error_msg) { + *out_data = stream->error_msg; + *out_size = strlen(stream->error_msg); + stream->error_msg = NULL; + return; + } + + avail = ch_http_stream_available(stream); + if (avail == 0 || !stream->buf) { + *out_data = NULL; + *out_size = 0; + return; + } + + if (stream->parse_pos > 0) { + memmove(stream->buf, stream->buf + stream->parse_pos, avail); + } + stream->buf[avail] = '\0'; + + *out_data = stream->buf; + *out_size = avail; + stream->buf = NULL; } diff --git a/src/include/binary.h b/src/include/binary.h index 3ca9b7b..a95e3fc 100644 --- a/src/include/binary.h +++ b/src/include/binary.h @@ -12,9 +12,9 @@ #include "postgres.h" +#include #include #include -#include #include "access/tupdesc.h" #include "executor/tuptable.h" @@ -28,24 +28,34 @@ typedef struct ch_binary_response_t ch_binary_response_t; typedef struct ch_binary_insert_handle ch_binary_insert_handle; /* Connection. */ -extern ch_binary_connection_t * ch_binary_connect(ch_connection_details * details); -extern void ch_binary_close(ch_binary_connection_t * conn); +extern ch_binary_connection_t* +ch_binary_connect(ch_connection_details* details); +extern void +ch_binary_close(ch_binary_connection_t* conn); /* * Returns true if connection encountered an unrecoverable error * (server exception, IO failure, mid-protocol break). Callers should * drop cached connection instead of reusing it. */ -extern bool ch_binary_is_broken(const ch_binary_connection_t * conn); +extern bool +ch_binary_is_broken(const ch_binary_connection_t* conn); /* SELECT. */ -extern ch_binary_response_t * ch_binary_simple_query(ch_binary_connection_t * conn, - const ch_query * query, - bool (*check_cancel) (void)); -extern void ch_binary_response_free(ch_binary_response_t * resp); -extern const char *ch_binary_response_error(const ch_binary_response_t * resp); -extern bool ch_binary_response_success(const ch_binary_response_t * resp); -extern size_t ch_binary_response_columns(const ch_binary_response_t * resp); +extern ch_binary_response_t* +ch_binary_simple_query( + ch_binary_connection_t* conn, + const ch_query* query, + bool (*check_cancel)(void) +); +extern void +ch_binary_response_free(ch_binary_response_t* resp); +extern const char* +ch_binary_response_error(const ch_binary_response_t* resp); +extern bool +ch_binary_response_success(const ch_binary_response_t* resp); +extern size_t +ch_binary_response_columns(const ch_binary_response_t* resp); /* INSERT. */ @@ -54,63 +64,76 @@ extern size_t ch_binary_response_columns(const ch_binary_response_t * resp); * if the server raised. Idempotent; call exactly once from the FDW happy * path before tearing down the handle. */ -extern void ch_binary_finalize_insert(ch_binary_insert_handle * h); +extern void +ch_binary_finalize_insert(ch_binary_insert_handle* h); /* PG-typed surface follows. */ -typedef struct -{ - ch_binary_response_t *resp; - Oid *coltypes; - Datum *values; - bool *nulls; - - size_t block; /* current block */ - size_t row; /* row in current block */ - const chc_block *cur; /* borrowed from resp; NULL when unloaded */ - void *gc; /* allocated objects while reading */ - char *error; - bool done; -} ch_binary_read_state_t; - -typedef struct -{ - MemoryContext memcxt; /* used for cleanup */ - MemoryContextCallback callback; - - TupleDesc outdesc; - ch_binary_insert_handle *insert_block; - size_t len; - void *conversion_states; - char *table_name; - Oid relid; /* foreign table relid, for column_name - * lookups */ - - Datum *values; - bool *nulls; - bool success; - - ch_binary_connection_t *conn; -} ch_binary_insert_state; +typedef struct { + ch_binary_response_t* resp; + Oid* coltypes; + Datum* values; + bool* nulls; + + size_t block; /* current block */ + size_t row; /* row in current block */ + const chc_block* cur; /* borrowed from resp; NULL when unloaded */ + void* gc; /* allocated objects while reading */ + char* error; + bool done; +} ch_binary_read_state_t; + +typedef struct { + MemoryContext memcxt; /* used for cleanup */ + MemoryContextCallback callback; + + TupleDesc outdesc; + ch_binary_insert_handle* insert_block; + size_t len; + void* conversion_states; + char* table_name; + Oid relid; /* foreign table relid, for column_name + * lookups */ + + Datum* values; + bool* nulls; + bool success; + + ch_binary_connection_t* conn; +} ch_binary_insert_state; /* SELECT helpers (decode.c). */ -extern void ch_binary_read_state_init(ch_binary_read_state_t * state, ch_binary_response_t * resp); -extern void ch_binary_read_state_free(ch_binary_read_state_t * state); -extern bool ch_binary_read_row(ch_binary_read_state_t * state); +extern void +ch_binary_read_state_init(ch_binary_read_state_t* state, ch_binary_response_t* resp); +extern void +ch_binary_read_state_free(ch_binary_read_state_t* state); +extern bool +ch_binary_read_row(ch_binary_read_state_t* state); /* SELECT/INSERT type conversion (convert.c). */ -extern Datum ch_binary_convert_datum(void *state, Datum val); -extern void *ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype); -extern void ch_binary_free_convert_state(void *state); +extern Datum +ch_binary_convert_datum(void* state, Datum val); +extern void* +ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype); +extern void +ch_binary_free_convert_state(void* state); /* INSERT helpers (encode.c). */ -extern void ch_binary_prepare_insert(void *conn, const ch_query * query, - ch_binary_insert_state * state); -extern void ch_binary_insert_columns(ch_binary_insert_state * state); -extern void ch_binary_column_append_data(ch_binary_insert_state * state, size_t colidx); -extern void *ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid); -extern void ch_binary_insert_state_free(void *c); -extern void ch_binary_do_output_conversion(ch_binary_insert_state * state, - TupleTableSlot * slot); - -#endif /* CLICKHOUSE_BINARY_H */ +extern void +ch_binary_prepare_insert( + void* conn, + const ch_query* query, + ch_binary_insert_state* state +); +extern void +ch_binary_insert_columns(ch_binary_insert_state* state); +extern void +ch_binary_column_append_data(ch_binary_insert_state* state, size_t colidx); +extern void* +ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid); +extern void +ch_binary_insert_state_free(void* c); +extern void +ch_binary_do_output_conversion(ch_binary_insert_state* state, TupleTableSlot* slot); + +#endif /* CLICKHOUSE_BINARY_H */ diff --git a/src/include/engine.h b/src/include/engine.h index c7d8bf7..da7d9b9 100644 --- a/src/include/engine.h +++ b/src/include/engine.h @@ -1,62 +1,59 @@ #ifndef CLICKHOUSE_ENGINE_H #define CLICKHOUSE_ENGINE_H -#include "kv_list.h" #include "access/tupdesc.h" +#include "kv_list.h" /* * ch_connection_details defines the details for connecting to ClickHouse. */ /* TLS mode for the "secure" FDW option (auto=heuristic, on=force, off=never) */ -typedef enum -{ - CH_TLS_AUTO = 0, /* cloud-hostname heuristic (default) */ - CH_TLS_ON, /* force TLS; default port 8443 or 9440 */ - CH_TLS_OFF, /* force plaintext; default port 8123 or 9000 */ -} tls_mode; +typedef enum { + CH_TLS_AUTO = 0, /* cloud-hostname heuristic (default) */ + CH_TLS_ON, /* force TLS; default port 8443 or 9440 */ + CH_TLS_OFF, /* force plaintext; default port 8123 or 9000 */ +} tls_mode; /* Minimum TLS protocol version for the "min_tls_version" FDW option */ -typedef enum -{ - CH_TLS_DEFAULT = 0, /* library default; no minimum forced */ - CH_TLS_V1_0, - CH_TLS_V1_1, - CH_TLS_V1_2, - CH_TLS_V1_3, -} tls_version; +typedef enum { + CH_TLS_DEFAULT = 0, /* library default; no minimum forced */ + CH_TLS_V1_0, + CH_TLS_V1_1, + CH_TLS_V1_2, + CH_TLS_V1_3, +} tls_version; -typedef struct -{ - char *host; - int port; - char *username; - char *password; - char *dbname; - char *compression; - tls_mode tls; /* TLS mode; CH_TLS_AUTO when not specified */ - tls_version min_tls_version; /* minimum TLS version; CH_TLS_DEFAULT - * when not specified */ -} ch_connection_details; +typedef struct { + char* host; + int port; + char* username; + char* password; + char* dbname; + char* compression; + tls_mode tls; /* TLS mode; CH_TLS_AUTO when not specified */ + tls_version min_tls_version; /* minimum TLS version; CH_TLS_DEFAULT + * when not specified */ +} ch_connection_details; /* * ch_query an SQL query to execute on ClickHouse. */ -typedef struct -{ - /* The SQL query. */ - const char *sql; - /* The number of parameters in the query. */ - const int num_params; - /* The list of parameters to pass when executing the query. */ - const char **param_values; - /* A description of the Tuple for the query. */ - const TupleDesc tupdesc; - /* The numbers of the attributes in tupdesc that the query selects. */ - const List *attr_nums; - /* List of settings to pass to ClickHouse upon execution. */ - const kv_list *settings; -} ch_query; +typedef struct { + /* The SQL query. */ + const char* sql; + /* The number of parameters in the query. */ + const int num_params; + /* The list of parameters to pass when executing the query. */ + const char** param_values; + /* A description of the Tuple for the query. */ + const TupleDesc tupdesc; + /* The numbers of the attributes in tupdesc that the query selects. */ + const List* attr_nums; + /* List of settings to pass to ClickHouse upon execution. */ + const kv_list* settings; +} ch_query; -#define new_query(sql, num, vals, tupdesc, attrs) {sql, num, vals, tupdesc, attrs, chfdw_get_session_settings()} +#define new_query(sql, num, vals, tupdesc, attrs) \ + { sql, num, vals, tupdesc, attrs, chfdw_get_session_settings() } -#endif /* CLICKHOUSE_ENGINE_H */ +#endif /* CLICKHOUSE_ENGINE_H */ diff --git a/src/include/fdw.h b/src/include/fdw.h index d5a8f7a..7ff4ff7 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -16,16 +16,16 @@ #ifndef CLICKHOUSE_FDW_H #define CLICKHOUSE_FDW_H +#include "access/heapam.h" +#include "catalog/pg_operator.h" +#include "engine.h" #include "foreign/foreign.h" +#include "funcapi.h" #include "lib/stringinfo.h" -#include "utils/relcache.h" -#include "catalog/pg_operator.h" #include "nodes/execnodes.h" -#include "optimizer/optimizer.h" #include "nodes/pathnodes.h" -#include "access/heapam.h" -#include "engine.h" -#include "funcapi.h" +#include "optimizer/optimizer.h" +#include "utils/relcache.h" #if PG_VERSION_NUM < 150000 #define FirstUnpinnedObjectId FirstBootstrapObjectId @@ -37,86 +37,96 @@ * there are a bunch of wrinkles, and names in Postgres can't be more than * NAMEDATALEN (currently 64), so just make enough room for every character to * be escaped plus opening and closing quotation marks. -*/ -#define CH_ESCAPED_NAMEDATALEN NAMEDATALEN*2 + */ +#define CH_ESCAPED_NAMEDATALEN NAMEDATALEN * 2 /* pglink.c */ typedef struct ch_cursor ch_cursor; -typedef struct ch_cursor -{ - MemoryContext memcxt; /* used for cleanup */ - MemoryContextCallback callback; - - void *query_response; - void *read_state; - void *conn; - char *query; - double request_time; - double total_time; - size_t columns_count; - uintptr_t *conversion_states; /* for binary */ -} ch_cursor; - -typedef struct ChFdwScanRowContext -{ - TupleDesc tupdesc; /* tuple descriptor for row */ - List *retrieved_attrs; /* list of retrieved attribute numbers */ - AttInMetadata *attinmeta; /* list of incoming attributes */ - ch_cursor *cursor; /* result of query from clickhouse */ - Datum *values; /* collected values for each column */ - bool *nulls; /* indicates null columns */ -} ChFdwScanRowContext; - -typedef void (*disconnect_method) (void *conn); -typedef void (*check_conn_method) (const char *password, UserMapping * user); -typedef ch_cursor * (*simple_query_method) (void *conn, const ch_query * query); -typedef void (*simple_insert_method) (void *conn, const ch_query * query); -typedef Datum * (*cursor_fetch_row_method) (ChFdwScanRowContext * ctx); -typedef void *(*prepare_insert_method) (void *conn, ResultRelInfo *, List *, - const ch_query *, char *); -typedef void (*insert_tuple_method) (void *state, TupleTableSlot * slot); -typedef void (*finalize_insert_method) (void *state); -typedef ch_cursor * (*streaming_query_method) (void *conn, - const ch_query * query, - int32 fetch_size); -typedef bool (*is_broken_method) (const void *conn); - -typedef struct -{ - disconnect_method disconnect; - simple_query_method simple_query; - cursor_fetch_row_method fetch_row; - prepare_insert_method prepare_insert; - insert_tuple_method insert_tuple; - finalize_insert_method finalize_insert; /* NULL if no explicit finalize - * step needed */ - streaming_query_method streaming_query; /* NULL if not supported */ - cursor_fetch_row_method streaming_fetch_row; /* NULL if not supported */ - is_broken_method is_broken; /* NULL means connection is never broken */ -} libclickhouse_methods; - -typedef struct -{ - libclickhouse_methods *methods; - void *conn; - bool is_binary; -} ch_connection; - -ch_connection_details *connstring_parse(const char *connstring); -ch_connection chfdw_http_connect(ch_connection_details * details); -ch_connection chfdw_binary_connect(ch_connection_details * details); -text *chfdw_http_fetch_raw_data(ch_cursor * cursor); -List *chfdw_construct_create_tables(ImportForeignSchemaStmt * stmt, ForeignServer * server); -char *ch_quote_literal(const char *rawstr); -char *chfdw_datum_to_ch_literal(Datum value, Oid type); -const char *ch_quote_ident(const char *rawstr); - -typedef enum -{ - CH_DEFAULT, - CH_COLLAPSING_MERGE_TREE, - CH_AGGREGATING_MERGE_TREE -} CHRemoteTableEngine; +typedef struct ch_cursor { + MemoryContext memcxt; /* used for cleanup */ + MemoryContextCallback callback; + + void* query_response; + void* read_state; + void* conn; + char* query; + double request_time; + double total_time; + size_t columns_count; + uintptr_t* conversion_states; /* for binary */ +} ch_cursor; + +typedef struct ChFdwScanRowContext { + TupleDesc tupdesc; /* tuple descriptor for row */ + List* retrieved_attrs; /* list of retrieved attribute numbers */ + AttInMetadata* attinmeta; /* list of incoming attributes */ + ch_cursor* cursor; /* result of query from clickhouse */ + Datum* values; /* collected values for each column */ + bool* nulls; /* indicates null columns */ +} ChFdwScanRowContext; + +typedef void (*disconnect_method)(void* conn); +typedef void (*check_conn_method)(const char* password, UserMapping* user); +typedef ch_cursor* (*simple_query_method)(void* conn, const ch_query* query); +typedef void (*simple_insert_method)(void* conn, const ch_query* query); +typedef Datum* (*cursor_fetch_row_method)(ChFdwScanRowContext* ctx); +typedef void* (*prepare_insert_method)( + void* conn, + ResultRelInfo*, + List*, + const ch_query*, + char* +); +typedef void (*insert_tuple_method)(void* state, TupleTableSlot* slot); +typedef void (*finalize_insert_method)(void* state); +typedef ch_cursor* (*streaming_query_method)( + void* conn, + const ch_query* query, + int32 fetch_size +); +typedef bool (*is_broken_method)(const void* conn); + +typedef struct { + disconnect_method disconnect; + simple_query_method simple_query; + cursor_fetch_row_method fetch_row; + prepare_insert_method prepare_insert; + insert_tuple_method insert_tuple; + finalize_insert_method finalize_insert; /* NULL if no explicit finalize + * step needed */ + streaming_query_method streaming_query; /* NULL if not supported */ + cursor_fetch_row_method streaming_fetch_row; /* NULL if not supported */ + is_broken_method is_broken; /* NULL means connection is never broken */ +} libclickhouse_methods; + +typedef struct { + libclickhouse_methods* methods; + void* conn; + bool is_binary; +} ch_connection; + +ch_connection_details* +connstring_parse(const char* connstring); +ch_connection +chfdw_http_connect(ch_connection_details* details); +ch_connection +chfdw_binary_connect(ch_connection_details* details); +text* +chfdw_http_fetch_raw_data(ch_cursor* cursor); +List* +chfdw_construct_create_tables(ImportForeignSchemaStmt* stmt, ForeignServer* server); +char* +ch_quote_literal(const char* rawstr); +char* +chfdw_datum_to_ch_literal(Datum value, Oid type); +const char* +ch_quote_ident(const char* rawstr); + +typedef enum { + CH_DEFAULT, + CH_COLLAPSING_MERGE_TREE, + CH_AGGREGATING_MERGE_TREE +} CHRemoteTableEngine; /* * FDW-specific planner information kept in RelOptInfo.fdw_private for a @@ -125,270 +135,315 @@ typedef enum * later. clickhouseGetForeignJoinPaths creates it for a joinrel, and * clickhouseGetForeignUpperPaths creates it for an upperrel. */ -typedef struct CHFdwRelationInfo -{ - /* - * True means that the relation can be pushed down. Always true for simple - * foreign scan. - */ - bool pushdown_safe; - - /* - * Restriction clauses, divided into safe and unsafe to pushdown subsets. - * All entries in these lists should have RestrictInfo wrappers; that - * improves efficiency of selectivity and cost estimation. - */ - List *remote_conds; - List *local_conds; - - /* Actual remote restriction clauses for scan (sans RestrictInfos) */ - List *final_remote_exprs; - - /* Bitmap of attr numbers we need to fetch from the remote server. */ - Bitmapset *attrs_used; - - /* True means that the query_pathkeys is safe to push down */ - bool qp_is_pushdown_safe; - - /* Cost and selectivity of local_conds. */ - QualCost local_conds_cost; - Selectivity local_conds_sel; - - /* Selectivity of join conditions */ - Selectivity joinclause_sel; - - /* Estimated size and cost for a scan or join. */ - double rows; - int width; - Cost startup_cost; - Cost total_cost; - - /* Costs excluding costs for transferring data from the foreign server */ - Cost rel_startup_cost; - Cost rel_total_cost; - - /* Options extracted from catalogs. */ - bool use_remote_estimate; - Cost fdw_startup_cost; - Cost fdw_tuple_cost; - List *shippable_extensions; /* OIDs of whitelisted extensions */ - - /* Cached catalog information. */ - ForeignTable *table; - ForeignServer *server; - UserMapping *user; /* only set in use_remote_estimate mode */ - - int32 fetch_size; /* fetch size for this remote table */ - - /* - * Name of the relation while EXPLAINing ForeignScan. It is used for join - * relations but is set for all relations. For join relation, the name - * indicates which foreign tables are being joined and the join type used. - */ - StringInfo relation_name; - - /* Join information */ - RelOptInfo *outerrel; - RelOptInfo *innerrel; - JoinType jointype; - - /* joinclauses contains only JOIN/ON conditions for an outer join */ - List *joinclauses; /* List of RestrictInfo */ - - /* Upper relation information */ - UpperRelationKind stage; - - /* Grouping information */ - List *grouped_tlist; - - /* Subquery information */ - bool make_outerrel_subquery; /* do we deparse outerrel as a - * subquery? */ - bool make_innerrel_subquery; /* do we deparse innerrel as a - * subquery? */ - Relids lower_subquery_rels; /* all relids appearing in lower - * subqueries */ - - /* - * Index of the relation. It is used to create an alias to a subquery - * representing the relation. - */ - int relation_index; - - /* Custom */ - CHRemoteTableEngine ch_table_engine; - - /* - * Long enough for a quoted identifier with all but two characters - * escaped. - */ - char ch_table_sign_field[CH_ESCAPED_NAMEDATALEN]; -} CHFdwRelationInfo; +typedef struct CHFdwRelationInfo { + /* + * True means that the relation can be pushed down. Always true for simple + * foreign scan. + */ + bool pushdown_safe; + + /* + * Restriction clauses, divided into safe and unsafe to pushdown subsets. + * All entries in these lists should have RestrictInfo wrappers; that + * improves efficiency of selectivity and cost estimation. + */ + List* remote_conds; + List* local_conds; + + /* Actual remote restriction clauses for scan (sans RestrictInfos) */ + List* final_remote_exprs; + + /* Bitmap of attr numbers we need to fetch from the remote server. */ + Bitmapset* attrs_used; + + /* True means that the query_pathkeys is safe to push down */ + bool qp_is_pushdown_safe; + + /* Cost and selectivity of local_conds. */ + QualCost local_conds_cost; + Selectivity local_conds_sel; + + /* Selectivity of join conditions */ + Selectivity joinclause_sel; + + /* Estimated size and cost for a scan or join. */ + double rows; + int width; + Cost startup_cost; + Cost total_cost; + + /* Costs excluding costs for transferring data from the foreign server */ + Cost rel_startup_cost; + Cost rel_total_cost; + + /* Options extracted from catalogs. */ + bool use_remote_estimate; + Cost fdw_startup_cost; + Cost fdw_tuple_cost; + List* shippable_extensions; /* OIDs of whitelisted extensions */ + + /* Cached catalog information. */ + ForeignTable* table; + ForeignServer* server; + UserMapping* user; /* only set in use_remote_estimate mode */ + + int32 fetch_size; /* fetch size for this remote table */ + + /* + * Name of the relation while EXPLAINing ForeignScan. It is used for join + * relations but is set for all relations. For join relation, the name + * indicates which foreign tables are being joined and the join type used. + */ + StringInfo relation_name; + + /* Join information */ + RelOptInfo* outerrel; + RelOptInfo* innerrel; + JoinType jointype; + + /* joinclauses contains only JOIN/ON conditions for an outer join */ + List* joinclauses; /* List of RestrictInfo */ + + /* Upper relation information */ + UpperRelationKind stage; + + /* Grouping information */ + List* grouped_tlist; + + /* Subquery information */ + bool make_outerrel_subquery; /* do we deparse outerrel as a + * subquery? */ + bool make_innerrel_subquery; /* do we deparse innerrel as a + * subquery? */ + Relids lower_subquery_rels; /* all relids appearing in lower + * subqueries */ + + /* + * Index of the relation. It is used to create an alias to a subquery + * representing the relation. + */ + int relation_index; + + /* Custom */ + CHRemoteTableEngine ch_table_engine; + + /* + * Long enough for a quoted identifier with all but two characters + * escaped. + */ + char ch_table_sign_field[CH_ESCAPED_NAMEDATALEN]; +} CHFdwRelationInfo; /* in fdw.c */ -extern ForeignServer * chfdw_get_foreign_server(Relation rel); -extern Expr * chfdw_find_em_expr_for_input_target(PlannerInfo * root, - EquivalenceClass * ec, - PathTarget * target); -extern Expr * chfdw_find_em_expr_for_rel(EquivalenceClass * ec, RelOptInfo * rel); +extern ForeignServer* +chfdw_get_foreign_server(Relation rel); +extern Expr* +chfdw_find_em_expr_for_input_target( + PlannerInfo* root, + EquivalenceClass* ec, + PathTarget* target +); +extern Expr* +chfdw_find_em_expr_for_rel(EquivalenceClass* ec, RelOptInfo* rel); /* in connection.c */ -extern ch_connection chfdw_get_connection(UserMapping * user); -extern void chfdw_exec_query(ch_connection conn, const char *query); -extern void chfdw_report_error(int elevel, ch_connection conn, - bool clear, const char *sql); +extern ch_connection +chfdw_get_connection(UserMapping* user); +extern void +chfdw_exec_query(ch_connection conn, const char* query); +extern void +chfdw_report_error(int elevel, ch_connection conn, bool clear, const char* sql); /* in option.c */ -extern kv_list * chfdw_get_session_settings(void); -bool chfdw_pushdown_regex_ok(void); +extern kv_list* +chfdw_get_session_settings(void); +bool +chfdw_pushdown_regex_ok(void); extern void - chfdw_extract_options(List * defelems, char **driver, char **host, int *port, - char **dbname, char **username, char **password, - char **compression, tls_mode * tls, - tls_version * min_tls_version); -extern List * chfdw_parse_options(const char *options, bool with_comma, bool with_equal); +chfdw_extract_options( + List* defelems, + char** driver, + char** host, + int* port, + char** dbname, + char** username, + char** password, + char** compression, + tls_mode* tls, + tls_version* min_tls_version +); +extern List* +chfdw_parse_options(const char* options, bool with_comma, bool with_equal); /* in deparse.c */ -extern void chfdw_classify_conditions(PlannerInfo * root, - RelOptInfo * baserel, - List * input_conds, - List * *remote_conds, - List * *local_conds); -extern bool chfdw_is_foreign_expr(PlannerInfo * root, - RelOptInfo * baserel, - Expr * expr); -extern bool is_foreign_param(PlannerInfo * root, - RelOptInfo * baserel, - Expr * expr); -extern char *chfdw_deparse_insert_sql(StringInfo buf, RangeTblEntry * rte, - Index rtindex, Relation rel, - List * targetAttrs); -extern List * chfdw_build_tlist_to_deparse(RelOptInfo * foreignrel); -extern void chfdw_deparse_select_stmt_for_rel(StringInfo buf, PlannerInfo * root, RelOptInfo * rel, - List * tlist, List * remote_conds, List * pathkeys, - bool has_final_sort, bool has_limit, bool is_subquery, - List * *retrieved_attrs, List * *params_list); -extern const char *chfdw_get_jointype_name(JoinType jointype); -char *chfdw_array_to_ch_literal(Datum arr); +extern void +chfdw_classify_conditions( + PlannerInfo* root, + RelOptInfo* baserel, + List* input_conds, + List** remote_conds, + List** local_conds +); +extern bool +chfdw_is_foreign_expr(PlannerInfo* root, RelOptInfo* baserel, Expr* expr); +extern bool +is_foreign_param(PlannerInfo* root, RelOptInfo* baserel, Expr* expr); +extern char* +chfdw_deparse_insert_sql( + StringInfo buf, + RangeTblEntry* rte, + Index rtindex, + Relation rel, + List* targetAttrs +); +extern List* +chfdw_build_tlist_to_deparse(RelOptInfo* foreignrel); +extern void +chfdw_deparse_select_stmt_for_rel( + StringInfo buf, + PlannerInfo* root, + RelOptInfo* rel, + List* tlist, + List* remote_conds, + List* pathkeys, + bool has_final_sort, + bool has_limit, + bool is_subquery, + List** retrieved_attrs, + List** params_list +); +extern const char* +chfdw_get_jointype_name(JoinType jointype); +char* +chfdw_array_to_ch_literal(Datum arr); /* in shippable.c */ -extern bool chfdw_is_builtin(Oid objectId); -extern int chfdw_is_equal_op(Oid opno); +extern bool +chfdw_is_builtin(Oid objectId); +extern int +chfdw_is_equal_op(Oid opno); /* * Connection cache hash table entry */ -typedef struct ConnCacheKey -{ - Oid userid; -} ConnCacheKey; - -typedef struct ConnCacheEntry -{ - ConnCacheKey key; /* hash key (must be first) */ - ch_connection gate; /* connection to foreign server, or NULL */ - /* Remaining fields are invalid when conn is NULL: */ - bool invalidated; /* true if reconnect is pending */ - uint32 server_hashvalue; /* hash value of foreign server OID */ - uint32 mapping_hashvalue; /* hash value of user mapping OID */ -} ConnCacheEntry; +typedef struct ConnCacheKey { + Oid userid; +} ConnCacheKey; + +typedef struct ConnCacheEntry { + ConnCacheKey key; /* hash key (must be first) */ + ch_connection gate; /* connection to foreign server, or NULL */ + /* Remaining fields are invalid when conn is NULL: */ + bool invalidated; /* true if reconnect is pending */ + uint32 server_hashvalue; /* hash value of foreign server OID */ + uint32 mapping_hashvalue; /* hash value of user mapping OID */ +} ConnCacheEntry; /* Custom behavior types */ -typedef enum -{ - CF_USUAL = 0, - CF_SIGN_SUM, /* SUM aggregation */ - CF_SIGN_AVG, /* AVG aggregation */ - CF_SIGN_COUNT, /* COUNT aggregation */ - CF_DATE_TRUNC, /* date_trunc function */ - CF_DATE_PART, /* date_part function */ - CF_TIMESTAMPTZ_PL_INTERVAL, /* timestamptz + interval */ - CF_TIMEZONE, /* timezone */ - CF_HSTORE_FETCHVAL, /* -> operation on hstore */ - CF_INTARRAY_IDX, - CF_CH_FUNCTION, /* adapted clickhouse function */ - CF_MATCH, /* regexp_match function */ - CF_SPLIT_BY_REGEX, /* regexp_split_to_array → splitByRegexp */ - CF_REPLACE_REGEX, /* regexp_replace → replaceRegexpOne or - * replaceRegexpAll */ - CF_REGEX_MATCH, /* ~ POSIX regex operator */ - CF_REGEX_PG_MATCH, /* regexp_match → extract or extractAll */ - CF_REGEX_NO_MATCH, /* !~ POSIX regex operator */ - CF_REGEX_ICASE_MATCH, /* ~* case-insensitive regex operator */ - CF_REGEX_ICASE_NO_MATCH, /* !~* case-insensitive regex operator */ - CF_JSON_FETCHVAL, /* -> operator on json & jsonb */ - CF_JSON_FETCHVAL_TEXT, /* ->> operator on json & jsonb */ - CF_JSON_EXTRACT_PATH_TEXT, /* jsonb?_extract_path_text() → - * col."k1"."k2" */ - CF_JSON_EXTRACT_PATH, /* json?_extract_path() → - * toJSONString(col."k1"."k2") */ - CF_STRING_AGG, /* string_agg → groupConcat(delim)(expr) */ - CF_CURRENT_DATABASE, /* CURRENT_DATABASE → string literal */ - CF_CURRENT_SCHEMA, /* CF_CURRENT_SCHEMA → string literal */ - CF_CLOCK_TIMESTAMP, /* clock_timestamp → nowInBlock64(6, $TZ) */ - CF_ARRAY_LENGTH, /* array_length → length, drop dim arg */ - CF_ARRAY_PREPEND, /* array_prepend → arrayPushFront, swap args */ - CF_STRING_TO_ARRAY, /* string_to_array → splitByString, swap - * args */ - CF_STRING_TO_ARRAY_PART, /* split_part → splitByString()[n] */ - CF_TRIM_ARRAY, /* trim_array → arrayResize(arr, - * length(arr)-n) */ - CF_ARRAY_SORT_DESC, /* array_sort(arr,desc) → - * arrayReverseSort/arraySort */ - CF_ARRAY_FILL, /* array_fill → arrayWithConstant, - * swap+extract */ - CF_ARRAY_CONTAINS, /* @> → hasAll(left, right) */ - CF_ARRAY_CONTAINED_BY, /* <@ → hasAll(right, left) */ - CF_ARRAY_OVERLAP, /* && → hasAny(left, right) */ - CF_TO_CHAR, /* to_char(timestamp[tz], fmt) → - * formatDateTime, with strict format - * translation */ -} custom_object_type; - -typedef enum -{ - CF_AGGR_USUAL = 0, - CF_AGGR_FUNC = 1, - CF_AGGR_SIMPLE = 2 +typedef enum { + CF_USUAL = 0, + CF_SIGN_SUM, /* SUM aggregation */ + CF_SIGN_AVG, /* AVG aggregation */ + CF_SIGN_COUNT, /* COUNT aggregation */ + CF_DATE_TRUNC, /* date_trunc function */ + CF_DATE_PART, /* date_part function */ + CF_TIMESTAMPTZ_PL_INTERVAL, /* timestamptz + interval */ + CF_TIMEZONE, /* timezone */ + CF_HSTORE_FETCHVAL, /* -> operation on hstore */ + CF_INTARRAY_IDX, + CF_CH_FUNCTION, /* adapted clickhouse function */ + CF_MATCH, /* regexp_match function */ + CF_SPLIT_BY_REGEX, /* regexp_split_to_array → splitByRegexp */ + CF_REPLACE_REGEX, /* regexp_replace → replaceRegexpOne or + * replaceRegexpAll */ + CF_REGEX_MATCH, /* ~ POSIX regex operator */ + CF_REGEX_PG_MATCH, /* regexp_match → extract or extractAll */ + CF_REGEX_NO_MATCH, /* !~ POSIX regex operator */ + CF_REGEX_ICASE_MATCH, /* ~* case-insensitive regex operator */ + CF_REGEX_ICASE_NO_MATCH, /* !~* case-insensitive regex operator */ + CF_JSON_FETCHVAL, /* -> operator on json & jsonb */ + CF_JSON_FETCHVAL_TEXT, /* ->> operator on json & jsonb */ + CF_JSON_EXTRACT_PATH_TEXT, /* jsonb?_extract_path_text() → + * col."k1"."k2" */ + CF_JSON_EXTRACT_PATH, /* json?_extract_path() → + * toJSONString(col."k1"."k2") */ + CF_STRING_AGG, /* string_agg → groupConcat(delim)(expr) */ + CF_CURRENT_DATABASE, /* CURRENT_DATABASE → string literal */ + CF_CURRENT_SCHEMA, /* CF_CURRENT_SCHEMA → string literal */ + CF_CLOCK_TIMESTAMP, /* clock_timestamp → nowInBlock64(6, $TZ) */ + CF_ARRAY_LENGTH, /* array_length → length, drop dim arg */ + CF_ARRAY_PREPEND, /* array_prepend → arrayPushFront, swap args */ + CF_STRING_TO_ARRAY, /* string_to_array → splitByString, swap + * args */ + CF_STRING_TO_ARRAY_PART, /* split_part → splitByString()[n] */ + CF_TRIM_ARRAY, /* trim_array → arrayResize(arr, + * length(arr)-n) */ + CF_ARRAY_SORT_DESC, /* array_sort(arr,desc) → + * arrayReverseSort/arraySort */ + CF_ARRAY_FILL, /* array_fill → arrayWithConstant, + * swap+extract */ + CF_ARRAY_CONTAINS, /* @> → hasAll(left, right) */ + CF_ARRAY_CONTAINED_BY, /* <@ → hasAll(right, left) */ + CF_ARRAY_OVERLAP, /* && → hasAny(left, right) */ + CF_TO_CHAR, /* to_char(timestamp[tz], fmt) → + * formatDateTime, with strict format + * translation */ +} custom_object_type; + +typedef enum { + CF_AGGR_USUAL = 0, + CF_AGGR_FUNC = 1, + CF_AGGR_SIMPLE = 2 } ch_aggregate_func_type; -typedef struct CustomObjectDef -{ - Oid cf_oid; - custom_object_type cf_type; - char custom_name[NAMEDATALEN]; /* \0 - no custom name, \1 - many - * names */ - int paren_count; /* Number closing parens; defaults to 1 */ - Oid rowfunc; - void *cf_context; -} CustomObjectDef; - -typedef struct CustomColumnInfo -{ - Oid relid; - int varattno; - char colname[NAMEDATALEN]; - ch_aggregate_func_type is_AggregateFunction; - custom_object_type coltype; - - CHRemoteTableEngine table_engine; -} CustomColumnInfo; - -extern bool chfdw_check_for_ordered_aggregate(Aggref * agg); -extern CustomObjectDef * chfdw_check_for_custom_function(Oid funcid); -extern CustomObjectDef * chfdw_check_for_custom_type(Oid typeoid); -extern void chfdw_apply_custom_table_options(CHFdwRelationInfo * fpinfo, Oid relid); -extern CustomColumnInfo * chfdw_get_custom_column_info(Oid relid, uint16 varattno); -extern CustomObjectDef * chfdw_check_for_custom_operator(Oid opoid, Form_pg_operator form); +typedef struct CustomObjectDef { + Oid cf_oid; + custom_object_type cf_type; + char custom_name[NAMEDATALEN]; /* \0 - no custom name, \1 - many + * names */ + int paren_count; /* Number closing parens; defaults to 1 */ + Oid rowfunc; + void* cf_context; +} CustomObjectDef; + +typedef struct CustomColumnInfo { + Oid relid; + int varattno; + char colname[NAMEDATALEN]; + ch_aggregate_func_type is_AggregateFunction; + custom_object_type coltype; + + CHRemoteTableEngine table_engine; +} CustomColumnInfo; + +extern bool +chfdw_check_for_ordered_aggregate(Aggref* agg); +extern CustomObjectDef* +chfdw_check_for_custom_function(Oid funcid); +extern CustomObjectDef* +chfdw_check_for_custom_type(Oid typeoid); +extern void +chfdw_apply_custom_table_options(CHFdwRelationInfo* fpinfo, Oid relid); +extern CustomColumnInfo* +chfdw_get_custom_column_info(Oid relid, uint16 varattno); +extern CustomObjectDef* +chfdw_check_for_custom_operator(Oid opoid, Form_pg_operator form); extern Datum ch_timestamp_out(PG_FUNCTION_ARGS); extern Datum ch_date_out(PG_FUNCTION_ARGS); extern Datum ch_time_out(PG_FUNCTION_ARGS); -extern bool chfdw_is_shippable(Node * node, Oid objectId, Oid classId, CHFdwRelationInfo * fpinfo, - CustomObjectDef * *outcdef); -extern double time_diff(struct timeval *prior, struct timeval *latter); +extern bool +chfdw_is_shippable( + Node* node, + Oid objectId, + Oid classId, + CHFdwRelationInfo* fpinfo, + CustomObjectDef** outcdef +); +extern double +time_diff(struct timeval* prior, struct timeval* latter); /* * Translate a Postgres `to_char()` template into a ClickHouse @@ -397,34 +452,33 @@ extern double time_diff(struct timeval *prior, struct timeval *latter); * only). Returns false on any unsupported PG keyword or modifier; * when used to gate pushdown, false means caller must evaluate locally. */ -extern bool chfdw_translate_to_char_format(const char *pgfmt, StringInfo out); +extern bool +chfdw_translate_to_char_format(const char* pgfmt, StringInfo out); /* compat */ #if PG_VERSION_NUM < 120000 -#define CreateTemplateTupleDescCompat(natts) \ - CreateTemplateTupleDesc(natts, false) -#define ExecStoreHeapTuple(tup,slot,free) \ - ExecStoreTuple(tup,slot,InvalidBuffer,free) +#define CreateTemplateTupleDescCompat(natts) CreateTemplateTupleDesc(natts, false) +#define ExecStoreHeapTuple(tup, slot, free) \ + ExecStoreTuple(tup, slot, InvalidBuffer, free) -#define execute_attr_map_tuple do_convert_tuple -#define create_foreign_join_path create_foreignscan_path +#define execute_attr_map_tuple do_convert_tuple +#define create_foreign_join_path create_foreignscan_path -#define T_SubscriptingRef T_ArrayRef -#define SubscriptingRef ArrayRef +#define T_SubscriptingRef T_ArrayRef +#define SubscriptingRef ArrayRef #else -#define CreateTemplateTupleDescCompat(natts) \ - CreateTemplateTupleDesc(natts) +#define CreateTemplateTupleDescCompat(natts) CreateTemplateTupleDesc(natts) #endif #if PG_VERSION_NUM < 130000 -#define table_open_compat(i,l) heap_open(i, l) -#define table_close_compat(r,l) heap_close(r, l) -#define lnext_compat(l,i) lnext(i) +#define table_open_compat(i, l) heap_open(i, l) +#define table_close_compat(r, l) heap_close(r, l) +#define lnext_compat(l, i) lnext(i) #else -#define table_open_compat(i,l) table_open(i, l) -#define table_close_compat(r,l) table_close(r, l) -#define lnext_compat(l,i) lnext(l,i) +#define table_open_compat(i, l) table_open(i, l) +#define table_close_compat(r, l) table_close(r, l) +#define lnext_compat(l, i) lnext(l, i) #endif -#endif /* CLICKHOUSE_FDW_H */ +#endif /* CLICKHOUSE_FDW_H */ diff --git a/src/include/http.h b/src/include/http.h index b462422..b75274d 100644 --- a/src/include/http.h +++ b/src/include/http.h @@ -2,9 +2,10 @@ #define CLICKHOUSE_HTTP_H #include "postgres.h" -#include "nodes/pg_list.h" -#include "lib/stringinfo.h" + #include "engine.h" +#include "lib/stringinfo.h" +#include "nodes/pg_list.h" #define CH_HTTP_QUERY_ID_LEN 37 @@ -17,55 +18,58 @@ #define CH_HTTP_STATUS_TRANSPORT_ERROR 419L typedef struct ch_http_connection_t ch_http_connection_t; -typedef struct ch_http_response_t -{ - char *data; - size_t datasize; - long http_status; - char query_id[CH_HTTP_QUERY_ID_LEN]; - double pretransfer_time; - double total_time; -} ch_http_response_t; +typedef struct ch_http_response_t { + char* data; + size_t datasize; + long http_status; + char query_id[CH_HTTP_QUERY_ID_LEN]; + double pretransfer_time; + double total_time; +} ch_http_response_t; -typedef enum -{ - CH_CONT, - CH_EOL, - CH_EOF -} ch_read_status; +typedef enum { CH_CONT, CH_EOL, CH_EOF } ch_read_status; -typedef struct -{ - char *data; - size_t datalen; - size_t curpos; - StringInfoData val; - bool done; - bool is_null; /* set when the parser saw the wire NULL - * marker `\N` for the field just read */ -} ch_http_read_state; +typedef struct { + char* data; + size_t datalen; + size_t curpos; + StringInfoData val; + bool done; + bool is_null; /* set when the parser saw the wire NULL + * marker `\N` for the field just read */ +} ch_http_read_state; -typedef struct -{ - StringInfoData sql; - char *sql_begin; /* beginning part of constructed sql */ - List *target_attrs; /* list of target attribute numbers */ - int p_nums; /* number of parameters to transmit */ - ch_http_connection_t *conn; -} ch_http_insert_state; +typedef struct { + StringInfoData sql; + char* sql_begin; /* beginning part of constructed sql */ + List* target_attrs; /* list of target attribute numbers */ + int p_nums; /* number of parameters to transmit */ + ch_http_connection_t* conn; +} ch_http_insert_state; -void ch_http_init(int verbose, uint32_t query_id_prefix); -void ch_http_set_progress_func(void *progressfunc); -void *ch_http_get_progress_func(void); -long ch_http_get_verbose(void); -ch_http_connection_t *ch_http_connection(ch_connection_details * details); -void ch_http_close(ch_http_connection_t * conn); -ch_http_response_t *ch_http_simple_query(ch_http_connection_t * conn, const ch_query * query); -char *ch_http_last_error(void); +void +ch_http_init(int verbose, uint32_t query_id_prefix); +void +ch_http_set_progress_func(void* progressfunc); +void* +ch_http_get_progress_func(void); +long +ch_http_get_verbose(void); +ch_http_connection_t* +ch_http_connection(ch_connection_details* details); +void +ch_http_close(ch_http_connection_t* conn); +ch_http_response_t* +ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query); +char* +ch_http_last_error(void); /* read */ -void ch_http_read_state_init(ch_http_read_state * state, char *data, size_t datalen); -int ch_http_read_next(ch_http_read_state * state, bool is_array); -void ch_http_response_free(ch_http_response_t * resp); +void +ch_http_read_state_init(ch_http_read_state* state, char* data, size_t datalen); +int +ch_http_read_next(ch_http_read_state* state, bool is_array); +void +ch_http_response_free(ch_http_response_t* resp); -#endif /* CLICKHOUSE_HTTP_H */ +#endif /* CLICKHOUSE_HTTP_H */ diff --git a/src/include/http_streaming.h b/src/include/http_streaming.h index 80b7c46..135ddaa 100644 --- a/src/include/http_streaming.h +++ b/src/include/http_streaming.h @@ -5,29 +5,43 @@ typedef struct ch_http_connection_t ch_http_connection_t; - /* - * Opaque handle to a streaming HTTP query. The real type is the HttpStream - * struct, defined in http_streaming.c. - */ +/* + * Opaque handle to a streaming HTTP query. The real type is the HttpStream + * struct, defined in http_streaming.c. + */ typedef struct HttpStream HttpStream; - /* lifecycle */ -HttpStream *ch_http_stream_begin(ch_http_connection_t * conn, - const ch_query * query, - int32 fetch_size); -int ch_http_stream_pump(HttpStream * stream); -void ch_http_stream_end(HttpStream * stream); +/* lifecycle */ +HttpStream* +ch_http_stream_begin( + ch_http_connection_t* conn, + const ch_query* query, + int32 fetch_size +); +int +ch_http_stream_pump(HttpStream* stream); +void +ch_http_stream_end(HttpStream* stream); - /* accessors — let pglink.c read stream state without seeing the struct */ -char *ch_http_stream_buffer(HttpStream * stream); -size_t ch_http_stream_available(HttpStream * stream); -void ch_http_stream_advance(HttpStream * stream, size_t n); -bool ch_http_stream_transfer_done(HttpStream * stream); -long ch_http_stream_status(HttpStream * stream); -const char *ch_http_stream_query_id(HttpStream * stream); -const char *ch_http_stream_error(HttpStream * stream); -double ch_http_stream_request_time(HttpStream * stream); -double ch_http_stream_total_time(HttpStream * stream); +/* accessors — let pglink.c read stream state without seeing the struct */ +char* +ch_http_stream_buffer(HttpStream* stream); +size_t +ch_http_stream_available(HttpStream* stream); +void +ch_http_stream_advance(HttpStream* stream, size_t n); +bool +ch_http_stream_transfer_done(HttpStream* stream); +long +ch_http_stream_status(HttpStream* stream); +const char* +ch_http_stream_query_id(HttpStream* stream); +const char* +ch_http_stream_error(HttpStream* stream); +double +ch_http_stream_request_time(HttpStream* stream); +double +ch_http_stream_total_time(HttpStream* stream); /* * Transfer ownership of the response body to the caller. On return, *out_data @@ -36,7 +50,7 @@ double ch_http_stream_total_time(HttpStream * stream); * itself is unchanged otherwise and should still be released with * ch_http_stream_end(). */ -void ch_http_stream_take_body(HttpStream * stream, - char **out_data, size_t * out_size); +void +ch_http_stream_take_body(HttpStream* stream, char** out_data, size_t* out_size); -#endif /* CLICKHOUSE_HTTP_STREAMING_H */ +#endif /* CLICKHOUSE_HTTP_STREAMING_H */ diff --git a/src/include/internal.h b/src/include/internal.h index 7c89476..d7c0be7 100644 --- a/src/include/internal.h +++ b/src/include/internal.h @@ -3,25 +3,25 @@ #include "curl/curl.h" -typedef struct ch_http_connection_t -{ - CURL *curl; - char *dbname; - char *base_url; - long ssl_version; /* CURLOPT_SSLVERSION min; DEFAULT means unset */ -} ch_http_connection_t; +typedef struct ch_http_connection_t { + CURL* curl; + char* dbname; + char* base_url; + long ssl_version; /* CURLOPT_SSLVERSION min; DEFAULT means unset */ +} ch_http_connection_t; -typedef struct ch_binary_connection_t -{ - void *client; - void *options; - char *error; -} ch_binary_connection_t; +typedef struct ch_binary_connection_t { + void* client; + void* options; + char* error; +} ch_binary_connection_t; /* * Check whether the given string matches a ClickHouse Cloud host name. */ -extern int ch_is_cloud_host(const char *host); -int ends_with(const char *s, const char *suffix); +extern int +ch_is_cloud_host(const char* host); +int +ends_with(const char* s, const char* suffix); -#endif /* CLICKHOUSE_INTERNAL_H */ +#endif /* CLICKHOUSE_INTERNAL_H */ diff --git a/src/include/kv_list.h b/src/include/kv_list.h index a867a01..6a6d6b2 100644 --- a/src/include/kv_list.h +++ b/src/include/kv_list.h @@ -2,19 +2,20 @@ #define PG_CLICKHOUSE_KV_LIST_H #include + #include "postgres.h" + #include "nodes/pathnodes.h" /* * A simple data structure with a list of key/value string pairs. Use * new_kv_list_from_list() to create. */ -typedef struct kv_list -{ - int length; - /* key/value char * pairs follow the length field. */ - char data[]; -} kv_list; +typedef struct kv_list { + int length; + /* key/value char * pairs follow the length field. */ + char data[]; +} kv_list; /* * Iterator for a kv_list. Use new_kv_iter() to create. @@ -24,38 +25,40 @@ typedef struct kv_list * printf("%i, %s => %s\n", iter.num, iter.name, iter.value); * } */ -typedef struct kv_iter -{ - int togo; - char *name; - char *value; -} kv_iter; +typedef struct kv_iter { + int togo; + char* name; + char* value; +} kv_iter; /* * Defines the allocator to use when creating a new kv_list. * kv_pair_guc_malloc is the same as kv_pair_malloc on Postgres 15 and * earlier, so be sure to free() the memory on those versions. */ -enum kv_pair_alloc -{ - kv_pair_guc_malloc, - kv_pair_malloc, - kv_pair_palloc, +enum kv_pair_alloc { + kv_pair_guc_malloc, + kv_pair_malloc, + kv_pair_palloc, }; /* * Create a new kv_list from a PostgreSQL List of DefElem. Allocate the memory * using the specified allocator. -*/ -kv_list *new_kv_list_from_pg_list(List * list, int allocate); + */ +kv_list* +new_kv_list_from_pg_list(List* list, int allocate); /* Create a new kv_iter for a key_pairs. */ -kv_iter new_kv_iter(const kv_list * ns); +kv_iter +new_kv_iter(const kv_list* ns); /* Iterate to the next item. Returns false if there are no items. */ -bool kv_iter_next(kv_iter * state); +bool +kv_iter_next(kv_iter* state); /* Returns true if iteration by kv_iter is complete. */ -bool kv_iter_done(kv_iter * state); +bool +kv_iter_done(kv_iter* state); -#endif /* PG_CLICKHOUSE_KV_LIST_H */ +#endif /* PG_CLICKHOUSE_KV_LIST_H */ diff --git a/src/internal.c b/src/internal.c index cd9baac..2fce91b 100644 --- a/src/internal.c +++ b/src/internal.c @@ -1,24 +1,23 @@ -#include #include +#include int -ends_with(const char *s, const char *suffix) -{ - size_t slen = strlen(s); - size_t suffix_len = strlen(suffix); +ends_with(const char* s, const char* suffix) { + size_t slen = strlen(s); + size_t suffix_len = strlen(suffix); - return suffix_len <= slen && !strcmp(s + slen - suffix_len, suffix); + return suffix_len <= slen && !strcmp(s + slen - suffix_len, suffix); } /* * Check whether the given string matches a ClickHouse Cloud host name. */ int -ch_is_cloud_host(const char *host) -{ - if (!host) - return 0; - return ends_with(host, ".clickhouse.cloud") - || ends_with(host, ".clickhouse-staging.com") - || ends_with(host, ".clickhouse-dev.com"); +ch_is_cloud_host(const char* host) { + if (!host) { + return 0; + } + return ends_with(host, ".clickhouse.cloud") || + ends_with(host, ".clickhouse-staging.com") || + ends_with(host, ".clickhouse-dev.com"); } diff --git a/src/kv_list.c b/src/kv_list.c index 0f499e0..68e8d8c 100644 --- a/src/kv_list.c +++ b/src/kv_list.c @@ -1,120 +1,111 @@ -#include #include "kv_list.h" -#include "utils/elog.h" #include "nodes/pathnodes.h" +#include "utils/elog.h" #include "utils/guc.h" +#include -static kv_list * allocate_list(size_t size, int allocate) -{ - kv_list *pairs; +static kv_list* +allocate_list(size_t size, int allocate) { + kv_list* pairs; - switch (allocate) - { - case kv_pair_guc_malloc: - /* Fall through to malloc prior to Postgres 16. */ + switch (allocate) { + case kv_pair_guc_malloc: + /* Fall through to malloc prior to Postgres 16. */ #if PG_VERSION_NUM >= 160000 - pairs = guc_malloc(ERROR, size); - break; + pairs = guc_malloc(ERROR, size); + break; #endif - case kv_pair_malloc: - pairs = malloc(size); - break; - case kv_pair_palloc: - pairs = palloc(size); - break; - default: - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("unknown kv_pair_alloc %i", allocate))); - } - - if (!pairs) - ereport(ERROR, - (errcode(ERRCODE_FDW_OUT_OF_MEMORY), - errmsg("out of memory"))); - - return pairs; + case kv_pair_malloc: + pairs = malloc(size); + break; + case kv_pair_palloc: + pairs = palloc(size); + break; + default: + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), errmsg("unknown kv_pair_alloc %i", allocate)) + ); + } + + if (!pairs) { + ereport(ERROR, (errcode(ERRCODE_FDW_OUT_OF_MEMORY), errmsg("out of memory"))); + } + + return pairs; } -extern kv_list * new_kv_list_from_pg_list(List * list, int allocate) -{ - ListCell *lc; - DefElem *elem; - kv_list *pairs; - size_t kv_size = 0; - size_t ck_size = 0; - - /* Count up space for dynamic key/value pairs. */ - foreach(lc, list) - { - elem = (DefElem *) lfirst(lc); - kv_size += 2 + strlen(elem->defname) + strlen(strVal(elem->arg)); - } - - /* Alloc the result ... */ - pairs = allocate_list(offsetof(kv_list, data) + kv_size, allocate); - - /* ... and fill it in */ - pairs->length = list_length(list); - - /* In this loop, size ck_size reprises the kv_size calculation above. */ - foreach(lc, list) - { - char *str; - - /* Append the element name and arg that constitute the pair. */ - elem = (DefElem *) lfirst(lc); - str = (char *) pairs->data + ck_size; - size_t len = strlen(elem->defname) + 1; - - memcpy(str, elem->defname, len); - ck_size += len; - str = (char *) pairs->data + ck_size; - len = strlen(strVal(elem->arg)) + 1; - memcpy(str, strVal(elem->arg), len); - ck_size += len; - } - - /* Assert the two loops agreed on size calculations. */ - Assert(kv_size == ck_size); - - return pairs; +extern kv_list* +new_kv_list_from_pg_list(List* list, int allocate) { + ListCell* lc; + DefElem* elem; + kv_list* pairs; + size_t kv_size = 0; + size_t ck_size = 0; + + /* Count up space for dynamic key/value pairs. */ + foreach (lc, list) { + elem = (DefElem*)lfirst(lc); + kv_size += 2 + strlen(elem->defname) + strlen(strVal(elem->arg)); + } + + /* Alloc the result ... */ + pairs = allocate_list(offsetof(kv_list, data) + kv_size, allocate); + + /* ... and fill it in */ + pairs->length = list_length(list); + + /* In this loop, size ck_size reprises the kv_size calculation above. */ + foreach (lc, list) { + char* str; + + /* Append the element name and arg that constitute the pair. */ + elem = (DefElem*)lfirst(lc); + str = (char*)pairs->data + ck_size; + size_t len = strlen(elem->defname) + 1; + + memcpy(str, elem->defname, len); + ck_size += len; + str = (char*)pairs->data + ck_size; + len = strlen(strVal(elem->arg)) + 1; + memcpy(str, strVal(elem->arg), len); + ck_size += len; + } + + /* Assert the two loops agreed on size calculations. */ + Assert(kv_size == ck_size); + + return pairs; } -extern kv_iter new_kv_iter(const kv_list * ns) -{ - char *name; - - /* The list may be NULL or empty. */ - if (!ns || ns->length < 1) - return (kv_iter) - { - 0, - }; - - /* Grab the number of pairs point to the first name and value. */ - name = (char *) ns->data; - return (kv_iter) - { - ns->length, name, name + strlen(name) + 1 - }; +extern kv_iter +new_kv_iter(const kv_list* ns) { + char* name; + + /* The list may be NULL or empty. */ + if (!ns || ns->length < 1) { + return (kv_iter){ 0 }; + } + + /* Grab the number of pairs point to the first name and value. */ + name = (char*)ns->data; + return (kv_iter){ ns->length, name, name + strlen(name) + 1 }; } extern bool -kv_iter_next(kv_iter * iter) -{ - if (iter->togo == 0) - return false; - - /* Point to the the next name and value. */ - iter->togo--; - iter->name = iter->value + strlen(iter->value) + 1; - iter->value = iter->name + strlen(iter->name) + 1; - return true; +kv_iter_next(kv_iter* iter) { + if (iter->togo == 0) { + return false; + } + + /* Point to the the next name and value. */ + iter->togo--; + iter->name = iter->value + strlen(iter->value) + 1; + iter->value = iter->name + strlen(iter->name) + 1; + return true; } extern bool -kv_iter_done(kv_iter * iter) -{ - return iter->togo == 0; +kv_iter_done(kv_iter* iter) { + return iter->togo == 0; } diff --git a/src/option.c b/src/option.c index b3ba1ef..170f5f1 100644 --- a/src/option.c +++ b/src/option.c @@ -9,7 +9,7 @@ * * IDENTIFICATION * github.com/clickhouse/pg_clickhouse/src/option.c -* + * *------------------------------------------------------------------------- */ #include "postgres.h" @@ -24,65 +24,69 @@ #include "commands/defrem.h" #include "commands/extension.h" #include "nodes/makefuncs.h" +#include "utils/builtins.h" #include "utils/guc.h" #include "utils/varlena.h" -#include "utils/builtins.h" -static char *DEFAULT_DBNAME = "default"; +static char* DEFAULT_DBNAME = "default"; #if PG_VERSION_NUM < 160000 -extern PGDLLEXPORT void _PG_init(void); +extern PGDLLEXPORT void +_PG_init(void); #endif /* * Describes the valid options for objects that this wrapper uses. */ -typedef struct ChFdwOption -{ - const char *keyword; - Oid optcontext; /* OID of catalog in which option may appear */ - bool is_ch_opt; /* true if it's used in clickhouseclient */ - char dispchar[10]; -} ChFdwOption; +typedef struct ChFdwOption { + const char* keyword; + Oid optcontext; /* OID of catalog in which option may appear */ + bool is_ch_opt; /* true if it's used in clickhouseclient */ + char dispchar[10]; +} ChFdwOption; /* * Valid options for clickhouse_fdw. * Allocated and filled in InitChFdwOptions. */ -static ChFdwOption * clickhouse_fdw_options; +static ChFdwOption* clickhouse_fdw_options; /* * Valid options for clickhouse client. * Allocated and filled in InitChFdwOptions. */ -static const ChFdwOption ch_options[] = -{ - {"host", 0, false}, - {"secure", 0, false}, - {"min_tls_version", 0, false}, - {"port", 0, false}, - {"dbname", 0, false}, - {"user", 0, false}, - {"password", 0, false}, - {"compression", 0, false}, - {NULL} +static const ChFdwOption ch_options[] = { + { "host", 0, false }, + { "secure", 0, false }, + { "min_tls_version", 0, false }, + { "port", 0, false }, + { "dbname", 0, false }, + { "user", 0, false }, + { "password", 0, false }, + { "compression", 0, false }, + { NULL } }; /* * GUC parameters */ -static char *ch_session_settings = NULL; -static kv_list * ch_session_settings_list = NULL; -static bool ch_pushdown_regex = true; +static char* ch_session_settings = NULL; +static kv_list* ch_session_settings_list = NULL; +static bool ch_pushdown_regex = true; /* * Helper functions */ -static void InitChFdwOptions(void); -static bool is_valid_option(const char *keyword, Oid context); -static bool is_ch_option(const char *keyword); -static void validate_fetch_size_option(DefElem * def); -static bool parse_min_tls_version(const char *val, tls_version * out); +static void +InitChFdwOptions(void); +static bool +is_valid_option(const char* keyword, Oid context); +static bool +is_ch_option(const char* keyword); +static void +validate_fetch_size_option(DefElem* def); +static bool +parse_min_tls_version(const char* val, tls_version* out); /* * Validate the generic options given to a FOREIGN DATA WRAPPER, SERVER, @@ -93,178 +97,173 @@ static bool parse_min_tls_version(const char *val, tls_version * out); PG_FUNCTION_INFO_V1(clickhouse_fdw_validator); Datum -clickhouse_fdw_validator(PG_FUNCTION_ARGS) -{ - List *options_list = untransformRelOptions(PG_GETARG_DATUM(0)); - Oid catalog = PG_GETARG_OID(1); - ListCell *cell; - - /* Build our options lists if we didn't yet. */ - InitChFdwOptions(); - - /* - * Check that only options supported by clickhouse_fdw, and allowed for - * the current object type, are given. - */ - foreach(cell, options_list) - { - DefElem *def = (DefElem *) lfirst(cell); - - if (!is_valid_option(def->defname, catalog)) - { - /* - * Unknown option specified, complain about it. Provide a hint - * with list of valid options for the object. - */ - ChFdwOption *opt; - StringInfoData buf; - - initStringInfo(&buf); - for (opt = clickhouse_fdw_options; opt->keyword; opt++) - { - if (catalog == opt->optcontext) - appendStringInfo(&buf, "%s%s", (buf.len > 0) ? ", " : "", - opt->keyword); - } - - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), - errmsg("invalid option \"%s\"", def->defname), - errhint("Valid options in this context are: %s", - buf.data))); - } - - if (strcmp(def->defname, "fetch_size") == 0) - validate_fetch_size_option(def); - - if (strcmp(def->defname, "secure") == 0) - { - const char *val = defGetString(def); - - if (strcmp(val, "on") != 0 && strcmp(val, "true") != 0 && - strcmp(val, "yes") != 0 && strcmp(val, "1") != 0 && - strcmp(val, "off") != 0 && strcmp(val, "false") != 0 && - strcmp(val, "no") != 0 && strcmp(val, "0") != 0 && - strcmp(val, "auto") != 0) - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_STRING_FORMAT), - errmsg("invalid value for option \"secure\": \"%s\"", - val), - errhint("Valid values are: on, off, auto."))); - } - - if (strcmp(def->defname, "min_tls_version") == 0) - { - const char *val = defGetString(def); - tls_version v; - - if (!parse_min_tls_version(val, &v)) - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_STRING_FORMAT), - errmsg("invalid value for option \"min_tls_version\": \"%s\"", - val), - errhint("Valid values are: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3."))); - } - } - - PG_RETURN_VOID(); +clickhouse_fdw_validator(PG_FUNCTION_ARGS) { + List* options_list = untransformRelOptions(PG_GETARG_DATUM(0)); + Oid catalog = PG_GETARG_OID(1); + ListCell* cell; + + /* Build our options lists if we didn't yet. */ + InitChFdwOptions(); + + /* + * Check that only options supported by clickhouse_fdw, and allowed for + * the current object type, are given. + */ + foreach (cell, options_list) { + DefElem* def = (DefElem*)lfirst(cell); + + if (!is_valid_option(def->defname, catalog)) { + /* + * Unknown option specified, complain about it. Provide a hint + * with list of valid options for the object. + */ + ChFdwOption* opt; + StringInfoData buf; + + initStringInfo(&buf); + for (opt = clickhouse_fdw_options; opt->keyword; opt++) { + if (catalog == opt->optcontext) { + appendStringInfo( + &buf, "%s%s", (buf.len > 0) ? ", " : "", opt->keyword + ); + } + } + + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid option \"%s\"", def->defname), + errhint("Valid options in this context are: %s", buf.data)) + ); + } + + if (strcmp(def->defname, "fetch_size") == 0) { + validate_fetch_size_option(def); + } + + if (strcmp(def->defname, "secure") == 0) { + const char* val = defGetString(def); + + if (strcmp(val, "on") != 0 && strcmp(val, "true") != 0 && + strcmp(val, "yes") != 0 && strcmp(val, "1") != 0 && + strcmp(val, "off") != 0 && strcmp(val, "false") != 0 && + strcmp(val, "no") != 0 && strcmp(val, "0") != 0 && + strcmp(val, "auto") != 0) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_STRING_FORMAT), + errmsg("invalid value for option \"secure\": \"%s\"", val), + errhint("Valid values are: on, off, auto.")) + ); + } + } + + if (strcmp(def->defname, "min_tls_version") == 0) { + const char* val = defGetString(def); + tls_version v; + + if (!parse_min_tls_version(val, &v)) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_STRING_FORMAT), + errmsg( + "invalid value for option \"min_tls_version\": \"%s\"", val + ), + errhint("Valid values are: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3.")) + ); + } + } + } + + PG_RETURN_VOID(); } /* * Initialize option lists. */ static void -InitChFdwOptions(void) -{ - int num_ch_opts; - const ChFdwOption *lopt; - ChFdwOption *popt; - - /* non-clickhouseclient FDW-specific FDW options */ - static const ChFdwOption non_ch_options[] = - { - {"database", ForeignTableRelationId, false}, - {"table_name", ForeignTableRelationId, false}, - {"engine", ForeignTableRelationId, false}, - {"driver", ForeignServerRelationId, false}, - {"fetch_size", ForeignServerRelationId, false}, - {"fetch_size", ForeignTableRelationId, false}, - {"aggregatefunction", AttributeRelationId, false}, - {"simpleaggregatefunction", AttributeRelationId, false}, - {"column_name", AttributeRelationId, false}, - {NULL, InvalidOid, false} - }; - - /* Prevent redundant initialization. */ - if (clickhouse_fdw_options) - { - return; - } - - - /* Count how many clickhouseclient options are available. */ - num_ch_opts = 0; - for (lopt = ch_options; lopt->keyword; lopt++) - { - num_ch_opts++; - } - - /* - * We use plain malloc here to allocate clickhouse_fdw_options because it - * lives as long as the backend process does. Besides, keeping ch_options - * in memory allows us to avoid copying every keyword string. - */ - clickhouse_fdw_options = (ChFdwOption *) malloc(sizeof( - ChFdwOption) * num_ch_opts + sizeof(non_ch_options)); - if (clickhouse_fdw_options == NULL) - ereport(ERROR, - (errcode(ERRCODE_FDW_OUT_OF_MEMORY), - errmsg("out of memory"))); - - popt = clickhouse_fdw_options; - for (lopt = ch_options; lopt->keyword; lopt++) - { - /* We don't have to copy keyword string, as described above. */ - popt->keyword = lopt->keyword; - - /* - * "user" and any secret options are allowed only on user mappings. - * Everything else is a server option. - */ - if (strcmp(lopt->keyword, "user") == 0 || - strcmp(lopt->keyword, "password") == 0 || - strchr(lopt->dispchar, '*')) - { - popt->optcontext = UserMappingRelationId; - } - else - { - popt->optcontext = ForeignServerRelationId; - } - popt->is_ch_opt = true; - - popt++; - } - - /* Append FDW-specific options and dummy terminator. */ - memcpy(popt, non_ch_options, sizeof(non_ch_options)); - popt->is_ch_opt = true; - popt++; +InitChFdwOptions(void) { + int num_ch_opts; + const ChFdwOption* lopt; + ChFdwOption* popt; + + /* non-clickhouseclient FDW-specific FDW options */ + static const ChFdwOption non_ch_options[] = { + { "database", ForeignTableRelationId, false }, + { "table_name", ForeignTableRelationId, false }, + { "engine", ForeignTableRelationId, false }, + { "driver", ForeignServerRelationId, false }, + { "fetch_size", ForeignServerRelationId, false }, + { "fetch_size", ForeignTableRelationId, false }, + { "aggregatefunction", AttributeRelationId, false }, + { "simpleaggregatefunction", AttributeRelationId, false }, + { "column_name", AttributeRelationId, false }, + { NULL, InvalidOid, false } + }; + + /* Prevent redundant initialization. */ + if (clickhouse_fdw_options) { + return; + } + + /* Count how many clickhouseclient options are available. */ + num_ch_opts = 0; + for (lopt = ch_options; lopt->keyword; lopt++) { + num_ch_opts++; + } + + /* + * We use plain malloc here to allocate clickhouse_fdw_options because it + * lives as long as the backend process does. Besides, keeping ch_options + * in memory allows us to avoid copying every keyword string. + */ + clickhouse_fdw_options = (ChFdwOption*)malloc( + sizeof(ChFdwOption) * num_ch_opts + sizeof(non_ch_options) + ); + if (clickhouse_fdw_options == NULL) { + ereport(ERROR, (errcode(ERRCODE_FDW_OUT_OF_MEMORY), errmsg("out of memory"))); + } + + popt = clickhouse_fdw_options; + for (lopt = ch_options; lopt->keyword; lopt++) { + /* We don't have to copy keyword string, as described above. */ + popt->keyword = lopt->keyword; + + /* + * "user" and any secret options are allowed only on user mappings. + * Everything else is a server option. + */ + if (strcmp(lopt->keyword, "user") == 0 || + strcmp(lopt->keyword, "password") == 0 || strchr(lopt->dispchar, '*')) { + popt->optcontext = UserMappingRelationId; + } else { + popt->optcontext = ForeignServerRelationId; + } + popt->is_ch_opt = true; + + popt++; + } + + /* Append FDW-specific options and dummy terminator. */ + memcpy(popt, non_ch_options, sizeof(non_ch_options)); + popt->is_ch_opt = true; + popt++; } static void -validate_fetch_size_option(DefElem * def) -{ - int fetch_size = pg_strtoint32(defGetString(def)); - - if (fetch_size < 0) - { - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), - errmsg("invalid value for option \"%s\": %s", - def->defname, defGetString(def)), - errhint("fetch_size must be greater than or equal to 0"))); - } +validate_fetch_size_option(DefElem* def) { + int fetch_size = pg_strtoint32(defGetString(def)); + + if (fetch_size < 0) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), + errmsg( + "invalid value for option \"%s\": %s", def->defname, defGetString(def) + ), + errhint("fetch_size must be greater than or equal to 0")) + ); + } } /* @@ -273,19 +272,19 @@ validate_fetch_size_option(DefElem * def) * if the value is unrecognized. */ static bool -parse_min_tls_version(const char *val, tls_version * out) -{ - if (pg_strcasecmp(val, "TLSv1") == 0) - *out = CH_TLS_V1_0; - else if (pg_strcasecmp(val, "TLSv1.1") == 0) - *out = CH_TLS_V1_1; - else if (pg_strcasecmp(val, "TLSv1.2") == 0) - *out = CH_TLS_V1_2; - else if (pg_strcasecmp(val, "TLSv1.3") == 0) - *out = CH_TLS_V1_3; - else - return false; - return true; +parse_min_tls_version(const char* val, tls_version* out) { + if (pg_strcasecmp(val, "TLSv1") == 0) { + *out = CH_TLS_V1_0; + } else if (pg_strcasecmp(val, "TLSv1.1") == 0) { + *out = CH_TLS_V1_1; + } else if (pg_strcasecmp(val, "TLSv1.2") == 0) { + *out = CH_TLS_V1_2; + } else if (pg_strcasecmp(val, "TLSv1.3") == 0) { + *out = CH_TLS_V1_3; + } else { + return false; + } + return true; } /* @@ -293,42 +292,36 @@ parse_min_tls_version(const char *val, tls_version * out) * context is the Oid of the catalog holding the object the option is for. */ static bool -is_valid_option(const char *keyword, Oid context) -{ - ChFdwOption *opt; +is_valid_option(const char* keyword, Oid context) { + ChFdwOption* opt; - Assert(clickhouse_fdw_options); /* must be initialized already */ + Assert(clickhouse_fdw_options); /* must be initialized already */ - for (opt = clickhouse_fdw_options; opt->keyword; opt++) - { - if (context == opt->optcontext && strcmp(opt->keyword, keyword) == 0) - { - return true; - } - } + for (opt = clickhouse_fdw_options; opt->keyword; opt++) { + if (context == opt->optcontext && strcmp(opt->keyword, keyword) == 0) { + return true; + } + } - return false; + return false; } /* * Check whether the given option is one of the valid clickhouseclient options. */ static bool -is_ch_option(const char *keyword) -{ - ChFdwOption *opt; +is_ch_option(const char* keyword) { + ChFdwOption* opt; - Assert(clickhouse_fdw_options); /* must be initialized already */ + Assert(clickhouse_fdw_options); /* must be initialized already */ - for (opt = clickhouse_fdw_options; opt->keyword; opt++) - { - if (opt->is_ch_opt && strcmp(opt->keyword, keyword) == 0) - { - return true; - } - } + for (opt = clickhouse_fdw_options; opt->keyword; opt++) { + if (opt->is_ch_opt && strcmp(opt->keyword, keyword) == 0) { + return true; + } + } - return false; + return false; } /* @@ -337,68 +330,75 @@ is_ch_option(const char *keyword) * allocated large-enough arrays. Returns number of options found. */ void -chfdw_extract_options(List * defelems, char **driver, char **host, int *port, - char **dbname, char **username, char **password, - char **compression, tls_mode * tls, - tls_version * min_tls_version) -{ - ListCell *lc; - - /* Build our options lists if we didn't yet. */ - InitChFdwOptions(); - - foreach(lc, defelems) - { - DefElem *def = (DefElem *) lfirst(lc); - - if (driver && strcmp(def->defname, "driver") == 0) - *driver = defGetString(def); - - if (is_ch_option(def->defname)) - { - if (host && strcmp(def->defname, "host") == 0) - *host = defGetString(def); - else if (port && strcmp(def->defname, "port") == 0) - { - char *endptr = NULL; - long val = strtol(defGetString(def), &endptr, 10); - - /* Just ignore invalid values. */ - if (val <= INT_MAX && val > 0 && (!*endptr || *endptr == 0)) - *port = val; - } - else if (username && strcmp(def->defname, "user") == 0) - *username = defGetString(def); - else if (password && strcmp(def->defname, "password") == 0) - *password = defGetString(def); - else if (compression && strcmp(def->defname, "compression") == 0) - *compression = defGetString(def); - else if (dbname && strcmp(def->defname, "dbname") == 0) - { - *dbname = defGetString(def); - if (*dbname[0] == '\0') - *dbname = DEFAULT_DBNAME; - } - else if (tls && strcmp(def->defname, "secure") == 0) - { - const char *val = defGetString(def); - - if (strcmp(val, "on") == 0 || strcmp(val, "true") == 0 || - strcmp(val, "yes") == 0 || strcmp(val, "1") == 0) - *tls = CH_TLS_ON; - else if (strcmp(val, "off") == 0 || strcmp(val, "false") == 0 || - strcmp(val, "no") == 0 || strcmp(val, "0") == 0) - *tls = CH_TLS_OFF; - /* else: "auto" or anything else → CH_TLS_AUTO (already 0) */ - } - else if (min_tls_version && strcmp(def->defname, "min_tls_version") == 0) - /* invalid values rejected by the validator; ignore here */ - parse_min_tls_version(defGetString(def), min_tls_version); - } - } +chfdw_extract_options( + List* defelems, + char** driver, + char** host, + int* port, + char** dbname, + char** username, + char** password, + char** compression, + tls_mode* tls, + tls_version* min_tls_version +) { + ListCell* lc; + + /* Build our options lists if we didn't yet. */ + InitChFdwOptions(); + + foreach (lc, defelems) { + DefElem* def = (DefElem*)lfirst(lc); + + if (driver && strcmp(def->defname, "driver") == 0) { + *driver = defGetString(def); + } + + if (is_ch_option(def->defname)) { + if (host && strcmp(def->defname, "host") == 0) { + *host = defGetString(def); + } else if (port && strcmp(def->defname, "port") == 0) { + char* endptr = NULL; + long val = strtol(defGetString(def), &endptr, 10); + + /* Just ignore invalid values. */ + if (val <= INT_MAX && val > 0 && (!*endptr || *endptr == 0)) { + *port = val; + } + } else if (username && strcmp(def->defname, "user") == 0) { + *username = defGetString(def); + } else if (password && strcmp(def->defname, "password") == 0) { + *password = defGetString(def); + } else if (compression && strcmp(def->defname, "compression") == 0) { + *compression = defGetString(def); + } else if (dbname && strcmp(def->defname, "dbname") == 0) { + *dbname = defGetString(def); + if (*dbname[0] == '\0') { + *dbname = DEFAULT_DBNAME; + } + } else if (tls && strcmp(def->defname, "secure") == 0) { + const char* val = defGetString(def); + + if (strcmp(val, "on") == 0 || strcmp(val, "true") == 0 || + strcmp(val, "yes") == 0 || strcmp(val, "1") == 0) { + *tls = CH_TLS_ON; + } else if ( + strcmp(val, "off") == 0 || strcmp(val, "false") == 0 || + strcmp(val, "no") == 0 || strcmp(val, "0") == 0 + ) { + *tls = CH_TLS_OFF; + } + /* else: "auto" or anything else → CH_TLS_AUTO (already 0) */ + } else if ( + min_tls_version && strcmp(def->defname, "min_tls_version") == 0 + ) { + /* invalid values rejected by the validator; ignore here */ + parse_min_tls_version(defGetString(def), min_tls_version); + } + } + } } - /* * Parse options as key/value pairs. Used for connection parameters and * ClickHouse settings. Based on the Postgres conninfo_parse() function. The @@ -428,268 +428,294 @@ chfdw_extract_options(List * defelems, char **driver, char **host, int *port, * * Returns a PostgreSQL List containing DefElem cells. */ -List * -chfdw_parse_options(const char *options_string, bool with_comma, bool with_equal) -{ - char *pname; - char *pval; - char *buf; - char *cp; - char *cp2; - List *options = NIL; - - /* Need a modifiable copy of the input string */ - buf = pstrdup(options_string); - cp = buf; - - while (*cp) - { - /* Skip blanks before the parameter name */ - if (isspace((unsigned char) *cp)) - { - cp++; - continue; - } - - /* Get the parameter name */ - pname = cp; - while (*cp) - { - if (with_equal && *cp == '=') - break; - if (isspace((unsigned char) *cp)) - { - *cp++ = '\0'; - while (*cp) - { - if (!isspace((unsigned char) *cp)) - break; - cp++; - } - break; - } - cp++; - } - - if (with_equal) - { - /* Check that there is a following '=' */ - if (*cp != '=') - ereport(ERROR, - errcode(ERRCODE_SYNTAX_ERROR), - errmsg("pg_clickhouse: missing \"=\" after \"%s\" in options string", pname)); - *cp++ = '\0'; - - /* Skip blanks after the '=' */ - while (isspace((unsigned char) *cp)) - cp++; - } - - /* Get the parameter value */ - pval = cp; - - if (*cp != '\'') - { - cp2 = pval; - while (*cp) - { - if (isspace((unsigned char) *cp)) - { - if (with_comma) - { - while (isspace((unsigned char) *cp)) - cp++; - - if (*cp != ',' && *cp != '\0') - ereport(ERROR, - errcode(ERRCODE_SYNTAX_ERROR), - errmsg("pg_clickhouse: missing comma after \"%s\" value in options string", pname)); - while (isspace((unsigned char) *cp)) - cp++; - } - else - *cp++ = '\0'; - break; - } - if (*cp == ',' && with_comma) - { - *cp++ = '\0'; - break; - } - if (*cp == '\\') - { - cp++; - if (*cp != '\0') - *cp2++ = *cp++; - } - else - *cp2++ = *cp++; - } - *cp2 = '\0'; - if (cp2 == pval) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("pg_clickhouse: missing value for parameter \"%s\" in options string", pname))); - } - else - { - cp2 = pval; - cp++; - for (;;) - { - if (*cp == '\0') - ereport(ERROR, - errcode(ERRCODE_SYNTAX_ERROR), - errmsg("pg_clickhouse: unterminated quoted string in options string")); - if (*cp == '\\') - { - cp++; - if (*cp != '\0') - *cp2++ = *cp++; - continue; - } - if (*cp == '\'') - { - *cp2 = '\0'; - cp++; - break; - } - *cp2++ = *cp++; - } - if (with_comma) - { - /* Make sure there's a trailing comma or end of the input. */ - while (isspace((unsigned char) *cp)) - cp++; - if (*cp == ',') - cp++; - else if (*cp != '\0') - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("pg_clickhouse: missing comma after \"%s\" value in options string", pname))); - } - } - - /* - * Now that we have the name and the value, store the record. - */ - options = lappend(options, makeDefElem(pstrdup(pname), (Node *) makeString(pstrdup(pval)), -1)); - } - - return options; +List* +chfdw_parse_options(const char* options_string, bool with_comma, bool with_equal) { + char* pname; + char* pval; + char* buf; + char* cp; + char* cp2; + List* options = NIL; + + /* Need a modifiable copy of the input string */ + buf = pstrdup(options_string); + cp = buf; + + while (*cp) { + /* Skip blanks before the parameter name */ + if (isspace((unsigned char)*cp)) { + cp++; + continue; + } + + /* Get the parameter name */ + pname = cp; + while (*cp) { + if (with_equal && *cp == '=') { + break; + } + if (isspace((unsigned char)*cp)) { + *cp++ = '\0'; + while (*cp) { + if (!isspace((unsigned char)*cp)) { + break; + } + cp++; + } + break; + } + cp++; + } + + if (with_equal) { + /* Check that there is a following '=' */ + if (*cp != '=') { + ereport( + ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "pg_clickhouse: missing \"=\" after \"%s\" in options string", + pname + ) + ); + } + *cp++ = '\0'; + + /* Skip blanks after the '=' */ + while (isspace((unsigned char)*cp)) { + cp++; + } + } + + /* Get the parameter value */ + pval = cp; + + if (*cp != '\'') { + cp2 = pval; + while (*cp) { + if (isspace((unsigned char)*cp)) { + if (with_comma) { + while (isspace((unsigned char)*cp)) { + cp++; + } + + if (*cp != ',' && *cp != '\0') { + ereport( + ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "pg_clickhouse: missing comma after \"%s\" value " + "in options string", + pname + ) + ); + } + while (isspace((unsigned char)*cp)) { + cp++; + } + } else { + *cp++ = '\0'; + } + break; + } + if (*cp == ',' && with_comma) { + *cp++ = '\0'; + break; + } + if (*cp == '\\') { + cp++; + if (*cp != '\0') { + *cp2++ = *cp++; + } + } else { + *cp2++ = *cp++; + } + } + *cp2 = '\0'; + if (cp2 == pval) { + ereport( + ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "pg_clickhouse: missing value for parameter \"%s\" in options " + "string", + pname + )) + ); + } + } else { + cp2 = pval; + cp++; + for (;;) { + if (*cp == '\0') { + ereport( + ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "pg_clickhouse: unterminated quoted string in options " + "string" + ) + ); + } + if (*cp == '\\') { + cp++; + if (*cp != '\0') { + *cp2++ = *cp++; + } + continue; + } + if (*cp == '\'') { + *cp2 = '\0'; + cp++; + break; + } + *cp2++ = *cp++; + } + if (with_comma) { + /* Make sure there's a trailing comma or end of the input. */ + while (isspace((unsigned char)*cp)) { + cp++; + } + if (*cp == ',') { + cp++; + } else if (*cp != '\0') { + ereport( + ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "pg_clickhouse: missing comma after \"%s\" value in " + "options string", + pname + )) + ); + } + } + } + + /* + * Now that we have the name and the value, store the record. + */ + options = lappend( + options, makeDefElem(pstrdup(pname), (Node*)makeString(pstrdup(pval)), -1) + ); + } + + return options; } /* * Return the current list of current session settings parsed from the * session_settings GUC. */ -kv_list * -chfdw_get_session_settings(void) -{ - return ch_session_settings_list; +kv_list* +chfdw_get_session_settings(void) { + return ch_session_settings_list; } /* * Return the current value of the `pushdown_regex` GUC. */ bool -chfdw_pushdown_regex_ok(void) -{ - return ch_pushdown_regex; +chfdw_pushdown_regex_ok(void) { + return ch_pushdown_regex; } /* * Validates the provided settings key/value pairs. */ static bool -chfdw_check_settings_guc(char **newval, void **extra, GucSource source) -{ - /* - * The value may be an empty string, so we have to accept that value. - * Leave extra unset; chfdw_settings_assign_hook() will assign NULL to - * ch_session_settings_list. - */ - if (*newval == NULL || *newval[0] == '\0') - return true; - - /* Make sure we can parse the settings. */ - List *list = chfdw_parse_options(*newval, true, false); - - if (!list) - return false; - - /* Convert them into a guc_malloc'd kv_list. */ - kv_list *settings = new_kv_list_from_pg_list(list, kv_pair_guc_malloc); - - list_free(list); - - if (!settings) - return false; - - /* All good; stash for chfdw_settings_assign_hook and return true. */ - *extra = settings; - return true; +chfdw_check_settings_guc(char** newval, void** extra, GucSource source) { + /* + * The value may be an empty string, so we have to accept that value. + * Leave extra unset; chfdw_settings_assign_hook() will assign NULL to + * ch_session_settings_list. + */ + if (*newval == NULL || *newval[0] == '\0') { + return true; + } + + /* Make sure we can parse the settings. */ + List* list = chfdw_parse_options(*newval, true, false); + + if (!list) { + return false; + } + + /* Convert them into a guc_malloc'd kv_list. */ + kv_list* settings = new_kv_list_from_pg_list(list, kv_pair_guc_malloc); + + list_free(list); + + if (!settings) { + return false; + } + + /* All good; stash for chfdw_settings_assign_hook and return true. */ + *extra = settings; + return true; } /* * Assigns the kv_list stored in extra to the ch_session_settings_list global. */ static void -chfdw_settings_assign_hook(const char *newval, void *extra) -{ - /* - * From PostgreSQL's POV: (a) failure here is not acceptable, and (b) it - * is not necessarily called inside a transaction, so e.g. catalog lookups - * are not okay. IOW, keep it as simple as possible, and leave error - * returning behavior to chfdw_check_settings_guc(). - */ - ch_session_settings_list = (kv_list *) extra; +chfdw_settings_assign_hook(const char* newval, void* extra) { + /* + * From PostgreSQL's POV: (a) failure here is not acceptable, and (b) it + * is not necessarily called inside a transaction, so e.g. catalog lookups + * are not okay. IOW, keep it as simple as possible, and leave error + * returning behavior to chfdw_check_settings_guc(). + */ + ch_session_settings_list = (kv_list*)extra; } /* * Module load callback */ void -_PG_init(void) -{ - /* - * Key/value pairs for ClickHouse session settings. The format - * - * key = 'val', key = 'val' - * - * Spaces are optional. Full list of options: - * https://clickhouse.com/docs/operations/settings/settings - * - */ - DefineCustomStringVariable("pg_clickhouse.session_settings", - "Sets the default ClickHouse session settings.", - NULL, - &ch_session_settings, - "join_use_nulls 1, group_by_use_nulls 1, final 1", - PGC_USERSET, - 0, - chfdw_check_settings_guc, - chfdw_settings_assign_hook, - NULL); - - DefineCustomBoolVariable("pg_clickhouse.pushdown_regex", - "Whether to push down regular expression operations.", - "By default, pg_clickhouse attempts to push down regular " - "expression functions and operations, which is fine for " - "simple expressions. However, ClickHouse and Postgres use " - "fundamentally different regular expression engines, so it " - "may not make sense to push them all down. Set this option " - "to false to prevent the Postgres regular expression functions " - "and operators from pushing down to ClickHouse.", - &ch_pushdown_regex, - true, - PGC_USERSET, - 0, - NULL, - NULL, - NULL); +_PG_init(void) { + /* + * Key/value pairs for ClickHouse session settings. The format + * + * key = 'val', key = 'val' + * + * Spaces are optional. Full list of options: + * https://clickhouse.com/docs/operations/settings/settings + * + */ + DefineCustomStringVariable( + "pg_clickhouse.session_settings", + "Sets the default ClickHouse session settings.", + NULL, + &ch_session_settings, + "join_use_nulls 1, group_by_use_nulls 1, final 1", + PGC_USERSET, + 0, + chfdw_check_settings_guc, + chfdw_settings_assign_hook, + NULL + ); + + DefineCustomBoolVariable( + "pg_clickhouse.pushdown_regex", + "Whether to push down regular expression operations.", + "By default, pg_clickhouse attempts to push down regular " + "expression functions and operations, which is fine for " + "simple expressions. However, ClickHouse and Postgres use " + "fundamentally different regular expression engines, so it " + "may not make sense to push them all down. Set this option " + "to false to prevent the Postgres regular expression functions " + "and operators from pushing down to ClickHouse.", + &ch_pushdown_regex, + true, + PGC_USERSET, + 0, + NULL, + NULL, + NULL + ); #if PG_VERSION_NUM >= 150000 - MarkGUCPrefixReserved("pg_clickhouse"); + MarkGUCPrefixReserved("pg_clickhouse"); #endif } diff --git a/src/parser.c b/src/parser.c index f2d8f49..4c0fdf6 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1,17 +1,21 @@ -#include +#include +#include #include +#include #include #include -#include -#include #include #include -static void ch_http_read_array_string_literal(ch_http_read_state * state); -static void ch_http_read_array(ch_http_read_state * state); -static int ch_http_read_eof(ch_http_read_state * state); -static void ch_http_parse_error(const char *msg); +static void +ch_http_read_array_string_literal(ch_http_read_state* state); +static void +ch_http_read_array(ch_http_read_state* state); +static int +ch_http_read_eof(ch_http_read_state* state); +static void +ch_http_parse_error(const char* msg); /* * The streaming path can pass the parser a bounded slice rather than a @@ -19,32 +23,31 @@ static void ch_http_parse_error(const char *msg); * '\0' checks. */ inline static int -ch_http_read_eof(ch_http_read_state * state) -{ - state->done = true; - return CH_EOF; +ch_http_read_eof(ch_http_read_state* state) { + state->done = true; + return CH_EOF; } inline static void -ch_http_parse_error(const char *msg) -{ - ereport(ERROR, - (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("pg_clickhouse: %s", msg))); +ch_http_parse_error(const char* msg) { + ereport( + ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("pg_clickhouse: %s", msg)) + ); } void -ch_http_read_state_init(ch_http_read_state * state, char *data, size_t datalen) -{ - state->data = datalen > 0 ? data : NULL; - state->datalen = datalen; - state->curpos = 0; - state->done = false; - state->is_null = false; - if (state->val.data == NULL) - initStringInfo(&state->val); - else - resetStringInfo(&state->val); +ch_http_read_state_init(ch_http_read_state* state, char* data, size_t datalen) { + state->data = datalen > 0 ? data : NULL; + state->datalen = datalen; + state->curpos = 0; + state->done = false; + state->is_null = false; + if (state->val.data == NULL) { + initStringInfo(&state->val); + } else { + resetStringInfo(&state->val); + } } /* @@ -60,110 +63,106 @@ ch_http_read_state_init(ch_http_read_state * state, char *data, size_t datalen) * makes the call. */ int -ch_http_read_next(ch_http_read_state * state, bool is_array) -{ - char *data = state->data; +ch_http_read_next(ch_http_read_state* state, bool is_array) { + char* data = state->data; - if (state->done) - return CH_EOF; - if (data == NULL) - return ch_http_read_eof(state); + if (state->done) { + return CH_EOF; + } + if (data == NULL) { + return ch_http_read_eof(state); + } - resetStringInfo(&state->val); - state->is_null = false; - if (state->curpos >= state->datalen) - return ch_http_read_eof(state); + resetStringInfo(&state->val); + state->is_null = false; + if (state->curpos >= state->datalen) { + return ch_http_read_eof(state); + } - /* - * Detect the wire NULL marker. ClickHouse's TabSeparated format encodes - * NULL as the bare 2-byte sequence `\N`. A literal backslash in data is - * sent as `\\`, so legitimate non-null output never contains `\N` at the - * start of a field followed by a delimiter. Detect it here, before - * unescaping, so a non-null String value whose unescaped content happens - * to be `\N` is not collapsed with the NULL marker. - */ - if (state->curpos + 1 < state->datalen - && data[state->curpos] == '\\' - && data[state->curpos + 1] == 'N' - && (state->curpos + 2 == state->datalen - || data[state->curpos + 2] == '\t' - || data[state->curpos + 2] == '\n')) - { - state->is_null = true; - state->curpos += 2; - } - else if (is_array && data[state->curpos] == '[') - /* Parse array literal. */ - ch_http_read_array(state); - else - { - while (state->curpos < state->datalen && - data[state->curpos] != '\t' && - data[state->curpos] != '\n') - { - if (data[state->curpos] == '\\') - { - state->curpos++; - if (state->curpos >= state->datalen) - return ch_http_read_eof(state); - /* unescape some sequences */ - switch (data[state->curpos]) - { - case 'n': - appendStringInfoChar(&state->val, '\n'); - break; - case 't': - appendStringInfoChar(&state->val, '\t'); - break; - case '0': - appendStringInfoChar(&state->val, '\0'); - break; - case 'r': - appendStringInfoChar(&state->val, '\r'); - break; - case 'b': - appendStringInfoChar(&state->val, '\b'); - break; - case 'f': - appendStringInfoChar(&state->val, '\f'); - break; - case 'N': - /* NULL (format_tsv_null_representation) */ - appendStringInfoString(&state->val, "\\N"); - break; - default: - appendStringInfoChar(&state->val, data[state->curpos]); - } - state->curpos++; - } - else - appendStringInfoChar(&state->val, data[state->curpos++]); - } - } + /* + * Detect the wire NULL marker. ClickHouse's TabSeparated format encodes + * NULL as the bare 2-byte sequence `\N`. A literal backslash in data is + * sent as `\\`, so legitimate non-null output never contains `\N` at the + * start of a field followed by a delimiter. Detect it here, before + * unescaping, so a non-null String value whose unescaped content happens + * to be `\N` is not collapsed with the NULL marker. + */ + if (state->curpos + 1 < state->datalen && data[state->curpos] == '\\' && + data[state->curpos + 1] == 'N' && + (state->curpos + 2 == state->datalen || data[state->curpos + 2] == '\t' || + data[state->curpos + 2] == '\n')) { + state->is_null = true; + state->curpos += 2; + } else if (is_array && data[state->curpos] == '[') { + /* Parse array literal. */ + ch_http_read_array(state); + } else { + while (state->curpos < state->datalen && data[state->curpos] != '\t' && + data[state->curpos] != '\n') { + if (data[state->curpos] == '\\') { + state->curpos++; + if (state->curpos >= state->datalen) { + return ch_http_read_eof(state); + } + /* unescape some sequences */ + switch (data[state->curpos]) { + case 'n': + appendStringInfoChar(&state->val, '\n'); + break; + case 't': + appendStringInfoChar(&state->val, '\t'); + break; + case '0': + appendStringInfoChar(&state->val, '\0'); + break; + case 'r': + appendStringInfoChar(&state->val, '\r'); + break; + case 'b': + appendStringInfoChar(&state->val, '\b'); + break; + case 'f': + appendStringInfoChar(&state->val, '\f'); + break; + case 'N': + /* NULL (format_tsv_null_representation) */ + appendStringInfoString(&state->val, "\\N"); + break; + default: + appendStringInfoChar(&state->val, data[state->curpos]); + } + state->curpos++; + } else { + appendStringInfoChar(&state->val, data[state->curpos++]); + } + } + } - if (state->curpos >= state->datalen) - return ch_http_read_eof(state); + if (state->curpos >= state->datalen) { + return ch_http_read_eof(state); + } - if (data[state->curpos] == '\t') - { - /* There are more fields. */ - state->curpos++; - return CH_CONT; - } + if (data[state->curpos] == '\t') { + /* There are more fields. */ + state->curpos++; + return CH_CONT; + } - /* Should be at the end of the line or the file. */ - if (data[state->curpos] != '\n') - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("unexpected byte (%d) after array", data[state->curpos]))); - } + /* Should be at the end of the line or the file. */ + if (data[state->curpos] != '\n') { + ereport( + ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("unexpected byte (%d) after array", data[state->curpos])) + ); + } - state->curpos++; - if (state->curpos >= state->datalen) - return ch_http_read_eof(state); + state->curpos++; + if (state->curpos >= state->datalen) { + return ch_http_read_eof(state); + } - return CH_EOL; + return CH_EOL; } /* @@ -181,78 +180,67 @@ ch_http_read_next(ch_http_read_state * state, bool is_array) * https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO */ void -ch_http_read_array_string_literal(ch_http_read_state * state) -{ - /* Postgres array string is double-quoted. */ - appendStringInfoChar(&state->val, '"'); - state->curpos++; +ch_http_read_array_string_literal(ch_http_read_state* state) { + /* Postgres array string is double-quoted. */ + appendStringInfoChar(&state->val, '"'); + state->curpos++; - while (state->curpos < state->datalen && - state->data[state->curpos] != '\'') - { - char ch = state->data[state->curpos]; + while (state->curpos < state->datalen && state->data[state->curpos] != '\'') { + char ch = state->data[state->curpos]; - if (ch == '"') - { - /* Escape double quotation mark. */ - appendStringInfoChar(&state->val, '\\'); - appendStringInfoChar(&state->val, ch); - state->curpos++; - } - else if (ch == '\\') - { - /* Emit the escaped character. */ - state->curpos++; - if (state->curpos >= state->datalen) - ch_http_parse_error("invalid array string"); + if (ch == '"') { + /* Escape double quotation mark. */ + appendStringInfoChar(&state->val, '\\'); + appendStringInfoChar(&state->val, ch); + state->curpos++; + } else if (ch == '\\') { + /* Emit the escaped character. */ + state->curpos++; + if (state->curpos >= state->datalen) { + ch_http_parse_error("invalid array string"); + } - switch (state->data[state->curpos]) - { - case '\\': - appendStringInfoChar(&state->val, - state->data[state->curpos]); - appendStringInfoChar(&state->val, - state->data[state->curpos]); - break; - case 'b': - appendStringInfoChar(&state->val, '\b'); - break; - case 'f': - appendStringInfoChar(&state->val, '\f'); - break; - case 'r': - appendStringInfoChar(&state->val, '\r'); - break; - case 'n': - appendStringInfoChar(&state->val, '\n'); - break; - case 't': - appendStringInfoChar(&state->val, '\t'); - break; - case '0': - appendStringInfoChar(&state->val, '\0'); - break; - default: - /* Includes ' and probably no other character. */ - appendStringInfoChar(&state->val, - state->data[state->curpos]); - } - state->curpos++; - } - else - { - /* Append any other character. */ - appendStringInfoChar(&state->val, ch); - state->curpos++; - } - } + switch (state->data[state->curpos]) { + case '\\': + appendStringInfoChar(&state->val, state->data[state->curpos]); + appendStringInfoChar(&state->val, state->data[state->curpos]); + break; + case 'b': + appendStringInfoChar(&state->val, '\b'); + break; + case 'f': + appendStringInfoChar(&state->val, '\f'); + break; + case 'r': + appendStringInfoChar(&state->val, '\r'); + break; + case 'n': + appendStringInfoChar(&state->val, '\n'); + break; + case 't': + appendStringInfoChar(&state->val, '\t'); + break; + case '0': + appendStringInfoChar(&state->val, '\0'); + break; + default: + /* Includes ' and probably no other character. */ + appendStringInfoChar(&state->val, state->data[state->curpos]); + } + state->curpos++; + } else { + /* Append any other character. */ + appendStringInfoChar(&state->val, ch); + state->curpos++; + } + } - if (state->curpos >= state->datalen || - state->data[state->curpos] != '\'') - ch_http_parse_error("invalid array string"); + if (state->curpos >= state->datalen || state->data[state->curpos] != '\'') { + ch_http_parse_error("invalid array string"); + } - appendStringInfoChar(&state->val, '"'); - state->curpos++; + appendStringInfoChar(&state->val, '"'); + state->curpos++; } /* @@ -271,37 +259,35 @@ ch_http_read_array_string_literal(ch_http_read_state * state) * https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO */ void -ch_http_read_array(ch_http_read_state * state) -{ - size_t balance = 1; +ch_http_read_array(ch_http_read_state* state) { + size_t balance = 1; - /* Postgres arrays are wrapped in { and }. */ - appendStringInfoChar(&state->val, '{'); - state->curpos++; + /* Postgres arrays are wrapped in { and }. */ + appendStringInfoChar(&state->val, '{'); + state->curpos++; - while (state->curpos < state->datalen && balance) - { - switch (state->data[state->curpos]) - { - case '\'': - ch_http_read_array_string_literal(state); - break; - case '[': - ++balance; - appendStringInfoChar(&state->val, '{'); - state->curpos++; - break; - case ']': - --balance; - appendStringInfoChar(&state->val, '}'); - state->curpos++; - break; - default: - appendStringInfoChar(&state->val, state->data[state->curpos]); - state->curpos++; - } - } + while (state->curpos < state->datalen && balance) { + switch (state->data[state->curpos]) { + case '\'': + ch_http_read_array_string_literal(state); + break; + case '[': + ++balance; + appendStringInfoChar(&state->val, '{'); + state->curpos++; + break; + case ']': + --balance; + appendStringInfoChar(&state->val, '}'); + state->curpos++; + break; + default: + appendStringInfoChar(&state->val, state->data[state->curpos]); + state->curpos++; + } + } - if (balance != 0) - ch_http_parse_error("malformed array literal"); + if (balance != 0) { + ch_http_parse_error("malformed array literal"); + } } diff --git a/src/pglink.c b/src/pglink.c index d34af1f..202c4ad 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -16,457 +16,507 @@ #include "utils/typcache.h" #include "utils/uuid.h" +#include "binary.h" #include "fdw.h" #include "http.h" #include "http_streaming.h" -#include "binary.h" -#include #include +#include #include static bool initialized = false; -static void http_disconnect(void *conn); -static ch_cursor * http_simple_query(void *conn, const ch_query * query); -static ch_cursor * http_streaming_query(void *conn, const ch_query * query, - int32 fetch_size); -static void http_simple_insert(void *conn, const ch_query * query); -static void http_cursor_free(void *); -static void http_streaming_cursor_free(void *); -static Datum * http_fetch_row(ChFdwScanRowContext * ctx); -static Datum * http_streaming_fetch_row(ChFdwScanRowContext * ctx); -static Datum * http_fetch_row_from_state(ChFdwScanRowContext * ctx, - ch_http_read_state * state); -static void *http_prepare_insert(void *, ResultRelInfo *, List *, const ch_query *, char *); -static void http_insert_tuple(void *, TupleTableSlot *); -static void char_to_datum(ChFdwScanRowContext * ctx, int attnum, char *data, size_t len); -static void report_http_stream_query_failure(void *conn, const ch_query * query, - HttpStream * stream); - -static libclickhouse_methods http_methods = -{ - .disconnect = http_disconnect, - .simple_query = http_simple_query, - .fetch_row = http_fetch_row, - .prepare_insert = http_prepare_insert, - .insert_tuple = http_insert_tuple, - .streaming_query = http_streaming_query, - .streaming_fetch_row = http_streaming_fetch_row, +static void +http_disconnect(void* conn); +static ch_cursor* +http_simple_query(void* conn, const ch_query* query); +static ch_cursor* +http_streaming_query(void* conn, const ch_query* query, int32 fetch_size); +static void +http_simple_insert(void* conn, const ch_query* query); +static void +http_cursor_free(void*); +static void +http_streaming_cursor_free(void*); +static Datum* +http_fetch_row(ChFdwScanRowContext* ctx); +static Datum* +http_streaming_fetch_row(ChFdwScanRowContext* ctx); +static Datum* +http_fetch_row_from_state(ChFdwScanRowContext* ctx, ch_http_read_state* state); +static void* +http_prepare_insert(void*, ResultRelInfo*, List*, const ch_query*, char*); +static void +http_insert_tuple(void*, TupleTableSlot*); +static void +char_to_datum(ChFdwScanRowContext* ctx, int attnum, char* data, size_t len); +static void +report_http_stream_query_failure(void* conn, const ch_query* query, HttpStream* stream); + +static libclickhouse_methods http_methods = { + .disconnect = http_disconnect, + .simple_query = http_simple_query, + .fetch_row = http_fetch_row, + .prepare_insert = http_prepare_insert, + .insert_tuple = http_insert_tuple, + .streaming_query = http_streaming_query, + .streaming_fetch_row = http_streaming_fetch_row, }; -static void binary_disconnect(void *conn); -static ch_cursor * binary_simple_query(void *conn, const ch_query * query); -static void binary_cursor_free(void *cursor); -static bool binary_is_broken(const void *conn); +static void +binary_disconnect(void* conn); +static ch_cursor* +binary_simple_query(void* conn, const ch_query* query); +static void +binary_cursor_free(void* cursor); +static bool +binary_is_broken(const void* conn); /* static void binary_simple_insert(void *conn, const char *query); */ -static Datum * binary_fetch_row(ChFdwScanRowContext * ctx); -static void binary_insert_tuple(void *, TupleTableSlot * slot); -static void binary_finalize_insert(void *istate); -static void *binary_prepare_insert(void *, ResultRelInfo *, List *, - const ch_query * query, char *table_name); -static char *ch_escape_string(const char *s, size_t len); -static void ch_quote_literal_internal(char *dst, const char *src, size_t len); -extern char *ch_quote_literal(const char *rawstr); -extern const char *ch_quote_ident(const char *rawstr); - -static libclickhouse_methods binary_methods = -{ - .disconnect = binary_disconnect, - .simple_query = binary_simple_query, - .fetch_row = binary_fetch_row, - .prepare_insert = binary_prepare_insert, - .insert_tuple = binary_insert_tuple, - .finalize_insert = binary_finalize_insert, - .streaming_query = NULL, - .streaming_fetch_row = NULL, - .is_broken = binary_is_broken, +static Datum* +binary_fetch_row(ChFdwScanRowContext* ctx); +static void +binary_insert_tuple(void*, TupleTableSlot* slot); +static void +binary_finalize_insert(void* istate); +static void* +binary_prepare_insert( + void*, + ResultRelInfo*, + List*, + const ch_query* query, + char* table_name +); +static char* +ch_escape_string(const char* s, size_t len); +static void +ch_quote_literal_internal(char* dst, const char* src, size_t len); +extern char* +ch_quote_literal(const char* rawstr); +extern const char* +ch_quote_ident(const char* rawstr); + +static libclickhouse_methods binary_methods = { + .disconnect = binary_disconnect, + .simple_query = binary_simple_query, + .fetch_row = binary_fetch_row, + .prepare_insert = binary_prepare_insert, + .insert_tuple = binary_insert_tuple, + .finalize_insert = binary_finalize_insert, + .streaming_query = NULL, + .streaming_fetch_row = NULL, + .is_broken = binary_is_broken, }; static int -http_progress_callback(void *clientp, double dltotal, double dlnow, - double ultotal, double ulnow) -{ - if (ProcDiePending || QueryCancelPending) - return 1; - - return 0; +http_progress_callback( + void* clientp, + double dltotal, + double dlnow, + double ultotal, + double ulnow +) { + if (ProcDiePending || QueryCancelPending) { + return 1; + } + + return 0; } static bool -is_canceled(void) -{ - /* this variable is bool on pg < 12, but sig_atomic_t on above versions */ - if (QueryCancelPending) - return true; +is_canceled(void) { + /* this variable is bool on pg < 12, but sig_atomic_t on above versions */ + if (QueryCancelPending) { + return true; + } - return false; + return false; } ch_connection -chfdw_http_connect(ch_connection_details * details) -{ - ch_connection res; - ch_http_connection_t *conn; - - if (!initialized) - { - initialized = true; - ch_http_init(0, (uint32_t) MyProcPid); - } - - /* - * Since http.c will set the database name in a plain text header, we - * cannot allow line endings because they could allow header injection. - */ - if (details->dbname) - { - for (char *c = details->dbname; *c != '\0'; c++) - { - if (*c == '\n' || *c == '\r') - ereport(ERROR, - errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: unsupported line ending character in database name"), - errdetail("Invalid database name: %s", ch_quote_literal(details->dbname))); - } - } - - conn = ch_http_connection(details); - if (conn == NULL) - { - char *error = ch_http_last_error(); - - if (error == NULL) - error = "undefined"; - - ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("could not connect to server: %s", error))); - } - - res.conn = conn; - res.methods = &http_methods; - res.is_binary = false; - return res; +chfdw_http_connect(ch_connection_details* details) { + ch_connection res; + ch_http_connection_t* conn; + + if (!initialized) { + initialized = true; + ch_http_init(0, (uint32_t)MyProcPid); + } + + /* + * Since http.c will set the database name in a plain text header, we + * cannot allow line endings because they could allow header injection. + */ + if (details->dbname) { + for (char* c = details->dbname; *c != '\0'; c++) { + if (*c == '\n' || *c == '\r') { + ereport( + ERROR, + errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), + errmsg( + "pg_clickhouse: unsupported line ending character in database " + "name" + ), + errdetail( + "Invalid database name: %s", ch_quote_literal(details->dbname) + ) + ); + } + } + } + + conn = ch_http_connection(details); + if (conn == NULL) { + char* error = ch_http_last_error(); + + if (error == NULL) { + error = "undefined"; + } + + ereport( + ERROR, + (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), + errmsg("could not connect to server: %s", error)) + ); + } + + res.conn = conn; + res.methods = &http_methods; + res.is_binary = false; + return res; } /* * Disconnect any open connection for a connection cache entry. */ static void -http_disconnect(void *conn) -{ - if (conn != NULL) - ch_http_close((ch_http_connection_t *) conn); +http_disconnect(void* conn) { + if (conn != NULL) { + ch_http_close((ch_http_connection_t*)conn); + } } /* * Return text before version mentioning */ -static char * -format_error(char *errstring) -{ - size_t n = strlen(errstring); - - for (int i = 0; i < n; i++) - { - if (strncmp(errstring + i, "version", 7) == 0) - return pnstrdup(errstring, i - 2); - } - - /* - * For some reason ClickHouse 25.12 added a newline to an auth failure - * error. Strip it out. - */ - if (n > 0 && errstring[n - 1] == '\n') - errstring[--n] = '\0'; - - return errstring; +static char* +format_error(char* errstring) { + size_t n = strlen(errstring); + + for (int i = 0; i < n; i++) { + if (strncmp(errstring + i, "version", 7) == 0) { + return pnstrdup(errstring, i - 2); + } + } + + /* + * For some reason ClickHouse 25.12 added a newline to an auth failure + * error. Strip it out. + */ + if (n > 0 && errstring[n - 1] == '\n') { + errstring[--n] = '\0'; + } + + return errstring; } static void -kill_query(void *conn, const char *query_id) -{ - ch_http_response_t *resp; - ch_query query = new_query(psprintf( - "kill query where query_id=%s", - ch_quote_literal(query_id)), 0, NULL, NULL, NULL); - - ch_http_set_progress_func(NULL); - resp = ch_http_simple_query(conn, &query); - if (resp != NULL) - ch_http_response_free(resp); +kill_query(void* conn, const char* query_id) { + ch_http_response_t* resp; + ch_query query = new_query( + psprintf("kill query where query_id=%s", ch_quote_literal(query_id)), + 0, + NULL, + NULL, + NULL + ); + + ch_http_set_progress_func(NULL); + resp = ch_http_simple_query(conn, &query); + if (resp != NULL) { + ch_http_response_free(resp); + } } static void -report_http_stream_query_failure(void *conn, const ch_query * query, - HttpStream * stream) -{ - long status = ch_http_stream_status(stream); - - PG_TRY(); - { - if (status == CH_HTTP_STATUS_CANCELED) - { - char qid[CH_HTTP_QUERY_ID_LEN]; - - memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); - kill_query(conn, qid); - ereport(ERROR, - (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: query was aborted"))); - } - else if (status == CH_HTTP_STATUS_TRANSPORT_ERROR) - { - const char *err = ch_http_stream_error(stream); - - ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: communication error: %s", - err ? err : "connection error"))); - } - else - { - char *error = pnstrdup(ch_http_stream_buffer(stream), - ch_http_stream_available(stream)); - - ereport(ERROR, - (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), - status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", - query->sql), - errcontext("HTTP status code: %li", status))); - } - } - PG_FINALLY(); - { - ch_http_stream_end(stream); - } - PG_END_TRY(); +report_http_stream_query_failure( + void* conn, + const ch_query* query, + HttpStream* stream +) { + long status = ch_http_stream_status(stream); + + PG_TRY(); + { + if (status == CH_HTTP_STATUS_CANCELED) { + char qid[CH_HTTP_QUERY_ID_LEN]; + + memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); + kill_query(conn, qid); + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: query was aborted")) + ); + } else if (status == CH_HTTP_STATUS_TRANSPORT_ERROR) { + const char* err = ch_http_stream_error(stream); + + ereport( + ERROR, + (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), + errmsg( + "pg_clickhouse: communication error: %s", + err ? err : "connection error" + )) + ); + } else { + char* error = pnstrdup( + ch_http_stream_buffer(stream), ch_http_stream_available(stream) + ); + + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", format_error(error)), + status < 404 + ? 0 + : errdetail_internal("Remote Query: %.64000s", query->sql), + errcontext("HTTP status code: %li", status)) + ); + } + } + PG_FINALLY(); + { ch_http_stream_end(stream); } + PG_END_TRY(); } -static ch_cursor * -http_simple_query(void *conn, const ch_query * query) -{ - int attempts = 0; - MemoryContext tempcxt = NULL, - oldcxt; - ch_cursor *cursor; - ch_http_response_t *resp; +static ch_cursor* +http_simple_query(void* conn, const ch_query* query) { + int attempts = 0; + MemoryContext tempcxt = NULL, oldcxt; + ch_cursor* cursor; + ch_http_response_t* resp; - ch_http_set_progress_func(http_progress_callback); + ch_http_set_progress_func(http_progress_callback); again: - resp = ch_http_simple_query(conn, query); - if (resp == NULL) - { - ereport(ERROR, - (errcode(ERRCODE_FDW_OUT_OF_MEMORY), - errmsg("out of memory"))); - } - - attempts++; - if (resp->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR) - { - char *error = pnstrdup(resp->data, resp->datasize); - - ch_http_response_free(resp); - - if (attempts < 3) - goto again; - - ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: communication error: %s", error))); - } - else if (resp->http_status == CH_HTTP_STATUS_CANCELED) - { - kill_query(conn, resp->query_id); - ch_http_response_free(resp); - - ereport(ERROR, - (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: query was aborted"))); - } - else if (resp->http_status != CH_HTTP_STATUS_OK) - { - char *error = pnstrdup(resp->data, resp->datasize); - long status = resp->http_status; - - ch_http_response_free(resp); - - ereport(ERROR, ( - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), - status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", query->sql), - errcontext("HTTP status code: %li", status) - )); - } - - PG_TRY(); - { - /* - * If any palloc below throws, use PG_CATCH to free the Curl response. - */ - tempcxt = AllocSetContextCreate(PortalContext, "pg_clickhouse cursor", - ALLOCSET_DEFAULT_SIZES); - oldcxt = MemoryContextSwitchTo(tempcxt); - - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = resp; - cursor->read_state = palloc0(sizeof(ch_http_read_state)); - cursor->query = pstrdup(query->sql); - cursor->request_time = resp->pretransfer_time * 1000; - cursor->total_time = resp->total_time * 1000; - ch_http_read_state_init(cursor->read_state, resp->data, resp->datasize); - - cursor->memcxt = tempcxt; - cursor->callback.func = http_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - MemoryContextSwitchTo(oldcxt); - } - PG_CATCH(); - { - if (resp) - ch_http_response_free(resp); - if (tempcxt) - MemoryContextDelete(tempcxt); - PG_RE_THROW(); - } - PG_END_TRY(); - - return cursor; + resp = ch_http_simple_query(conn, query); + if (resp == NULL) { + ereport(ERROR, (errcode(ERRCODE_FDW_OUT_OF_MEMORY), errmsg("out of memory"))); + } + + attempts++; + if (resp->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR) { + char* error = pnstrdup(resp->data, resp->datasize); + + ch_http_response_free(resp); + + if (attempts < 3) { + goto again; + } + + ereport( + ERROR, + (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), + errmsg("pg_clickhouse: communication error: %s", error)) + ); + } else if (resp->http_status == CH_HTTP_STATUS_CANCELED) { + kill_query(conn, resp->query_id); + ch_http_response_free(resp); + + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: query was aborted")) + ); + } else if (resp->http_status != CH_HTTP_STATUS_OK) { + char* error = pnstrdup(resp->data, resp->datasize); + long status = resp->http_status; + + ch_http_response_free(resp); + + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", format_error(error)), + status < 404 ? 0 + : errdetail_internal("Remote Query: %.64000s", query->sql), + errcontext("HTTP status code: %li", status)) + ); + } + + PG_TRY(); + { + /* + * If any palloc below throws, use PG_CATCH to free the Curl response. + */ + tempcxt = AllocSetContextCreate( + PortalContext, "pg_clickhouse cursor", ALLOCSET_DEFAULT_SIZES + ); + oldcxt = MemoryContextSwitchTo(tempcxt); + + cursor = palloc0(sizeof(ch_cursor)); + cursor->conn = conn; + cursor->query_response = resp; + cursor->read_state = palloc0(sizeof(ch_http_read_state)); + cursor->query = pstrdup(query->sql); + cursor->request_time = resp->pretransfer_time * 1000; + cursor->total_time = resp->total_time * 1000; + ch_http_read_state_init(cursor->read_state, resp->data, resp->datasize); + + cursor->memcxt = tempcxt; + cursor->callback.func = http_cursor_free; + cursor->callback.arg = cursor; + MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); + MemoryContextSwitchTo(oldcxt); + } + PG_CATCH(); + { + if (resp) { + ch_http_response_free(resp); + } + if (tempcxt) { + MemoryContextDelete(tempcxt); + } + PG_RE_THROW(); + } + PG_END_TRY(); + + return cursor; } static void -http_simple_insert(void *conn, const ch_query * query) -{ - ch_http_response_t *resp = ch_http_simple_query(conn, query); - - if (resp == NULL) - { - char *error = ch_http_last_error(); - - if (error == NULL) - error = "undefined"; - - ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: communication error: %s", error))); - } - - if (resp->http_status != CH_HTTP_STATUS_OK) - { - char *error = pnstrdup(resp->data, resp->datasize); - long status = resp->http_status; - - ch_http_response_free(resp); - - ereport(ERROR, ( - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), - status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", query->sql), - errcontext("HTTP status code: %li", status) - )); - } - - ch_http_response_free(resp); +http_simple_insert(void* conn, const ch_query* query) { + ch_http_response_t* resp = ch_http_simple_query(conn, query); + + if (resp == NULL) { + char* error = ch_http_last_error(); + + if (error == NULL) { + error = "undefined"; + } + + ereport( + ERROR, + (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), + errmsg("pg_clickhouse: communication error: %s", error)) + ); + } + + if (resp->http_status != CH_HTTP_STATUS_OK) { + char* error = pnstrdup(resp->data, resp->datasize); + long status = resp->http_status; + + ch_http_response_free(resp); + + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", format_error(error)), + status < 404 ? 0 + : errdetail_internal("Remote Query: %.64000s", query->sql), + errcontext("HTTP status code: %li", status)) + ); + } + + ch_http_response_free(resp); } inline static void -http_cursor_free(void *c) -{ - ch_http_response_free(((ch_cursor *) c)->query_response); +http_cursor_free(void* c) { + ch_http_response_free(((ch_cursor*)c)->query_response); } inline static void -http_streaming_cursor_free(void *c) -{ - if (((ch_cursor *) c)->query_response) - ch_http_stream_end(((ch_cursor *) c)->query_response); +http_streaming_cursor_free(void* c) { + if (((ch_cursor*)c)->query_response) { + ch_http_stream_end(((ch_cursor*)c)->query_response); + } } /* * Create a streaming cursor with row-aligned batches of ~fetch_size bytes * via CURL pause/resume, keeping memory proportional to batch size. */ -static ch_cursor * -http_streaming_query(void *conn, const ch_query * query, int32 fetch_size) -{ - int attempts = 0; - MemoryContext tempcxt = NULL, - oldcxt; - ch_cursor *cursor; - HttpStream *stream; +static ch_cursor* +http_streaming_query(void* conn, const ch_query* query, int32 fetch_size) { + int attempts = 0; + MemoryContext tempcxt = NULL, oldcxt; + ch_cursor* cursor; + HttpStream* stream; - ch_http_set_progress_func(http_progress_callback); + ch_http_set_progress_func(http_progress_callback); again: - stream = ch_http_stream_begin(conn, query, fetch_size); - if (stream == NULL) - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("pg_clickhouse: failed to initialize HTTP stream"))); - - attempts++; - if (ch_http_stream_status(stream) == CH_HTTP_STATUS_TRANSPORT_ERROR) - { - if (attempts < 3) - { - ch_http_stream_end(stream); - goto again; - } - } - if (ch_http_stream_status(stream) != CH_HTTP_STATUS_OK) - report_http_stream_query_failure(conn, query, stream); - - PG_TRY(); - { - /* - * If any palloc below throws, clean up the stream which is not - * tracked by a memory context yet. - */ - tempcxt = AllocSetContextCreate(PortalContext, - "pg_clickhouse streaming cursor", - ALLOCSET_DEFAULT_SIZES); - oldcxt = MemoryContextSwitchTo(tempcxt); - - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = stream; - cursor->read_state = palloc0(sizeof(ch_http_read_state)); - cursor->query = pstrdup(query->sql); - cursor->request_time = ch_http_stream_request_time(stream); - cursor->total_time = ch_http_stream_total_time(stream); - - ch_http_read_state_init(cursor->read_state, - ch_http_stream_buffer(stream), - ch_http_stream_available(stream)); - - cursor->memcxt = tempcxt; - cursor->callback.func = http_streaming_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - - MemoryContextSwitchTo(oldcxt); - - /* Ownership transferred to the cursor callback */ - stream = NULL; - } - PG_CATCH(); - { - if (stream) - ch_http_stream_end(stream); - if (tempcxt) - MemoryContextDelete(tempcxt); - PG_RE_THROW(); - } - PG_END_TRY(); - - return cursor; + stream = ch_http_stream_begin(conn, query, fetch_size); + if (stream == NULL) { + ereport( + ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("pg_clickhouse: failed to initialize HTTP stream")) + ); + } + + attempts++; + if (ch_http_stream_status(stream) == CH_HTTP_STATUS_TRANSPORT_ERROR) { + if (attempts < 3) { + ch_http_stream_end(stream); + goto again; + } + } + if (ch_http_stream_status(stream) != CH_HTTP_STATUS_OK) { + report_http_stream_query_failure(conn, query, stream); + } + + PG_TRY(); + { + /* + * If any palloc below throws, clean up the stream which is not + * tracked by a memory context yet. + */ + tempcxt = AllocSetContextCreate( + PortalContext, "pg_clickhouse streaming cursor", ALLOCSET_DEFAULT_SIZES + ); + oldcxt = MemoryContextSwitchTo(tempcxt); + + cursor = palloc0(sizeof(ch_cursor)); + cursor->conn = conn; + cursor->query_response = stream; + cursor->read_state = palloc0(sizeof(ch_http_read_state)); + cursor->query = pstrdup(query->sql); + cursor->request_time = ch_http_stream_request_time(stream); + cursor->total_time = ch_http_stream_total_time(stream); + + ch_http_read_state_init( + cursor->read_state, + ch_http_stream_buffer(stream), + ch_http_stream_available(stream) + ); + + cursor->memcxt = tempcxt; + cursor->callback.func = http_streaming_cursor_free; + cursor->callback.arg = cursor; + MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); + + MemoryContextSwitchTo(oldcxt); + + /* Ownership transferred to the cursor callback */ + stream = NULL; + } + PG_CATCH(); + { + if (stream) { + ch_http_stream_end(stream); + } + if (tempcxt) { + MemoryContextDelete(tempcxt); + } + PG_RE_THROW(); + } + PG_END_TRY(); + + return cursor; } /* @@ -474,127 +524,129 @@ http_streaming_query(void *conn, const ch_query * query, int32 fetch_size) * buffer and the transfer isn't done, pump more data from curl and * reinitialize the parser on the refilled buffer. */ -static Datum * -http_streaming_fetch_row(ChFdwScanRowContext * ctx) -{ - ch_cursor *cursor = ctx->cursor; - ch_http_read_state *state = cursor->read_state; - HttpStream *stream = cursor->query_response; - - /* Pump the next batch when the current one has been exhausted. */ - if (state->done || state->data == NULL) - { - /* Sync parse position: tell stream how far the parser advanced */ - ch_http_stream_advance(stream, state->curpos); - - if (ch_http_stream_pump(stream) < 0) - { - if (ch_http_stream_status(stream) == CH_HTTP_STATUS_CANCELED) - { - char qid[CH_HTTP_QUERY_ID_LEN]; - - memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); - ch_http_stream_end(stream); - cursor->query_response = NULL; - kill_query(cursor->conn, qid); - ereport(ERROR, - (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: query was aborted"))); - } - ereport(ERROR, - (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("pg_clickhouse: streaming error - %s", - ch_http_stream_error(stream) - ? ch_http_stream_error(stream) - : "unknown"))); - } - - /* Reinitialize parser on the (possibly compacted) buffer */ - ch_http_read_state_init(state, - ch_http_stream_buffer(stream), - ch_http_stream_available(stream)); - } - - return http_fetch_row_from_state(ctx, state); +static Datum* +http_streaming_fetch_row(ChFdwScanRowContext* ctx) { + ch_cursor* cursor = ctx->cursor; + ch_http_read_state* state = cursor->read_state; + HttpStream* stream = cursor->query_response; + + /* Pump the next batch when the current one has been exhausted. */ + if (state->done || state->data == NULL) { + /* Sync parse position: tell stream how far the parser advanced */ + ch_http_stream_advance(stream, state->curpos); + + if (ch_http_stream_pump(stream) < 0) { + if (ch_http_stream_status(stream) == CH_HTTP_STATUS_CANCELED) { + char qid[CH_HTTP_QUERY_ID_LEN]; + + memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); + ch_http_stream_end(stream); + cursor->query_response = NULL; + kill_query(cursor->conn, qid); + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: query was aborted")) + ); + } + ereport( + ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg( + "pg_clickhouse: streaming error - %s", + ch_http_stream_error(stream) ? ch_http_stream_error(stream) + : "unknown" + )) + ); + } + + /* Reinitialize parser on the (possibly compacted) buffer */ + ch_http_read_state_init( + state, ch_http_stream_buffer(stream), ch_http_stream_available(stream) + ); + } + + return http_fetch_row_from_state(ctx, state); } -static Datum * -http_fetch_row_from_state(ChFdwScanRowContext * ctx, ch_http_read_state * state) -{ - int rc = CH_CONT; - size_t attcount = list_length(ctx->retrieved_attrs); - Datum *values; - - /* All rows or empty table. */ - if (state->done || state->data == NULL) - return NULL; - - /* Special case: SELECT NULL. */ - if (attcount == 0) - { - Assert(ctx->values && ctx->nulls); - rc = ch_http_read_next(state, false); - if (rc != CH_CONT && state->is_null) - { - ctx->nulls[0] = true; - ctx->values[0] = (Datum) 0; - return ctx->values; - } - - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: unexpected response for a zero-column result"), - errdetail("Expected a NULL marker (\\N) in the TabSeparated response."))); - } - - /* - * Create Datums based on the retrieved_attrs for the TupleDesc. - * ctx->values and ctx->nulls must already be initialized with memory for - * ctx->tupdesc->natts Datums. - */ - if (ctx->tupdesc) - { - values = ctx->values; - ListCell *lc; - int i; - - Assert(ctx->values && ctx->nulls && ctx->attinmeta); - foreach(lc, ctx->retrieved_attrs) - { - Oid pgtype; - - i = lfirst_int(lc) - 1; - pgtype = TupleDescAttr(ctx->tupdesc, i)->atttypid; - rc = ch_http_read_next(state, type_is_array(pgtype)); - char_to_datum(ctx, i, - state->is_null ? NULL : state->val.data, - state->val.len); - } - } - /* No TupleDesc, everything is text. */ - else - { - values = palloc(attcount * sizeof(Datum)); - for (int idx = 0; idx < attcount; idx++) - { - rc = ch_http_read_next(state, false); - if (state->is_null) - values[idx] = (Datum) 0; - else - values[idx] = PointerGetDatum(cstring_to_text(state->val.data)); - } - } - - if (attcount > 0 && rc != CH_EOL && rc != CH_EOF) - { - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal("pg_clickhouse: columns mismatch"), - errdetail("Number of returned columns does not match " - "expected column count (%lu).", attcount))); - } - - return values; +static Datum* +http_fetch_row_from_state(ChFdwScanRowContext* ctx, ch_http_read_state* state) { + int rc = CH_CONT; + size_t attcount = list_length(ctx->retrieved_attrs); + Datum* values; + + /* All rows or empty table. */ + if (state->done || state->data == NULL) { + return NULL; + } + + /* Special case: SELECT NULL. */ + if (attcount == 0) { + Assert(ctx->values && ctx->nulls); + rc = ch_http_read_next(state, false); + if (rc != CH_CONT && state->is_null) { + ctx->nulls[0] = true; + ctx->values[0] = (Datum)0; + return ctx->values; + } + + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("pg_clickhouse: unexpected response for a zero-column result"), + errdetail("Expected a NULL marker (\\N) in the TabSeparated response.")) + ); + } + + /* + * Create Datums based on the retrieved_attrs for the TupleDesc. + * ctx->values and ctx->nulls must already be initialized with memory for + * ctx->tupdesc->natts Datums. + */ + if (ctx->tupdesc) { + values = ctx->values; + ListCell* lc; + int i; + + Assert(ctx->values && ctx->nulls && ctx->attinmeta); + foreach (lc, ctx->retrieved_attrs) { + Oid pgtype; + + i = lfirst_int(lc) - 1; + pgtype = TupleDescAttr(ctx->tupdesc, i)->atttypid; + rc = ch_http_read_next(state, type_is_array(pgtype)); + char_to_datum( + ctx, i, state->is_null ? NULL : state->val.data, state->val.len + ); + } + } + /* No TupleDesc, everything is text. */ + else { + values = palloc(attcount * sizeof(Datum)); + for (int idx = 0; idx < attcount; idx++) { + rc = ch_http_read_next(state, false); + if (state->is_null) { + values[idx] = (Datum)0; + } else { + values[idx] = PointerGetDatum(cstring_to_text(state->val.data)); + } + } + } + + if (attcount > 0 && rc != CH_EOL && rc != CH_EOF) { + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal("pg_clickhouse: columns mismatch"), + errdetail( + "Number of returned columns does not match " + "expected column count (%lu).", + attcount + )) + ); + } + + return values; } /* @@ -612,13 +664,12 @@ http_fetch_row_from_state(ChFdwScanRowContext * ctx, ch_http_read_state * state) * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, * which only cares about text. */ -static Datum * -http_fetch_row(ChFdwScanRowContext * ctx) -{ - ch_cursor *cursor = ctx->cursor; - ch_http_read_state *state = cursor->read_state; +static Datum* +http_fetch_row(ChFdwScanRowContext* ctx) { + ch_cursor* cursor = ctx->cursor; + ch_http_read_state* state = cursor->read_state; - return http_fetch_row_from_state(ctx, state); + return http_fetch_row_from_state(ctx, state); } /* @@ -627,116 +678,114 @@ http_fetch_row(ChFdwScanRowContext * ctx) * ctx->tupdesc and ctx->attinmeta. */ static void -char_to_datum(ChFdwScanRowContext * ctx, int attidx, char *data, size_t len) -{ - Oid pgtype = TupleDescAttr(ctx->tupdesc, attidx)->atttypid; - - if (data && (pgtype == TIMEOID || pgtype == TIMETZOID) - && data[strlen(data) - 1] == 'Z') - { - /* - * date_time_output_format=iso formats times as ISO timestamps. Remove - * the leading `YYYY-mm-ddT`. - */ - data += strlen("1970-01-01T"); - } - else if (pgtype == BYTEAOID) - { - /* Postgres input function won't work, we have raw data. */ - ctx->nulls[attidx] = data == NULL; - ctx->values[attidx] = PointerGetDatum((bytea *) cstring_to_text_with_len(data, len)); - return; - } - - /* Apply the input function even to nulls, to support domains */ - ctx->nulls[attidx] = data == NULL; - ctx->values[attidx] = InputFunctionCall(&ctx->attinmeta->attinfuncs[attidx], - data, - ctx->attinmeta->attioparams[attidx], - ctx->attinmeta->atttypmods[attidx]); +char_to_datum(ChFdwScanRowContext* ctx, int attidx, char* data, size_t len) { + Oid pgtype = TupleDescAttr(ctx->tupdesc, attidx)->atttypid; + + if (data && (pgtype == TIMEOID || pgtype == TIMETZOID) && + data[strlen(data) - 1] == 'Z') { + /* + * date_time_output_format=iso formats times as ISO timestamps. Remove + * the leading `YYYY-mm-ddT`. + */ + data += strlen("1970-01-01T"); + } else if (pgtype == BYTEAOID) { + /* Postgres input function won't work, we have raw data. */ + ctx->nulls[attidx] = data == NULL; + ctx->values[attidx] = + PointerGetDatum((bytea*)cstring_to_text_with_len(data, len)); + return; + } + + /* Apply the input function even to nulls, to support domains */ + ctx->nulls[attidx] = data == NULL; + ctx->values[attidx] = InputFunctionCall( + &ctx->attinmeta->attinfuncs[attidx], + data, + ctx->attinmeta->attioparams[attidx], + ctx->attinmeta->atttypmods[attidx] + ); } -text * -chfdw_http_fetch_raw_data(ch_cursor * cursor) -{ - ch_http_read_state *state = cursor->read_state; +text* +chfdw_http_fetch_raw_data(ch_cursor* cursor) { + ch_http_read_state* state = cursor->read_state; - if (state->data == NULL) - return NULL; + if (state->data == NULL) { + return NULL; + } - return cstring_to_text(state->data); + return cstring_to_text(state->data); } /* * Convert a Datum to a ClickHouse literal string. Returns NULL if the value * cannot be converted to a literal. */ -extern char * -chfdw_datum_to_ch_literal(Datum value, Oid type) -{ - if (type_is_array(type)) - return chfdw_array_to_ch_literal(value); - - switch (type) - { - case BOOLOID: - case INT2OID: - case INT4OID: - return psprintf("%d", DatumGetInt32(value)); - case INT8OID: - return psprintf(INT64_FORMAT, DatumGetInt64(value)); - case FLOAT4OID: - return psprintf("%f", DatumGetFloat4(value)); - case FLOAT8OID: - return psprintf("%f", DatumGetFloat8(value)); - case NUMERICOID: - return DatumGetCString(DirectFunctionCall1(numeric_out, value)); - case BPCHAROID: - case VARCHAROID: - case TEXTOID: - case JSONOID: - case JSONBOID: - case NAMEOID: - case BITOID: - case UUIDOID: - case INETOID: - { - char *text; - bool tl = false; - Oid typoutput = InvalidOid; - - getTypeOutputInfo(type, &typoutput, &tl); - text = OidOutputFunctionCall(typoutput, value); - return ch_escape_string(text, strlen(text)); - } - case BYTEAOID: - { - /* Copy all of the bytes into a ClickHouse literal string. */ - bytea *bytes = PG_DETOAST_DATUM(value); - - return ch_escape_string(VARDATA(bytes), VARSIZE_ANY_EXHDR(bytes)); - } - case DATEOID: - /* we expect Date on other side */ - return DatumGetCString(DirectFunctionCall1(ch_date_out, value)); - case TIMEOID: - { - /* we expect DateTime on other side */ - char *extval = DatumGetCString(DirectFunctionCall1(ch_time_out, value)); - char *retval = psprintf("1970-01-01 %s", extval); - - pfree(extval); - return retval; - } - case TIMESTAMPOID: - case TIMESTAMPTZOID: - /* we expect DateTime on other side */ - return DatumGetCString(DirectFunctionCall1(ch_timestamp_out, value)); - default: - ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("cannot convert value to clickhouse value"), - errhint("Value data type: %u", type))); - } +extern char* +chfdw_datum_to_ch_literal(Datum value, Oid type) { + if (type_is_array(type)) { + return chfdw_array_to_ch_literal(value); + } + + switch (type) { + case BOOLOID: + case INT2OID: + case INT4OID: + return psprintf("%d", DatumGetInt32(value)); + case INT8OID: + return psprintf(INT64_FORMAT, DatumGetInt64(value)); + case FLOAT4OID: + return psprintf("%f", DatumGetFloat4(value)); + case FLOAT8OID: + return psprintf("%f", DatumGetFloat8(value)); + case NUMERICOID: + return DatumGetCString(DirectFunctionCall1(numeric_out, value)); + case BPCHAROID: + case VARCHAROID: + case TEXTOID: + case JSONOID: + case JSONBOID: + case NAMEOID: + case BITOID: + case UUIDOID: + case INETOID: { + char* text; + bool tl = false; + Oid typoutput = InvalidOid; + + getTypeOutputInfo(type, &typoutput, &tl); + text = OidOutputFunctionCall(typoutput, value); + return ch_escape_string(text, strlen(text)); + } + case BYTEAOID: { + /* Copy all of the bytes into a ClickHouse literal string. */ + bytea* bytes = PG_DETOAST_DATUM(value); + + return ch_escape_string(VARDATA(bytes), VARSIZE_ANY_EXHDR(bytes)); + } + case DATEOID: + /* we expect Date on other side */ + return DatumGetCString(DirectFunctionCall1(ch_date_out, value)); + case TIMEOID: { + /* we expect DateTime on other side */ + char* extval = DatumGetCString(DirectFunctionCall1(ch_time_out, value)); + char* retval = psprintf("1970-01-01 %s", extval); + + pfree(extval); + return retval; + } + case TIMESTAMPOID: + case TIMESTAMPTZOID: + /* we expect DateTime on other side */ + return DatumGetCString(DirectFunctionCall1(ch_timestamp_out, value)); + default: + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("cannot convert value to clickhouse value"), + errhint("Value data type: %u", type)) + ); + } } /* @@ -744,188 +793,187 @@ chfdw_datum_to_ch_literal(Datum value, Oid type) * Construct values part of INSERT query */ static void -extend_insert_query(ch_http_insert_state * state, TupleTableSlot * slot) -{ +extend_insert_query(ch_http_insert_state* state, TupleTableSlot* slot) { #ifdef USE_ASSERT_CHECKING - int pindex = 0; + int pindex = 0; #endif - bool first = true; - - if (state->sql.len == 0) - appendStringInfoString(&state->sql, state->sql_begin); - - /* get following parameters from slot */ - if (slot != NULL && state->target_attrs != NIL) - { - ListCell *lc; - - foreach(lc, state->target_attrs) - { - int attnum = lfirst_int(lc); - Datum value; - Oid type; - bool isnull; - char *string; - - value = slot_getattr(slot, attnum, &isnull); - type = TupleDescAttr(slot->tts_tupleDescriptor, attnum - 1)->atttypid; - - if (!first) - appendStringInfoChar(&state->sql, '\t'); - first = false; - - if (isnull) - { - appendStringInfoString(&state->sql, "\\N"); + bool first = true; + + if (state->sql.len == 0) { + appendStringInfoString(&state->sql, state->sql_begin); + } + + /* get following parameters from slot */ + if (slot != NULL && state->target_attrs != NIL) { + ListCell* lc; + + foreach (lc, state->target_attrs) { + int attnum = lfirst_int(lc); + Datum value; + Oid type; + bool isnull; + char* string; + + value = slot_getattr(slot, attnum, &isnull); + type = TupleDescAttr(slot->tts_tupleDescriptor, attnum - 1)->atttypid; + + if (!first) { + appendStringInfoChar(&state->sql, '\t'); + } + first = false; + + if (isnull) { + appendStringInfoString(&state->sql, "\\N"); #ifdef USE_ASSERT_CHECKING - pindex++; + pindex++; #endif - continue; - } + continue; + } - string = chfdw_datum_to_ch_literal(value, type); - appendStringInfoString(&state->sql, string); - pfree(string); + string = chfdw_datum_to_ch_literal(value, type); + appendStringInfoString(&state->sql, string); + pfree(string); #ifdef USE_ASSERT_CHECKING - pindex++; + pindex++; #endif - } - appendStringInfoChar(&state->sql, '\n'); + } + appendStringInfoChar(&state->sql, '\n'); - Assert(pindex == state->p_nums); - } + Assert(pindex == state->p_nums); + } } -static void * -http_prepare_insert(void *conn, ResultRelInfo * rri, List * target_attrs, - const ch_query * query, char *table_name) -{ - ch_http_insert_state *state = palloc0(sizeof(ch_http_insert_state)); - - initStringInfo(&state->sql); - state->sql_begin = psprintf("%s FORMAT TSV\n", query->sql); - state->target_attrs = target_attrs; - state->p_nums = list_length(state->target_attrs); - state->conn = conn; - - return state; +static void* +http_prepare_insert( + void* conn, + ResultRelInfo* rri, + List* target_attrs, + const ch_query* query, + char* table_name +) { + ch_http_insert_state* state = palloc0(sizeof(ch_http_insert_state)); + + initStringInfo(&state->sql); + state->sql_begin = psprintf("%s FORMAT TSV\n", query->sql); + state->target_attrs = target_attrs; + state->p_nums = list_length(state->target_attrs); + state->conn = conn; + + return state; } static void -http_insert_tuple(void *istate, TupleTableSlot * slot) -{ - ch_http_insert_state *state = istate; +http_insert_tuple(void* istate, TupleTableSlot* slot) { + ch_http_insert_state* state = istate; - extend_insert_query(state, slot); + extend_insert_query(state, slot); - if ((slot == NULL && state->sql.len > 0) - || state->sql.len > (MaxAllocSize / 2 /* 512MB */ )) - { - ch_query query = new_query(state->sql.data, 0, NULL, NULL, NULL); + if ((slot == NULL && state->sql.len > 0) || + state->sql.len > (MaxAllocSize / 2 /* 512MB */)) { + ch_query query = new_query(state->sql.data, 0, NULL, NULL, NULL); - http_simple_insert(state->conn, &query); - resetStringInfo(&state->sql); - } + http_simple_insert(state->conn, &query); + resetStringInfo(&state->sql); + } } /*** BINARY PROTOCOL ***/ ch_connection -chfdw_binary_connect(ch_connection_details * details) -{ - ch_connection res; - - res.conn = ch_binary_connect(details); - res.methods = &binary_methods; - res.is_binary = true; - return res; +chfdw_binary_connect(ch_connection_details* details) { + ch_connection res; + + res.conn = ch_binary_connect(details); + res.methods = &binary_methods; + res.is_binary = true; + return res; } static void -binary_disconnect(void *conn) -{ - if (conn != NULL) - ch_binary_close((ch_binary_connection_t *) conn); +binary_disconnect(void* conn) { + if (conn != NULL) { + ch_binary_close((ch_binary_connection_t*)conn); + } } static bool -binary_is_broken(const void *conn) -{ - return ch_binary_is_broken((const ch_binary_connection_t *) conn); +binary_is_broken(const void* conn) { + return ch_binary_is_broken((const ch_binary_connection_t*)conn); } -static ch_cursor * -binary_simple_query(void *conn, const ch_query * query) -{ - MemoryContext tempcxt, - oldcxt; - ch_cursor *cursor; - ch_binary_read_state_t *state; - - ch_binary_response_t *resp = ch_binary_simple_query(conn, query, &is_canceled); - - if (!ch_binary_response_success(resp)) - { - char *error = pstrdup(ch_binary_response_error(resp)); - - ch_binary_response_free(resp); - ereport(ERROR, ( - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", error), - errdetail_internal("Remote Query: %.64000s", query->sql) - )); - } - - tempcxt = AllocSetContextCreate(PortalContext, "pg_clickhouse cursor", - ALLOCSET_DEFAULT_SIZES); - - oldcxt = MemoryContextSwitchTo(tempcxt); - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = resp; - state = (ch_binary_read_state_t *) palloc0(sizeof(ch_binary_read_state_t)); - cursor->query = pstrdup(query->sql); - cursor->read_state = state; - cursor->columns_count = ch_binary_response_columns(resp); - ch_binary_read_state_init(cursor->read_state, resp); - cursor->conversion_states = palloc0(sizeof(uintptr_t) * cursor->columns_count); - - /* - * CH JSON columns default to JSONBOID in state->coltypes. When foreign - * table column is declared `json` (JSONOID), override so - * binary_make_datum returns json Datum from CH's STRING bytes, skipping - * jsonb_in / jsonb_out round-trip that would reformat CH's emit and break - * expected outputs that pin CH's exact formatting. - */ - if (query->tupdesc && state->coltypes) - { - ListCell *lc; - size_t j = 0; - - foreach(lc, query->attr_nums) - { - int i = lfirst_int(lc); - - if (state->coltypes[j] == JSONBOID && - TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) - state->coltypes[j] = JSONOID; - j++; - } - } - - cursor->memcxt = tempcxt; - cursor->callback.func = binary_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - MemoryContextSwitchTo(oldcxt); - - if (state->error) - ereport(ERROR, - (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", state->error), - errdetail_internal("Remote Query: %.64000s", query->sql))); - - return cursor; +static ch_cursor* +binary_simple_query(void* conn, const ch_query* query) { + MemoryContext tempcxt, oldcxt; + ch_cursor* cursor; + ch_binary_read_state_t* state; + + ch_binary_response_t* resp = ch_binary_simple_query(conn, query, &is_canceled); + + if (!ch_binary_response_success(resp)) { + char* error = pstrdup(ch_binary_response_error(resp)); + + ch_binary_response_free(resp); + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", error), + errdetail_internal("Remote Query: %.64000s", query->sql)) + ); + } + + tempcxt = AllocSetContextCreate( + PortalContext, "pg_clickhouse cursor", ALLOCSET_DEFAULT_SIZES + ); + + oldcxt = MemoryContextSwitchTo(tempcxt); + cursor = palloc0(sizeof(ch_cursor)); + cursor->conn = conn; + cursor->query_response = resp; + state = (ch_binary_read_state_t*)palloc0(sizeof(ch_binary_read_state_t)); + cursor->query = pstrdup(query->sql); + cursor->read_state = state; + cursor->columns_count = ch_binary_response_columns(resp); + ch_binary_read_state_init(cursor->read_state, resp); + cursor->conversion_states = palloc0(sizeof(uintptr_t) * cursor->columns_count); + + /* + * CH JSON columns default to JSONBOID in state->coltypes. When foreign + * table column is declared `json` (JSONOID), override so + * binary_make_datum returns json Datum from CH's STRING bytes, skipping + * jsonb_in / jsonb_out round-trip that would reformat CH's emit and break + * expected outputs that pin CH's exact formatting. + */ + if (query->tupdesc && state->coltypes) { + ListCell* lc; + size_t j = 0; + + foreach (lc, query->attr_nums) { + int i = lfirst_int(lc); + + if (state->coltypes[j] == JSONBOID && + TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) { + state->coltypes[j] = JSONOID; + } + j++; + } + } + + cursor->memcxt = tempcxt; + cursor->callback.func = binary_cursor_free; + cursor->callback.arg = cursor; + MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); + MemoryContextSwitchTo(oldcxt); + + if (state->error) { + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", state->error), + errdetail_internal("Remote Query: %.64000s", query->sql)) + ); + } + + return cursor; } /* @@ -944,237 +992,240 @@ binary_simple_query(void *conn, const ch_query * query) * which only cares about text. */ static void -binary_fetch_row_errcb(void *arg) -{ - const char *sql = (const char *) arg; +binary_fetch_row_errcb(void* arg) { + const char* sql = (const char*)arg; - errdetail_internal("Remote Query: %.64000s", sql); + errdetail_internal("Remote Query: %.64000s", sql); } -static Datum * -binary_fetch_row(ChFdwScanRowContext * ctx) -{ - ListCell *lc; - ch_cursor *cursor = ctx->cursor; - List *attrs = ctx->retrieved_attrs; - TupleDesc tupdesc = ctx->tupdesc; - Datum *values = ctx->values; - bool *nulls = ctx->nulls; - ch_binary_read_state_t *state = cursor->read_state; - ErrorContextCallback errcallback; - - errcallback.callback = binary_fetch_row_errcb; - errcallback.arg = (void *) cursor->query; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - bool have_data = ch_binary_read_row(state); - size_t attcount = list_length(attrs); - - if (state->error) - ereport(ERROR, - (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: error while reading row: %s", - state->error))); - - if (!have_data) - { - error_context_stack = errcallback.previous; - return NULL; - } - - if (attcount == 0) - { - if (ch_binary_response_columns(state->resp) == 1 && state->nulls[0]) - { - nulls[0] = true; - goto ok; - } - else - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: unexpected state: attributes " - "count == 0 and haven't got NULL in the response"))); - } - else if (attcount != ch_binary_response_columns(state->resp)) - { - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal("pg_clickhouse: returned %lu columns, expected %lu", - ch_binary_response_columns(state->resp), attcount))); - } - - if (tupdesc) - { - size_t j = 0; - - Assert(values && nulls); - - foreach(lc, attrs) - { - int i = lfirst_int(lc); - bool isnull = state->nulls[j]; - intptr_t convstate; - - if (isnull) - values[i - 1] = (Datum) 0; - else - { - again: - convstate = cursor->conversion_states[j]; - switch (convstate) - { - case 0: - { - MemoryContext old_mcxt; - - Oid outtype = TupleDescAttr(tupdesc, i - 1)->atttypid; - void *s; - - /* - * now we're should be in temporary memory - * context, so make sure conversion states outlive - * it. - */ - old_mcxt = MemoryContextSwitchTo(cursor->memcxt); - - /* - * Convert the Postgres Datum, stored by - * ch_binary_read_row() as the type defined by by - * state->coltypes[j], to the type defined by the - * foreign table. No conversion if they're binary - * compatible, but required to allow a static - * ClickHouse type (which maps to a well-known - * Postgres type) to be converted to a compatible - * Postgres value via a CAST. - */ - s = ch_binary_init_convert_state(state->values[j], - state->coltypes[j], outtype); - MemoryContextSwitchTo(old_mcxt); - - if (s == NULL) - /* no conversion but state is initialized */ - cursor->conversion_states[j] = 1; - else - cursor->conversion_states[j] = (uintptr_t) s; - goto again; - } - case 1: - /* no conversion */ - values[i - 1] = state->values[j]; - break; - default: - values[i - 1] = ch_binary_convert_datum((void *) convstate, - state->values[j]); - } - } - - nulls[i - 1] = isnull; - j++; - } - } +static Datum* +binary_fetch_row(ChFdwScanRowContext* ctx) { + ListCell* lc; + ch_cursor* cursor = ctx->cursor; + List* attrs = ctx->retrieved_attrs; + TupleDesc tupdesc = ctx->tupdesc; + Datum* values = ctx->values; + bool* nulls = ctx->nulls; + ch_binary_read_state_t* state = cursor->read_state; + ErrorContextCallback errcallback; + + errcallback.callback = binary_fetch_row_errcb; + errcallback.arg = (void*)cursor->query; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + bool have_data = ch_binary_read_row(state); + size_t attcount = list_length(attrs); + + if (state->error) { + ereport( + ERROR, + (errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: error while reading row: %s", state->error)) + ); + } + + if (!have_data) { + error_context_stack = errcallback.previous; + return NULL; + } + + if (attcount == 0) { + if (ch_binary_response_columns(state->resp) == 1 && state->nulls[0]) { + nulls[0] = true; + goto ok; + } else { + ereport( + ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg( + "pg_clickhouse: unexpected state: attributes " + "count == 0 and haven't got NULL in the response" + )) + ); + } + } else if (attcount != ch_binary_response_columns(state->resp)) { + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal( + "pg_clickhouse: returned %lu columns, expected %lu", + ch_binary_response_columns(state->resp), + attcount + )) + ); + } + + if (tupdesc) { + size_t j = 0; + + Assert(values && nulls); + + foreach (lc, attrs) { + int i = lfirst_int(lc); + bool isnull = state->nulls[j]; + intptr_t convstate; + + if (isnull) { + values[i - 1] = (Datum)0; + } else { + again: + convstate = cursor->conversion_states[j]; + switch (convstate) { + case 0: { + MemoryContext old_mcxt; + + Oid outtype = TupleDescAttr(tupdesc, i - 1)->atttypid; + void* s; + + /* + * now we're should be in temporary memory + * context, so make sure conversion states outlive + * it. + */ + old_mcxt = MemoryContextSwitchTo(cursor->memcxt); + + /* + * Convert the Postgres Datum, stored by + * ch_binary_read_row() as the type defined by by + * state->coltypes[j], to the type defined by the + * foreign table. No conversion if they're binary + * compatible, but required to allow a static + * ClickHouse type (which maps to a well-known + * Postgres type) to be converted to a compatible + * Postgres value via a CAST. + */ + s = ch_binary_init_convert_state( + state->values[j], state->coltypes[j], outtype + ); + MemoryContextSwitchTo(old_mcxt); + + if (s == NULL) { + /* no conversion but state is initialized */ + cursor->conversion_states[j] = 1; + } else { + cursor->conversion_states[j] = (uintptr_t)s; + } + goto again; + } + case 1: + /* no conversion */ + values[i - 1] = state->values[j]; + break; + default: + values[i - 1] = + ch_binary_convert_datum((void*)convstate, state->values[j]); + } + } + + nulls[i - 1] = isnull; + j++; + } + } ok: - error_context_stack = errcallback.previous; - return state->values; + error_context_stack = errcallback.previous; + return state->values; } static void -binary_cursor_free(void *c) -{ - ch_cursor *cursor = c; - - for (size_t i = 0; i < cursor->columns_count; i++) - { - if (cursor->conversion_states[i] > 1) - ch_binary_free_convert_state((void *) cursor->conversion_states[i]); - } - - ch_binary_read_state_free(cursor->read_state); - ch_binary_response_free(cursor->query_response); +binary_cursor_free(void* c) { + ch_cursor* cursor = c; + + for (size_t i = 0; i < cursor->columns_count; i++) { + if (cursor->conversion_states[i] > 1) { + ch_binary_free_convert_state((void*)cursor->conversion_states[i]); + } + } + + ch_binary_read_state_free(cursor->read_state); + ch_binary_response_free(cursor->query_response); } -static void * -binary_prepare_insert(void *conn, ResultRelInfo * rri, List * target_attrs, - const ch_query * query, char *table_name) -{ - ch_binary_insert_state *state = NULL; - MemoryContext tempcxt, - oldcxt; - - if (table_name == NULL) - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("expected table name"))); - - tempcxt = AllocSetContextCreate(CurrentMemoryContext, - "pg_clickhouse binary insert state", ALLOCSET_DEFAULT_SIZES); - - /* prepare cleanup */ - oldcxt = MemoryContextSwitchTo(tempcxt); - state = (ch_binary_insert_state *) palloc0(sizeof(ch_binary_insert_state)); - state->memcxt = tempcxt; - state->callback.func = ch_binary_insert_state_free; - state->callback.arg = state; - state->conn = conn; - state->table_name = pstrdup(table_name); - state->relid = RelationGetRelid(rri->ri_RelationDesc); - MemoryContextRegisterResetCallback(tempcxt, &state->callback); - - /* time for c++ stuff */ - ch_binary_prepare_insert(conn, query, state); - - /* buffers */ - state->values = (Datum *) palloc0(sizeof(Datum) * state->len); - state->nulls = (bool *) palloc0(sizeof(bool) * state->len); - MemoryContextSwitchTo(oldcxt); - - return state; +static void* +binary_prepare_insert( + void* conn, + ResultRelInfo* rri, + List* target_attrs, + const ch_query* query, + char* table_name +) { + ch_binary_insert_state* state = NULL; + MemoryContext tempcxt, oldcxt; + + if (table_name == NULL) { + ereport(ERROR, (errcode(ERRCODE_FDW_ERROR), errmsg("expected table name"))); + } + + tempcxt = AllocSetContextCreate( + CurrentMemoryContext, + "pg_clickhouse binary insert state", + ALLOCSET_DEFAULT_SIZES + ); + + /* prepare cleanup */ + oldcxt = MemoryContextSwitchTo(tempcxt); + state = (ch_binary_insert_state*)palloc0(sizeof(ch_binary_insert_state)); + state->memcxt = tempcxt; + state->callback.func = ch_binary_insert_state_free; + state->callback.arg = state; + state->conn = conn; + state->table_name = pstrdup(table_name); + state->relid = RelationGetRelid(rri->ri_RelationDesc); + MemoryContextRegisterResetCallback(tempcxt, &state->callback); + + /* time for c++ stuff */ + ch_binary_prepare_insert(conn, query, state); + + /* buffers */ + state->values = (Datum*)palloc0(sizeof(Datum) * state->len); + state->nulls = (bool*)palloc0(sizeof(bool) * state->len); + MemoryContextSwitchTo(oldcxt); + + return state; } static void -binary_insert_tuple(void *istate, TupleTableSlot * slot) -{ - ch_binary_insert_state *state = istate; - - if (slot) - { - if (state->conversion_states == NULL) - { - MemoryContext old_mcxt; - - old_mcxt = MemoryContextSwitchTo(state->memcxt); - - /* - * Set up conversion states so that Postgres Datums of types - * defined by the foreign table can be converted to a well-known - * Postgres type that the binary INSERT path knows how to convert - * to the appropriate ClickHouse type. Binary compatible types - * don't convert; others require a CAST. Fails if there is no - * cast. - */ - state->conversion_states = ch_binary_make_tuple_map(slot->tts_tupleDescriptor, state->outdesc, state->relid); - MemoryContextSwitchTo(old_mcxt); - } - - ch_binary_do_output_conversion(state, slot); - - for (size_t i = 0; i < state->outdesc->natts; i++) - ch_binary_column_append_data(state, i); - } - else - { - ch_binary_insert_columns(state); - } +binary_insert_tuple(void* istate, TupleTableSlot* slot) { + ch_binary_insert_state* state = istate; + + if (slot) { + if (state->conversion_states == NULL) { + MemoryContext old_mcxt; + + old_mcxt = MemoryContextSwitchTo(state->memcxt); + + /* + * Set up conversion states so that Postgres Datums of types + * defined by the foreign table can be converted to a well-known + * Postgres type that the binary INSERT path knows how to convert + * to the appropriate ClickHouse type. Binary compatible types + * don't convert; others require a CAST. Fails if there is no + * cast. + */ + state->conversion_states = ch_binary_make_tuple_map( + slot->tts_tupleDescriptor, state->outdesc, state->relid + ); + MemoryContextSwitchTo(old_mcxt); + } + + ch_binary_do_output_conversion(state, slot); + + for (size_t i = 0; i < state->outdesc->natts; i++) { + ch_binary_column_append_data(state, i); + } + } else { + ch_binary_insert_columns(state); + } } static void -binary_finalize_insert(void *istate) -{ - ch_binary_insert_state *state = istate; +binary_finalize_insert(void* istate) { + ch_binary_insert_state* state = istate; - if (state && state->insert_block) - ch_binary_finalize_insert(state->insert_block); + if (state && state->insert_block) { + ch_binary_finalize_insert(state->insert_block); + } } /* @@ -1182,345 +1233,361 @@ binary_finalize_insert(void *istate) * str_types_map below. On change, re-run and paste the output into * doc/pg_clickhouse.md. Perl: https://stackoverflow.com/a/58443028/79202 - psql --no-psqlrc --pset border=2 --pset footer=off -c " - SELECT * FROM ( VALUES - ('Bool', 'boolean', ''), - ('Int8', 'smallint', ''), - ('UInt8', 'smallint', ''), - ('Int16', 'smallint', ''), - ('UInt16', 'integer', ''), - ('Int32', 'integer', ''), - ('UInt32', 'bigint', ''), - ('Int64', 'bigint', ''), - ('UInt64', 'bigint', 'Errors on values > BIGINT max'), - ('Float32', 'real', ''), - ('Float64', 'double precision', ''), - ('Decimal', 'numeric', ''), - ('String', 'text, bytea', ''), - ('DateTime', 'timestamptz', ''), - ('Date', 'date', ''), - ('Date32', 'date', ''), - ('UUID', 'uuid', ''), - ('IPv4', 'inet', ''), - ('IPv6', 'inet', ''), - ('JSON', 'jsonb, json', '') - ) AS v(\"ClickHouse\", \"PostgreSQL\", \"Notes\") - ORDER BY \"ClickHouse\"; - " | perl -ne 'my $m = $.%2; print $buf[$m] if defined $buf[$m]; $buf[$m] = s/\+/|/gr if $.>1' | pbcopy + psql --no-psqlrc --pset border=2 --pset footer=off -c " + SELECT * FROM ( VALUES + ('Bool', 'boolean', ''), + ('Int8', 'smallint', ''), + ('UInt8', 'smallint', ''), + ('Int16', 'smallint', ''), + ('UInt16', 'integer', ''), + ('Int32', 'integer', ''), + ('UInt32', 'bigint', ''), + ('Int64', 'bigint', ''), + ('UInt64', 'bigint', 'Errors on values > BIGINT max'), + ('Float32', 'real', ''), + ('Float64', 'double precision', ''), + ('Decimal', 'numeric', ''), + ('String', 'text, bytea', ''), + ('DateTime', 'timestamptz', ''), + ('Date', 'date', ''), + ('Date32', 'date', ''), + ('UUID', 'uuid', ''), + ('IPv4', 'inet', ''), + ('IPv6', 'inet', ''), + ('JSON', 'jsonb, json', '') + ) AS v(\"ClickHouse\", \"PostgreSQL\", \"Notes\") + ORDER BY \"ClickHouse\"; + " | perl -ne 'my $m = $.%2; print $buf[$m] if defined $buf[$m]; $buf[$m] = s/\+/|/gr + if $.>1' | pbcopy */ -static char *str_types_map[][2] = { - {"Bool", "BOOLEAN"}, - {"Int8", "INT2"}, - {"UInt8", "INT2"}, - {"Int16", "INT2"}, - {"UInt16", "INT4"}, - {"Int32", "INT4"}, - {"UInt32", "INT8"}, - {"Int64", "INT8"}, - {"UInt64", "INT8"}, - {"Float32", "REAL"}, - {"Float64", "DOUBLE PRECISION"}, - {"Decimal", "NUMERIC"}, - {"String", "TEXT"}, - {"DateTime", "TIMESTAMPTZ"}, - {"Date", "DATE"}, /* must come after other Date types */ - {"Date32", "DATE"}, - {"UUID", "UUID"}, - {"IPv4", "inet"}, - {"IPv6", "inet"}, - {"JSON", "JSONB"}, - {NULL, NULL}, +static char* str_types_map[][2] = { + { "Bool", "BOOLEAN" }, + { "Int8", "INT2" }, + { "UInt8", "INT2" }, + { "Int16", "INT2" }, + { "UInt16", "INT4" }, + { "Int32", "INT4" }, + { "UInt32", "INT8" }, + { "Int64", "INT8" }, + { "UInt64", "INT8" }, + { "Float32", "REAL" }, + { "Float64", "DOUBLE PRECISION" }, + { "Decimal", "NUMERIC" }, + { "String", "TEXT" }, + { "DateTime", "TIMESTAMPTZ" }, + { "Date", "DATE" }, /* must come after other Date types */ + { "Date32", "DATE" }, + { "UUID", "UUID" }, + { "IPv4", "inet" }, + { "IPv6", "inet" }, + { "JSON", "JSONB" }, + { NULL, NULL }, }; -static char * -parse_type(char *table_name, char *colname, char *part, bool *is_nullable, List * *options) -{ - char *typepart = part; - char *pos = strstr(typepart, "("); - - if (pos != NULL) - { - char *insidebr = pnstrdup(pos + 1, strrchr(typepart, ')') - pos - 1); - - if (strncmp(typepart, "Decimal", strlen("Decimal")) == 0) - { - if (strstr(insidebr, ",") == NULL) - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: could not import Decimal field, " - "should be two parameters on definition"))); - - return psprintf("NUMERIC(%s)", insidebr); - } - else if (strncmp(typepart, "FixedString", strlen("FixedString")) == 0) - return psprintf("VARCHAR(%s)", insidebr); - else if (strncmp(typepart, "Enum8", strlen("Enum8")) == 0) - return "TEXT"; - else if (strncmp(typepart, "Enum16", strlen("Enum16")) == 0) - return "TEXT"; - else if (strncmp(typepart, "DateTime64", strlen("DateTime64")) == 0) - return "TIMESTAMPTZ"; - else if (strncmp(typepart, "DateTime", strlen("DateTime")) == 0) - return "TIMESTAMPTZ"; - else if (strncmp(typepart, "Tuple", strlen("Tuple")) == 0) - { - elog(NOTICE, "pg_clickhouse: ClickHouse type was " - "translated to type for column \"%s\", please create composite type and alter the column if needed", colname); - return "TEXT"; - } - else if (strncmp(typepart, "Array", strlen("Array")) == 0) - { - return psprintf("%s[]", parse_type(table_name, colname, insidebr, NULL, options)); - } - else if (strncmp(typepart, "Nullable", strlen("Nullable")) == 0) - { - if (is_nullable == NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("pg_clickhouse: nested Nullable is not supported"))); - - *is_nullable = true; - return parse_type(table_name, colname, insidebr, NULL, options); - } - else if (strncmp(typepart, "LowCardinality", strlen("LowCardinality")) == 0) - { - return parse_type(table_name, colname, insidebr, is_nullable, options); - } - else if (strncmp(typepart, "AggregateFunction", strlen("AggregateFunction")) == 0 || - strncmp(typepart, "SimpleAggregateFunction", strlen("SimpleAggregateFunction")) == 0) - { - char *pos2 = strstr(pos, ","); - - if (pos2 == NULL) - { - /* Detect COUNT with no params. */ - if (strncmp(insidebr, "count", strlen("count")) == 0) - return "BIGINT"; - ereport(ERROR, - (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: expected comma in AggregateFunction"))); - } - - char *func = pnstrdup(pos + 1, strstr(pos + 1, ",") - pos - 1); - - if (options != NULL) - { - int val = typepart[0] == 'A' ? 1 : 2; - - *options = lappend(*options, makeInteger(val)); - *options = lappend(*options, makeString(func)); - } - - return parse_type(table_name, colname, pos2 + 2, is_nullable, options); - } - - typepart = pos + 1; - } - - size_t i = 0; - - while (str_types_map[i][0] != NULL) - { - if (strncmp(str_types_map[i][0], typepart, strlen(str_types_map[i][0])) == 0) - return pstrdup(str_types_map[i][1]); - i++; - } - - ereport(ERROR, (errmsg( - "pg_clickhouse: could not map %s.%s type <%s>", - quote_identifier(table_name), quote_identifier(colname), part - ))); +static char* +parse_type( + char* table_name, + char* colname, + char* part, + bool* is_nullable, + List** options +) { + char* typepart = part; + char* pos = strstr(typepart, "("); + + if (pos != NULL) { + char* insidebr = pnstrdup(pos + 1, strrchr(typepart, ')') - pos - 1); + + if (strncmp(typepart, "Decimal", strlen("Decimal")) == 0) { + if (strstr(insidebr, ",") == NULL) { + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: could not import Decimal field, " + "should be two parameters on definition" + )) + ); + } + + return psprintf("NUMERIC(%s)", insidebr); + } else if (strncmp(typepart, "FixedString", strlen("FixedString")) == 0) { + return psprintf("VARCHAR(%s)", insidebr); + } else if (strncmp(typepart, "Enum8", strlen("Enum8")) == 0) { + return "TEXT"; + } else if (strncmp(typepart, "Enum16", strlen("Enum16")) == 0) { + return "TEXT"; + } else if (strncmp(typepart, "DateTime64", strlen("DateTime64")) == 0) { + return "TIMESTAMPTZ"; + } else if (strncmp(typepart, "DateTime", strlen("DateTime")) == 0) { + return "TIMESTAMPTZ"; + } else if (strncmp(typepart, "Tuple", strlen("Tuple")) == 0) { + elog( + NOTICE, + "pg_clickhouse: ClickHouse type was " + "translated to type for column \"%s\", please create composite " + "type and alter the column if needed", + colname + ); + return "TEXT"; + } else if (strncmp(typepart, "Array", strlen("Array")) == 0) { + return psprintf( + "%s[]", parse_type(table_name, colname, insidebr, NULL, options) + ); + } else if (strncmp(typepart, "Nullable", strlen("Nullable")) == 0) { + if (is_nullable == NULL) { + ereport( + ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("pg_clickhouse: nested Nullable is not supported")) + ); + } + + *is_nullable = true; + return parse_type(table_name, colname, insidebr, NULL, options); + } else if (strncmp(typepart, "LowCardinality", strlen("LowCardinality")) == 0) { + return parse_type(table_name, colname, insidebr, is_nullable, options); + } else if ( + strncmp(typepart, "AggregateFunction", strlen("AggregateFunction")) == 0 || + strncmp( + typepart, "SimpleAggregateFunction", strlen("SimpleAggregateFunction") + ) == 0 + ) { + char* pos2 = strstr(pos, ","); + + if (pos2 == NULL) { + /* Detect COUNT with no params. */ + if (strncmp(insidebr, "count", strlen("count")) == 0) { + return "BIGINT"; + } + ereport( + ERROR, + (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("pg_clickhouse: expected comma in AggregateFunction")) + ); + } + + char* func = pnstrdup(pos + 1, strstr(pos + 1, ",") - pos - 1); + + if (options != NULL) { + int val = typepart[0] == 'A' ? 1 : 2; + + *options = lappend(*options, makeInteger(val)); + *options = lappend(*options, makeString(func)); + } + + return parse_type(table_name, colname, pos2 + 2, is_nullable, options); + } + + typepart = pos + 1; + } + + size_t i = 0; + + while (str_types_map[i][0] != NULL) { + if (strncmp(str_types_map[i][0], typepart, strlen(str_types_map[i][0])) == 0) { + return pstrdup(str_types_map[i][1]); + } + i++; + } + + ereport( + ERROR, + (errmsg( + "pg_clickhouse: could not map %s.%s type <%s>", + quote_identifier(table_name), + quote_identifier(colname), + part + )) + ); } -List * -chfdw_construct_create_tables(ImportForeignSchemaStmt * stmt, ForeignServer * server) -{ - Oid userid = GetUserId(); - UserMapping *user = GetUserMapping(userid, server->serverid); - ch_connection conn = chfdw_get_connection(user); - ch_cursor *cursor; - ch_query query = new_query(NULL, 0, NULL, NULL, NULL); - List *result = NIL; - Datum *row_values; - - query.sql = psprintf("SELECT name, engine, engine_full " - "FROM system.tables " - "WHERE name NOT LIKE '.inner%%' " - "AND database = %s", - ch_quote_literal(stmt->remote_schema)); - - cursor = conn.methods->simple_query(conn.conn, &query); - - ChFdwScanRowContext cols_ctx = { - NULL, - list_make2_int(1, 2), - NULL, - NULL, - NULL, - NULL - }; - - ChFdwScanRowContext tables_ctx = { - NULL, - list_make3_int(1, 2, 3), - NULL, - cursor, - NULL, - NULL - }; - - /* - * Drain the outer query into private strings before opening the per-table - * column queries: both use the same connection, and binary streaming only - * permits one in-flight response at a time. - */ - List *tables = NIL; - - while ((row_values = conn.methods->fetch_row(&tables_ctx)) != NULL) - { - List *triple = list_make3(pstrdup(TextDatumGetCString(row_values[0])), - pstrdup(TextDatumGetCString(row_values[1])), - pstrdup(TextDatumGetCString(row_values[2]))); - - CHECK_FOR_INTERRUPTS(); - tables = lappend(tables, triple); - } - MemoryContextDelete(cursor->memcxt); - - ListCell *tlc; - - foreach(tlc, tables) - { - List *triple = (List *) lfirst(tlc); - char *table_name = (char *) linitial(triple); - char *engine = (char *) lsecond(triple); - char *engine_full = (char *) lthird(triple); - StringInfoData buf; - Datum *dvalues; - bool first = true; - - if (table_name == NULL) - continue; - - if (list_length(stmt->table_list)) - { - ListCell *lc; - bool found = false; - - foreach(lc, stmt->table_list) - { - RangeVar *rv = (RangeVar *) lfirst(lc); - - if (strcmp(rv->relname, table_name) == 0) - found = true; - } - - if (stmt->list_type == FDW_IMPORT_SCHEMA_EXCEPT && found) - continue; - else if (stmt->list_type == FDW_IMPORT_SCHEMA_LIMIT_TO && !found) - continue; - } - - initStringInfo(&buf); - appendStringInfo(&buf, - "CREATE FOREIGN TABLE IF NOT EXISTS %s.%s (\n", - quote_identifier(stmt->local_schema), - quote_identifier(table_name)); - query.sql = psprintf("SELECT name, type " - "FROM system.columns " - "WHERE database = %s " - "AND table = %s", - ch_quote_literal(stmt->remote_schema), - ch_quote_literal(table_name)); - - cols_ctx.cursor = conn.methods->simple_query(conn.conn, &query); - while ((dvalues = conn.methods->fetch_row(&cols_ctx)) != NULL) - { - List *options = NIL; - bool is_nullable = false; - char *colname = TextDatumGetCString(dvalues[0]); - char *remote_type = parse_type(table_name, colname, TextDatumGetCString(dvalues[1]), &is_nullable, &options); - - if (!first) - appendStringInfoString(&buf, ",\n"); - first = false; - - /* name */ - appendStringInfo(&buf, "\t%s ", quote_identifier(colname)); - - /* type */ - appendStringInfoString(&buf, remote_type); - - if (options != NIL) - { - bool first_opt = true; - ListCell *lc; - - appendStringInfoString(&buf, " OPTIONS ("); - foreach(lc, options) - { - Node *val = lfirst(lc); - - if (IsA(val, Integer)) - { - if (!first_opt) - appendStringInfoString(&buf, ", "); - first_opt = false; - switch intVal - (val) - { - case 1: - appendStringInfoString(&buf, "AggregateFunction"); - break; - case 2: - appendStringInfoString(&buf, "SimpleAggregateFunction"); - break; - default: - elog(ERROR, "programming error"); - } - } - else - { - appendStringInfoChar(&buf, ' '); - appendStringInfoString(&buf, ch_quote_literal(strVal(val))); - } - } - appendStringInfoString(&buf, ")"); - list_free_deep(options); - } - - if (!is_nullable) - appendStringInfoString(&buf, " NOT NULL"); - } - - appendStringInfo( - &buf, - "\n) SERVER %s OPTIONS (database %s, table_name %s", - quote_identifier(server->servername), - ch_quote_literal(stmt->remote_schema), - ch_quote_literal(table_name)); - - if (engine && engine_full && strcmp(engine, "CollapsingMergeTree") == 0) - { - char *sub = strstr(engine_full, ")"); - - if (sub) - { - sub[1] = '\0'; - appendStringInfo(&buf, ", engine %s", ch_quote_literal(engine_full)); - } - } - else if (engine) - appendStringInfo(&buf, ", engine %s", ch_quote_literal(engine)); - - appendStringInfoString(&buf, ");\n"); - result = lappend(result, buf.data); - MemoryContextDelete(cols_ctx.cursor->memcxt); - } - - return result; +List* +chfdw_construct_create_tables(ImportForeignSchemaStmt* stmt, ForeignServer* server) { + Oid userid = GetUserId(); + UserMapping* user = GetUserMapping(userid, server->serverid); + ch_connection conn = chfdw_get_connection(user); + ch_cursor* cursor; + ch_query query = new_query(NULL, 0, NULL, NULL, NULL); + List* result = NIL; + Datum* row_values; + + query.sql = psprintf( + "SELECT name, engine, engine_full " + "FROM system.tables " + "WHERE name NOT LIKE '.inner%%' " + "AND database = %s", + ch_quote_literal(stmt->remote_schema) + ); + + cursor = conn.methods->simple_query(conn.conn, &query); + + ChFdwScanRowContext cols_ctx = { + NULL, list_make2_int(1, 2), NULL, NULL, NULL, NULL + }; + + ChFdwScanRowContext tables_ctx = { NULL, list_make3_int(1, 2, 3), + NULL, cursor, + NULL, NULL }; + + /* + * Drain the outer query into private strings before opening the per-table + * column queries: both use the same connection, and binary streaming only + * permits one in-flight response at a time. + */ + List* tables = NIL; + + while ((row_values = conn.methods->fetch_row(&tables_ctx)) != NULL) { + List* triple = list_make3( + pstrdup(TextDatumGetCString(row_values[0])), + pstrdup(TextDatumGetCString(row_values[1])), + pstrdup(TextDatumGetCString(row_values[2])) + ); + + CHECK_FOR_INTERRUPTS(); + tables = lappend(tables, triple); + } + MemoryContextDelete(cursor->memcxt); + + ListCell* tlc; + + foreach (tlc, tables) { + List* triple = (List*)lfirst(tlc); + char* table_name = (char*)linitial(triple); + char* engine = (char*)lsecond(triple); + char* engine_full = (char*)lthird(triple); + StringInfoData buf; + Datum* dvalues; + bool first = true; + + if (table_name == NULL) { + continue; + } + + if (list_length(stmt->table_list)) { + ListCell* lc; + bool found = false; + + foreach (lc, stmt->table_list) { + RangeVar* rv = (RangeVar*)lfirst(lc); + + if (strcmp(rv->relname, table_name) == 0) { + found = true; + } + } + + if (stmt->list_type == FDW_IMPORT_SCHEMA_EXCEPT && found) { + continue; + } else if (stmt->list_type == FDW_IMPORT_SCHEMA_LIMIT_TO && !found) { + continue; + } + } + + initStringInfo(&buf); + appendStringInfo( + &buf, + "CREATE FOREIGN TABLE IF NOT EXISTS %s.%s (\n", + quote_identifier(stmt->local_schema), + quote_identifier(table_name) + ); + query.sql = psprintf( + "SELECT name, type " + "FROM system.columns " + "WHERE database = %s " + "AND table = %s", + ch_quote_literal(stmt->remote_schema), + ch_quote_literal(table_name) + ); + + cols_ctx.cursor = conn.methods->simple_query(conn.conn, &query); + while ((dvalues = conn.methods->fetch_row(&cols_ctx)) != NULL) { + List* options = NIL; + bool is_nullable = false; + char* colname = TextDatumGetCString(dvalues[0]); + char* remote_type = parse_type( + table_name, + colname, + TextDatumGetCString(dvalues[1]), + &is_nullable, + &options + ); + + if (!first) { + appendStringInfoString(&buf, ",\n"); + } + first = false; + + /* name */ + appendStringInfo(&buf, "\t%s ", quote_identifier(colname)); + + /* type */ + appendStringInfoString(&buf, remote_type); + + if (options != NIL) { + bool first_opt = true; + ListCell* lc; + + appendStringInfoString(&buf, " OPTIONS ("); + foreach (lc, options) { + Node* val = lfirst(lc); + + if (IsA(val, Integer)) { + if (!first_opt) { + appendStringInfoString(&buf, ", "); + } + first_opt = false; + switch + intVal(val) { + case 1: + appendStringInfoString(&buf, "AggregateFunction"); + break; + case 2: + appendStringInfoString(&buf, "SimpleAggregateFunction"); + break; + default: + elog(ERROR, "programming error"); + } + } else { + appendStringInfoChar(&buf, ' '); + appendStringInfoString(&buf, ch_quote_literal(strVal(val))); + } + } + appendStringInfoString(&buf, ")"); + list_free_deep(options); + } + + if (!is_nullable) { + appendStringInfoString(&buf, " NOT NULL"); + } + } + + appendStringInfo( + &buf, + "\n) SERVER %s OPTIONS (database %s, table_name %s", + quote_identifier(server->servername), + ch_quote_literal(stmt->remote_schema), + ch_quote_literal(table_name) + ); + + if (engine && engine_full && strcmp(engine, "CollapsingMergeTree") == 0) { + char* sub = strstr(engine_full, ")"); + + if (sub) { + sub[1] = '\0'; + appendStringInfo(&buf, ", engine %s", ch_quote_literal(engine_full)); + } + } else if (engine) { + appendStringInfo(&buf, ", engine %s", ch_quote_literal(engine)); + } + + appendStringInfoString(&buf, ");\n"); + result = lappend(result, buf.data); + MemoryContextDelete(cols_ctx.cursor->memcxt); + } + + return result; } /* @@ -1531,71 +1598,68 @@ chfdw_construct_create_tables(ImportForeignSchemaStmt * stmt, ForeignServer * se * and writeAnyEscapedStringO() in src/IO/WriteHelpers.h in the ClickHouse * source code. */ -static char * -ch_escape_string(const char *from, size_t len) -{ - char *result; - size_t remaining = len; - const char *source = from; - - result = palloc(len * 2 + 1); - char *target = result; - - while (remaining > 0) - { - char c = *source; - - switch (c) - { - case '\'': - *target++ = '\\'; - *target++ = c; - break; - case '\\': - *target++ = c; - *target++ = c; - break; - case '\b': - *target++ = '\\'; - *target++ = 'b'; - break; - case '\f': - *target++ = '\\'; - *target++ = 'f'; - break; - case '\r': - *target++ = '\\'; - *target++ = 'r'; - break; - case '\n': - *target++ = '\\'; - *target++ = 'n'; - break; - case '\t': - *target++ = '\\'; - *target++ = 't'; - break; - case '\0': - *target++ = '\\'; - *target++ = '0'; - break; - case '\a': - *target++ = '\\'; - *target++ = 'a'; - break; - case '\v': - *target++ = '\\'; - *target++ = 'v'; - break; - default: - *target++ = c; - } - source++; - remaining--; - } - - *target = '\0'; - return result; +static char* +ch_escape_string(const char* from, size_t len) { + char* result; + size_t remaining = len; + const char* source = from; + + result = palloc(len * 2 + 1); + char* target = result; + + while (remaining > 0) { + char c = *source; + + switch (c) { + case '\'': + *target++ = '\\'; + *target++ = c; + break; + case '\\': + *target++ = c; + *target++ = c; + break; + case '\b': + *target++ = '\\'; + *target++ = 'b'; + break; + case '\f': + *target++ = '\\'; + *target++ = 'f'; + break; + case '\r': + *target++ = '\\'; + *target++ = 'r'; + break; + case '\n': + *target++ = '\\'; + *target++ = 'n'; + break; + case '\t': + *target++ = '\\'; + *target++ = 't'; + break; + case '\0': + *target++ = '\\'; + *target++ = '0'; + break; + case '\a': + *target++ = '\\'; + *target++ = 'a'; + break; + case '\v': + *target++ = '\\'; + *target++ = 'v'; + break; + default: + *target++ = c; + } + source++; + remaining--; + } + + *target = '\0'; + return result; } /* @@ -1603,40 +1667,38 @@ ch_escape_string(const char *from, size_t len) * PostgreSQL's quote_literal_cstr() by never returning an E-quoted string. */ static void -ch_quote_literal_internal(char *dst, const char *src, size_t len) -{ - *dst++ = '\''; - while (*src) - { - if (SQL_STR_DOUBLE(*src, true)) - *dst++ = *src; - *dst++ = *src++; - } - *dst++ = '\''; - *dst++ = '\0'; +ch_quote_literal_internal(char* dst, const char* src, size_t len) { + *dst++ = '\''; + while (*src) { + if (SQL_STR_DOUBLE(*src, true)) { + *dst++ = *src; + } + *dst++ = *src++; + } + *dst++ = '\''; + *dst++ = '\0'; } /* * Convenience function to escape and return a string as a ClickHouse literal. * Returns a palloc'd string. */ -char * -ch_quote_literal(const char *rawstr) -{ - char *result; - int len; - - len = strlen(rawstr); - /* We make a worst-case result area; wasting a little space is OK */ - result = palloc( - (len * 2) /* doubling for every character if each one is - * a quote */ - + 2 /* two outer quotes */ - + 1 /* null terminator */ - ); - - ch_quote_literal_internal(result, rawstr, len); - return result; +char* +ch_quote_literal(const char* rawstr) { + char* result; + int len; + + len = strlen(rawstr); + /* We make a worst-case result area; wasting a little space is OK */ + result = palloc( + (len * 2) /* doubling for every character if each one is + * a quote */ + + 2 /* two outer quotes */ + + 1 /* null terminator */ + ); + + ch_quote_literal_internal(result, rawstr, len); + return result; } /* @@ -1645,47 +1707,55 @@ ch_quote_literal(const char *rawstr) * PostgreSQL's `quote_identifier()`. Raises an error if the identifier length * is zero or greater than `NAMEDATALEN` (64) unquoted or * `CH_ESCAPED_NAMEDATALEN` quoted. -*/ -const char * -ch_quote_ident(const char *ident) -{ - /* https://clickhouse.com/docs/sql-reference/syntax#identifiers */ - int len = strlen(ident); - - if (len >= 2 && ((ident[0] == '"' && ident[len - 1] == '"') || (ident[0] == '`' && ident[len - 1] == '`'))) - { - /* - * Make sure it has no unescaped quote character. Allowed escapes: - * - * ": (""|\\.) - * - * `: (``|\\.) - */ - for (int i = 2; i <= len - 2; i++) - { - /* Skip escaped character. */ - if (ident[i] == '\\') - i++; - - /* Disallow unescaped quote character. */ - else if (ident[i] == ident[0] && ident[i + 1] != ident[0]) - ereport(ERROR, - errcode(ERRCODE_FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH), - errmsg_internal("pg_clickhouse: invalid identifier")); - } - - /* Allow already quoted identifier. */ - if (len == 2 || len > CH_ESCAPED_NAMEDATALEN - 1) - ereport(ERROR, - errcode(ERRCODE_FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH), - errmsg_internal("pg_clickhouse: invalid identifier")); - return ident; - } - - /* Rely on PostgreSQL 's identifier quoting. */ - if (len == 0 || len > NAMEDATALEN - 1) - ereport(ERROR, - errcode(ERRCODE_FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH), - errmsg_internal("pg_clickhouse: invalid identifier")); - return quote_identifier(ident); + */ +const char* +ch_quote_ident(const char* ident) { + /* https://clickhouse.com/docs/sql-reference/syntax#identifiers */ + int len = strlen(ident); + + if (len >= 2 && ((ident[0] == '"' && ident[len - 1] == '"') || + (ident[0] == '`' && ident[len - 1] == '`'))) { + /* + * Make sure it has no unescaped quote character. Allowed escapes: + * + * ": (""|\\.) + * + * `: (``|\\.) + */ + for (int i = 2; i <= len - 2; i++) { + /* Skip escaped character. */ + if (ident[i] == '\\') { + i++; + } + + /* Disallow unescaped quote character. */ + else if (ident[i] == ident[0] && ident[i + 1] != ident[0]) { + ereport( + ERROR, + errcode(ERRCODE_FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH), + errmsg_internal("pg_clickhouse: invalid identifier") + ); + } + } + + /* Allow already quoted identifier. */ + if (len == 2 || len > CH_ESCAPED_NAMEDATALEN - 1) { + ereport( + ERROR, + errcode(ERRCODE_FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH), + errmsg_internal("pg_clickhouse: invalid identifier") + ); + } + return ident; + } + + /* Rely on PostgreSQL 's identifier quoting. */ + if (len == 0 || len > NAMEDATALEN - 1) { + ereport( + ERROR, + errcode(ERRCODE_FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH), + errmsg_internal("pg_clickhouse: invalid identifier") + ); + } + return quote_identifier(ident); } diff --git a/src/shipable.c b/src/shipable.c index edabc7b..ba38f1f 100644 --- a/src/shipable.c +++ b/src/shipable.c @@ -19,37 +19,33 @@ #include "access/transam.h" #include "catalog/dependency.h" +#include "catalog/pg_operator.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" -#include "catalog/pg_operator.h" #include "utils/builtins.h" #include "utils/hsearch.h" #include "utils/inval.h" #include "utils/syscache.h" - /* Hash table for caching the results of shippability lookups */ -static HTAB * ShippableCacheHash = NULL; +static HTAB* ShippableCacheHash = NULL; /* * Hash key for shippability lookups. We include the FDW server OID because * decisions may differ per-server. Otherwise, objects are identified by * their (local!) OID and catalog OID. */ -typedef struct -{ - /* XXX we assume this struct contains no padding bytes */ - Oid objid; /* function/operator/type OID */ - Oid classid; /* OID of its catalog (pg_proc, etc) */ - Oid serverid; /* FDW server we are concerned with */ -} ShippableCacheKey; - -typedef struct -{ - ShippableCacheKey key; /* hash key - must be first */ - bool shippable; -} ShippableCacheEntry; - +typedef struct { + /* XXX we assume this struct contains no padding bytes */ + Oid objid; /* function/operator/type OID */ + Oid classid; /* OID of its catalog (pg_proc, etc) */ + Oid serverid; /* FDW server we are concerned with */ +} ShippableCacheKey; + +typedef struct { + ShippableCacheKey key; /* hash key - must be first */ + bool shippable; +} ShippableCacheEntry; /* * Flush cache entries when pg_foreign_server is updated. @@ -60,49 +56,43 @@ typedef struct * made for them, however. */ static void -InvalidateShippableCacheCallback(Datum arg, int cacheid, uint32 hashvalue) -{ - HASH_SEQ_STATUS status; - ShippableCacheEntry *entry; - - /* - * In principle we could flush only cache entries relating to the - * pg_foreign_server entry being outdated; but that would be more - * complicated, and it's probably not worth the trouble. So for now, just - * flush all entries. - */ - hash_seq_init(&status, ShippableCacheHash); - while ((entry = (ShippableCacheEntry *) hash_seq_search(&status)) != NULL) - { - if (hash_search(ShippableCacheHash, - (void *) &entry->key, - HASH_REMOVE, - NULL) == NULL) - { - elog(ERROR, "hash table corrupted"); - } - } +InvalidateShippableCacheCallback(Datum arg, int cacheid, uint32 hashvalue) { + HASH_SEQ_STATUS status; + ShippableCacheEntry* entry; + + /* + * In principle we could flush only cache entries relating to the + * pg_foreign_server entry being outdated; but that would be more + * complicated, and it's probably not worth the trouble. So for now, just + * flush all entries. + */ + hash_seq_init(&status, ShippableCacheHash); + while ((entry = (ShippableCacheEntry*)hash_seq_search(&status)) != NULL) { + if (hash_search(ShippableCacheHash, (void*)&entry->key, HASH_REMOVE, NULL) == + NULL) { + elog(ERROR, "hash table corrupted"); + } + } } /* * Initialize the backend-lifespan cache of shippability decisions. */ static void -InitializeShippableCache(void) -{ - HASHCTL ctl; - - /* Create the hash table. */ - MemSet(&ctl, 0, sizeof(ctl)); - ctl.keysize = sizeof(ShippableCacheKey); - ctl.entrysize = sizeof(ShippableCacheEntry); - ShippableCacheHash = - hash_create("Shippability cache", 256, &ctl, HASH_ELEM | HASH_BLOBS); - - /* Set up invalidation callback on pg_foreign_server. */ - CacheRegisterSyscacheCallback(FOREIGNSERVEROID, - InvalidateShippableCacheCallback, - (Datum) 0); +InitializeShippableCache(void) { + HASHCTL ctl; + + /* Create the hash table. */ + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(ShippableCacheKey); + ctl.entrysize = sizeof(ShippableCacheEntry); + ShippableCacheHash = + hash_create("Shippability cache", 256, &ctl, HASH_ELEM | HASH_BLOBS); + + /* Set up invalidation callback on pg_foreign_server. */ + CacheRegisterSyscacheCallback( + FOREIGNSERVEROID, InvalidateShippableCacheCallback, (Datum)0 + ); } /* @@ -114,24 +104,22 @@ InitializeShippableCache(void) * additionally have a whitelist of functions/operators declared one at a time. */ static bool -lookup_shippable(Oid objectId, Oid classId, CHFdwRelationInfo * fpinfo) -{ - Oid extensionOid; - - /* - * Is object a member of some extension? (Note: this is a fairly - * expensive lookup, which is why we try to cache the results.) - */ - extensionOid = getExtensionOfObject(classId, objectId); - - /* If so, is that extension in fpinfo->shippable_extensions? */ - if (OidIsValid(extensionOid) && - list_member_oid(fpinfo->shippable_extensions, extensionOid)) - { - return true; - } - - return false; +lookup_shippable(Oid objectId, Oid classId, CHFdwRelationInfo* fpinfo) { + Oid extensionOid; + + /* + * Is object a member of some extension? (Note: this is a fairly + * expensive lookup, which is why we try to cache the results.) + */ + extensionOid = getExtensionOfObject(classId, objectId); + + /* If so, is that extension in fpinfo->shippable_extensions? */ + if (OidIsValid(extensionOid) && + list_member_oid(fpinfo->shippable_extensions, extensionOid)) { + return true; + } + + return false; } /* @@ -152,41 +140,38 @@ lookup_shippable(Oid objectId, Oid classId, CHFdwRelationInfo * fpinfo) * track of that would be a huge exercise. */ bool -chfdw_is_builtin(Oid objectId) -{ - return (objectId < FirstUnpinnedObjectId); +chfdw_is_builtin(Oid objectId) { + return (objectId < FirstUnpinnedObjectId); } static bool -regex_flags_ok(char *flags, bool global_ok) -{ - while (*flags) - { - switch (*flags) - { - case 'i': - case 'm': - case 'n': - case 'p': - case 's': - case 't': - case 'w': - /* Pass through as-is. */ - break; - case 'g': - if (global_ok) - /* Pass through as-is; deparseFuncExpr will remove it. */ - break; - return false; - default: - /* Cannot pass down any other flags */ - return false; - } - flags++; - } - - /* All good. */ - return true; +regex_flags_ok(char* flags, bool global_ok) { + while (*flags) { + switch (*flags) { + case 'i': + case 'm': + case 'n': + case 'p': + case 's': + case 't': + case 'w': + /* Pass through as-is. */ + break; + case 'g': + if (global_ok) { + /* Pass through as-is; deparseFuncExpr will remove it. */ + break; + } + return false; + default: + /* Cannot pass down any other flags */ + return false; + } + flags++; + } + + /* All good. */ + return true; } /* @@ -194,226 +179,223 @@ regex_flags_ok(char *flags, bool global_ok) * Is this object (function/operator/type) shippable to foreign server? */ bool -chfdw_is_shippable(Node * node, Oid objectId, Oid classId, CHFdwRelationInfo * fpinfo, - CustomObjectDef * *outcdef) -{ - ShippableCacheKey key; - ShippableCacheEntry *entry; - - /* - * For operators, check for custom overrides before the builtin shortcut, - * since some builtin operators (e.g. ~, !~, ~*, !~*) need special - * handling. - */ - if (classId == OperatorRelationId) - { - CustomObjectDef *cdef = chfdw_check_for_custom_operator(objectId, NULL); - - if (cdef) - { - switch (cdef->cf_type) - { - case CF_REGEX_MATCH: - case CF_REGEX_NO_MATCH: - case CF_REGEX_ICASE_MATCH: - case CF_REGEX_ICASE_NO_MATCH: - /* Don't pushdown regular expressions if the GUC is false. */ - return chfdw_pushdown_regex_ok(); - default: - return true; - } - } - } - - /* - * For procedures, check for custom overrides before the builtin shortcut - * so the caller gets the CustomObjectDef it needs for deparse. - */ - if (classId == ProcedureRelationId && chfdw_is_builtin(objectId)) - { - CustomObjectDef *cdef = chfdw_check_for_custom_function(objectId); - - if (cdef) - { - if (outcdef != NULL) - *outcdef = cdef; - - /* - * Evaluate certain special functions whose arguments prevent - * pushdown. - */ - switch (cdef->cf_type) - { - case CF_ARRAY_SORT_DESC: - { - /* - * If the boolean argument passed to array_sort(arr, - * bool) is dynamic, we can't push it down. - */ - Expr *desc_arg = (Expr *) list_nth(((FuncExpr *) node)->args, 1); - - if (!IsA(desc_arg, Const)) - /* No support for a dynamic value here. */ - return false; - } - break; - case CF_TO_CHAR: - { - /* format must be a constant with exact CH translation */ - Expr *fmt = (Expr *) list_nth(((FuncExpr *) node)->args, 1); - Const *fmt_const; - char *pgfmt; - bool ok; - - if (!IsA(fmt, Const)) - return false; - fmt_const = (Const *) fmt; - if (fmt_const->constisnull) - return false; - - pgfmt = TextDatumGetCString(fmt_const->constvalue); - ok = chfdw_translate_to_char_format(pgfmt, NULL); - pfree(pgfmt); - if (!ok) - return false; - } - break; - case CF_MATCH: - case CF_SPLIT_BY_REGEX: - case CF_REPLACE_REGEX: - case CF_REGEX_PG_MATCH: - { - Expr *arg; - - /* - * Don't pushdown regular expressions if the GUC is - * false. - */ - if (!chfdw_pushdown_regex_ok()) - return false; - - /* Unshippable if using unsupported or dynamic flags */ - FuncExpr *fn = (FuncExpr *) node; - int flags_idx = cdef->cf_type == CF_REPLACE_REGEX ? 3 : 2; - - if ((list_length(fn->args) >= flags_idx + 1)) - { - arg = (Expr *) list_nth(((FuncExpr *) node)->args, flags_idx); - - if (!IsA(arg, Const)) - /* No support for a dynamic value here. */ - return false; - - if (!regex_flags_ok( - TextDatumGetCString(((Const *) arg)->constvalue), - cdef->cf_type == CF_REPLACE_REGEX - )) - /* Using flags unsupported by ClickHouse. */ - return false; - } - - arg = (Expr *) list_nth(((FuncExpr *) node)->args, 1); - if (!IsA(arg, Const)) - /* No support for a dynamic value here. */ - return false; - - /* - * XXX Additional checks: Examine the regex and: - * - * - don't ship if it starts with "***" - * - * - use a substring function if starts with "***=" - * - * - don't ship if it contains (?xyz) with unsupported - * flags - * - * Also applies to CF_REGEX_MATCH, CF_REGEX_NO_MATCH, - * CF_REGEX_ICASE_MATCH, and CF_REGEX_ICASE_NO_MATCH. - */ - } - break; - default: - break; - } - return true; - } - } - - /* - * Builtin operators and types ship by default. Builtin functions only - * ship when explicitly registered via chfdw_check_for_custom_function, so - * unrecognised builtins fail planner shippability checks rather than - * deparse to a name that may behave differently on ClickHouse. Cast - * coercions are an exception: deparse handles them as cast() or by - * dropping the implicit wrapper, regardless of the underlying funcid. - */ - if (chfdw_is_builtin(objectId)) - { - if (classId != ProcedureRelationId) - return true; - if (node && IsA(node, FuncExpr)) - { - FuncExpr *fe = (FuncExpr *) node; - - if (fe->funcformat == COERCE_IMPLICIT_CAST || - fe->funcformat == COERCE_EXPLICIT_CAST) - return true; - } - return false; - } - - if (classId == ProcedureRelationId) - { - CustomObjectDef *cdef = chfdw_check_for_custom_function(objectId); - - if (outcdef != NULL) - *outcdef = cdef; - - return cdef != NULL; - } - else if (classId == TypeRelationId && chfdw_check_for_custom_type(objectId) != NULL) - return true; - - /* Otherwise, give up if user hasn't specified any shippable extensions. */ - if (fpinfo->shippable_extensions == NIL) - return false; - - /* Initialize cache if first time through. */ - if (!ShippableCacheHash) - { - InitializeShippableCache(); - } - - /* Set up cache hash key */ - key.objid = objectId; - key.classid = classId; - key.serverid = fpinfo->server->serverid; - - /* See if we already cached the result. */ - entry = (ShippableCacheEntry *) - hash_search(ShippableCacheHash, - (void *) &key, - HASH_FIND, - NULL); - - if (!entry) - { - /* Not found in cache, so perform shippability lookup. */ - bool shippable = lookup_shippable(objectId, classId, fpinfo); - - /* - * Don't create a new hash entry until *after* we have the shippable - * result in hand, as the underlying catalog lookups might trigger a - * cache invalidation. - */ - entry = (ShippableCacheEntry *) - hash_search(ShippableCacheHash, - (void *) &key, - HASH_ENTER, - NULL); - - entry->shippable = shippable; - } - - return entry->shippable; +chfdw_is_shippable( + Node* node, + Oid objectId, + Oid classId, + CHFdwRelationInfo* fpinfo, + CustomObjectDef** outcdef +) { + ShippableCacheKey key; + ShippableCacheEntry* entry; + + /* + * For operators, check for custom overrides before the builtin shortcut, + * since some builtin operators (e.g. ~, !~, ~*, !~*) need special + * handling. + */ + if (classId == OperatorRelationId) { + CustomObjectDef* cdef = chfdw_check_for_custom_operator(objectId, NULL); + + if (cdef) { + switch (cdef->cf_type) { + case CF_REGEX_MATCH: + case CF_REGEX_NO_MATCH: + case CF_REGEX_ICASE_MATCH: + case CF_REGEX_ICASE_NO_MATCH: + /* Don't pushdown regular expressions if the GUC is false. */ + return chfdw_pushdown_regex_ok(); + default: + return true; + } + } + } + + /* + * For procedures, check for custom overrides before the builtin shortcut + * so the caller gets the CustomObjectDef it needs for deparse. + */ + if (classId == ProcedureRelationId && chfdw_is_builtin(objectId)) { + CustomObjectDef* cdef = chfdw_check_for_custom_function(objectId); + + if (cdef) { + if (outcdef != NULL) { + *outcdef = cdef; + } + + /* + * Evaluate certain special functions whose arguments prevent + * pushdown. + */ + switch (cdef->cf_type) { + case CF_ARRAY_SORT_DESC: { + /* + * If the boolean argument passed to array_sort(arr, + * bool) is dynamic, we can't push it down. + */ + Expr* desc_arg = (Expr*)list_nth(((FuncExpr*)node)->args, 1); + + if (!IsA(desc_arg, Const)) { + /* No support for a dynamic value here. */ + return false; + } + } break; + case CF_TO_CHAR: { + /* format must be a constant with exact CH translation */ + Expr* fmt = (Expr*)list_nth(((FuncExpr*)node)->args, 1); + Const* fmt_const; + char* pgfmt; + bool ok; + + if (!IsA(fmt, Const)) { + return false; + } + fmt_const = (Const*)fmt; + if (fmt_const->constisnull) { + return false; + } + + pgfmt = TextDatumGetCString(fmt_const->constvalue); + ok = chfdw_translate_to_char_format(pgfmt, NULL); + pfree(pgfmt); + if (!ok) { + return false; + } + } break; + case CF_MATCH: + case CF_SPLIT_BY_REGEX: + case CF_REPLACE_REGEX: + case CF_REGEX_PG_MATCH: { + Expr* arg; + + /* + * Don't pushdown regular expressions if the GUC is + * false. + */ + if (!chfdw_pushdown_regex_ok()) { + return false; + } + + /* Unshippable if using unsupported or dynamic flags */ + FuncExpr* fn = (FuncExpr*)node; + int flags_idx = cdef->cf_type == CF_REPLACE_REGEX ? 3 : 2; + + if ((list_length(fn->args) >= flags_idx + 1)) { + arg = (Expr*)list_nth(((FuncExpr*)node)->args, flags_idx); + + if (!IsA(arg, Const)) { + /* No support for a dynamic value here. */ + return false; + } + + if (!regex_flags_ok( + TextDatumGetCString(((Const*)arg)->constvalue), + cdef->cf_type == CF_REPLACE_REGEX + )) { + /* Using flags unsupported by ClickHouse. */ + return false; + } + } + + arg = (Expr*)list_nth(((FuncExpr*)node)->args, 1); + if (!IsA(arg, Const)) { + /* No support for a dynamic value here. */ + return false; + } + + /* + * XXX Additional checks: Examine the regex and: + * + * - don't ship if it starts with "***" + * + * - use a substring function if starts with "***=" + * + * - don't ship if it contains (?xyz) with unsupported + * flags + * + * Also applies to CF_REGEX_MATCH, CF_REGEX_NO_MATCH, + * CF_REGEX_ICASE_MATCH, and CF_REGEX_ICASE_NO_MATCH. + */ + } break; + default: + break; + } + return true; + } + } + + /* + * Builtin operators and types ship by default. Builtin functions only + * ship when explicitly registered via chfdw_check_for_custom_function, so + * unrecognised builtins fail planner shippability checks rather than + * deparse to a name that may behave differently on ClickHouse. Cast + * coercions are an exception: deparse handles them as cast() or by + * dropping the implicit wrapper, regardless of the underlying funcid. + */ + if (chfdw_is_builtin(objectId)) { + if (classId != ProcedureRelationId) { + return true; + } + if (node && IsA(node, FuncExpr)) { + FuncExpr* fe = (FuncExpr*)node; + + if (fe->funcformat == COERCE_IMPLICIT_CAST || + fe->funcformat == COERCE_EXPLICIT_CAST) { + return true; + } + } + return false; + } + + if (classId == ProcedureRelationId) { + CustomObjectDef* cdef = chfdw_check_for_custom_function(objectId); + + if (outcdef != NULL) { + *outcdef = cdef; + } + + return cdef != NULL; + } else if ( + classId == TypeRelationId && chfdw_check_for_custom_type(objectId) != NULL + ) { + return true; + } + + /* Otherwise, give up if user hasn't specified any shippable extensions. */ + if (fpinfo->shippable_extensions == NIL) { + return false; + } + + /* Initialize cache if first time through. */ + if (!ShippableCacheHash) { + InitializeShippableCache(); + } + + /* Set up cache hash key */ + key.objid = objectId; + key.classid = classId; + key.serverid = fpinfo->server->serverid; + + /* See if we already cached the result. */ + entry = (ShippableCacheEntry*)hash_search( + ShippableCacheHash, (void*)&key, HASH_FIND, NULL + ); + + if (!entry) { + /* Not found in cache, so perform shippability lookup. */ + bool shippable = lookup_shippable(objectId, classId, fpinfo); + + /* + * Don't create a new hash entry until *after* we have the shippable + * result in hand, as the underlying catalog lookups might trigger a + * cache invalidation. + */ + entry = (ShippableCacheEntry*)hash_search( + ShippableCacheHash, (void*)&key, HASH_ENTER, NULL + ); + + entry->shippable = shippable; + } + + return entry->shippable; }