From b97992ca851627555cf70446fee9ca183cfce6a4 Mon Sep 17 00:00:00 2001 From: Dimi Kot Date: Sun, 7 Jun 2026 21:02:25 -0700 Subject: [PATCH] v3.0.1: Move MIT source code and API docs into this repo Pull Request: https://github.com/dimikot/ent-framework/pull/3 (main) --- .github/workflows/ci.yml | 26 + .gitignore | 1 + .prettierrc | 8 + .vscode/extensions.json | 8 + .vscode/tasks.json | 8 + README.md | 12 + barrelsby.json | 15 + docker-compose.postgres.yml | 14 + docs/README.md | 33 + docs/_media/logo-berkshire-swash.svg | 1 + docs/classes/AllowIf.md | 77 + docs/classes/Batcher.md | 75 + docs/classes/CanDeleteOutgoingEdge.md | 72 + docs/classes/CanReadOutgoingEdge.md | 79 + docs/classes/CanUpdateOutgoingEdge.md | 72 + docs/classes/Client.md | 273 + docs/classes/ClientError.md | 57 + docs/classes/Cluster.md | 246 + docs/classes/Configuration.md | 70 + docs/classes/DenyIf.md | 79 + docs/classes/EntAccessError.md | 71 + docs/classes/EntNotFoundError.md | 65 + docs/classes/EntNotInsertableError.md | 65 + docs/classes/EntNotReadableError.md | 65 + docs/classes/EntNotUpdatableError.md | 65 + docs/classes/EntUniqueKeyError.md | 73 + docs/classes/EntValidationError.md | 66 + docs/classes/FieldIs.md | 101 + docs/classes/FuncToPredicate.md | 72 + docs/classes/IDsCache.md | 84 + docs/classes/IDsCacheCanReadIncomingEdge.md | 97 + docs/classes/IDsCacheDeletable.md | 97 + docs/classes/IDsCacheReadable.md | 97 + docs/classes/IDsCacheUpdatable.md | 97 + docs/classes/IncomingEdgeFromVCExists.md | 86 + docs/classes/Inverse.md | 143 + docs/classes/Island.md | 177 + docs/classes/Loader.md | 75 + docs/classes/LocalCache.md | 109 + docs/classes/Or.md | 69 + docs/classes/OutgoingEdgePointsToVC.md | 71 + docs/classes/PgClient.md | 425 ++ docs/classes/PgError.md | 71 + docs/classes/PgQueryCount.md | 96 + docs/classes/PgQueryDelete.md | 96 + docs/classes/PgQueryDeleteWhere.md | 96 + docs/classes/PgQueryExists.md | 96 + docs/classes/PgQueryIDGen.md | 96 + docs/classes/PgQueryInsert.md | 96 + docs/classes/PgQueryLoad.md | 96 + docs/classes/PgQueryLoadBy.md | 97 + docs/classes/PgQuerySelect.md | 96 + docs/classes/PgQuerySelectBy.md | 73 + docs/classes/PgQueryUpdate.md | 73 + docs/classes/PgQueryUpsert.md | 72 + docs/classes/PgRunner.md | 706 ++ docs/classes/PgSchema.md | 333 + docs/classes/PgShardNamer.md | 123 + docs/classes/PgTimelineStorage.md | 119 + docs/classes/QueryBase.md | 107 + docs/classes/QueryCache.md | 147 + docs/classes/QueryPing.md | 80 + docs/classes/Require.md | 77 + docs/classes/RowIs.md | 97 + docs/classes/Rule.md | 80 + docs/classes/Runner.md | 170 + docs/classes/Schema.md | 281 + docs/classes/Shard.md | 111 + docs/classes/ShardError.md | 43 + docs/classes/ShardIsNotDiscoverableError.md | 41 + docs/classes/ShardLocator.md | 147 + docs/classes/ShardNamer.md | 109 + docs/classes/Timeline.md | 164 + docs/classes/TimelineManager.md | 78 + docs/classes/TimelineStorage.md | 84 + docs/classes/ToolPing.md | 55 + docs/classes/ToolScoreboard.md | 73 + docs/classes/Triggers.md | 125 + docs/classes/True.md | 53 + docs/classes/VC.md | 598 ++ docs/classes/VCCaches.md | 89 + docs/classes/VCFlavor.md | 52 + docs/classes/VCHasFlavor.md | 61 + docs/classes/VCTrace.md | 53 + docs/classes/VCWithQueryCache.md | 65 + docs/classes/VCWithStacks.md | 55 + docs/classes/Validation.md | 124 + docs/functions/Base64BufferType.md | 63 + docs/functions/BaseEnt.md | 47 + docs/functions/BigIntArrayType.md | 71 + docs/functions/ByteaBufferType.md | 59 + docs/functions/CacheMixin.md | 31 + docs/functions/ConfigMixin.md | 34 + docs/functions/HelpersMixin.md | 33 + docs/functions/JSONType.md | 65 + docs/functions/PrimitiveMixin.md | 32 + docs/functions/StringArrayType.md | 66 + docs/functions/buildShape.md | 24 + docs/functions/escapeIdent.md | 23 + docs/functions/escapeLiteral.md | 29 + docs/functions/evaluate.md | 72 + docs/functions/isBigintStr.md | 24 + docs/functions/testSpecTypeIntegrity.md | 45 + docs/globals.md | 219 + docs/interfaces/AbstractIs.md | 52 + docs/interfaces/ClientConnectedLoggerProps.md | 21 + docs/interfaces/ClientConnectionIssue.md | 21 + docs/interfaces/ClientEndLoggerProps.md | 23 + docs/interfaces/ClientOptions.md | 24 + docs/interfaces/ClientPingInput.md | 19 + docs/interfaces/ClientQueryLoggerProps.md | 35 + docs/interfaces/ClusterOptions.md | 35 + docs/interfaces/ConfigClass.md | 63 + docs/interfaces/ConfigInstance.md | 13 + docs/interfaces/Ent.md | 54 + docs/interfaces/EntClass.md | 253 + docs/interfaces/EntValidationErrorInfo.md | 18 + docs/interfaces/Handler.md | 25 + docs/interfaces/HelpersClass.md | 90 + docs/interfaces/HelpersInstance.md | 195 + docs/interfaces/IslandOptions.md | 27 + docs/interfaces/LocalCacheOptions.md | 22 + docs/interfaces/Loggers.md | 30 + docs/interfaces/PgClientConn.md | 30 + docs/interfaces/PgClientOptions.md | 43 + docs/interfaces/PgClientSubPoolConfig.md | 23 + docs/interfaces/PgTimelineStorageOptions.md | 22 + docs/interfaces/Predicate.md | 56 + docs/interfaces/PrimitiveInstance.md | 77 + docs/interfaces/Query.md | 43 + docs/interfaces/QueryAnnotation.md | 24 + docs/interfaces/RuleResult.md | 20 + docs/interfaces/RunOnShardErrorLoggerProps.md | 16 + docs/interfaces/SchemaClass.md | 29 + docs/interfaces/ShardNamerOptions.md | 18 + .../StandardSchemaV1FailureResult.md | 18 + docs/interfaces/SwallowedErrorLoggerProps.md | 18 + docs/interfaces/TimelineStorageOptions.md | 20 + docs/interfaces/ToolPingOptions.md | 21 + docs/interfaces/ToolScoreboardOptions.md | 24 + docs/llms-full.md | 6572 +++++++++++++++++ docs/type-aliases/AfterMutationTrigger.md | 28 + docs/type-aliases/AfterUpdateTrigger.md | 30 + docs/type-aliases/AnyClass.md | 21 + docs/type-aliases/BeforeMutationTrigger.md | 28 + docs/type-aliases/BeforeUpdateTrigger.md | 31 + docs/type-aliases/ClientErrorKind.md | 14 + docs/type-aliases/ClientErrorPostAction.md | 13 + docs/type-aliases/ClientRole.md | 16 + docs/type-aliases/ClusterIslands.md | 20 + docs/type-aliases/CountInput.md | 19 + docs/type-aliases/DeleteTrigger.md | 29 + docs/type-aliases/DeleteWhereInput.md | 25 + docs/type-aliases/DepsBuilder.md | 28 + docs/type-aliases/ExistsInput.md | 19 + docs/type-aliases/Field.md | 26 + docs/type-aliases/FieldAliased.md | 20 + docs/type-aliases/FieldIsValidatorPlain.md | 32 + .../FieldIsValidatorStandardSchemaV1.md | 31 + .../FieldIsValidatorZodSafeParse.md | 31 + docs/type-aliases/FieldOfIDType.md | 19 + docs/type-aliases/FieldOfIDTypeRequired.md | 19 + .../type-aliases/FieldOfPotentialUniqueKey.md | 20 + docs/type-aliases/Hints.md | 14 + docs/type-aliases/InsertFieldsOptional.md | 19 + docs/type-aliases/InsertFieldsRequired.md | 19 + docs/type-aliases/InsertInput.md | 20 + docs/type-aliases/InsertTrigger.md | 65 + docs/type-aliases/JSONValue.md | 13 + docs/type-aliases/Literal.md | 15 + docs/type-aliases/LoadByInput.md | 21 + docs/type-aliases/LoadRule.md | 17 + docs/type-aliases/Order.md | 20 + docs/type-aliases/PrimitiveClass.md | 19 + docs/type-aliases/Row.md | 19 + docs/type-aliases/RowIsValidatorPlain.md | 30 + .../RowIsValidatorStandardSchemaV1.md | 30 + .../RowIsValidatorZodSafeParse.md | 30 + docs/type-aliases/RowWithID.md | 19 + docs/type-aliases/RuleDecision.md | 13 + docs/type-aliases/SelectByInput.md | 21 + docs/type-aliases/SelectInput.md | 37 + docs/type-aliases/SelectInputCustom.md | 14 + docs/type-aliases/ShardAffinity.md | 28 + docs/type-aliases/Spec.md | 31 + docs/type-aliases/SpecType.md | 95 + docs/type-aliases/Table.md | 17 + docs/type-aliases/TimelineCaughtUpReason.md | 14 + docs/type-aliases/TriggerInsertInput.md | 24 + docs/type-aliases/TriggerUpdateInput.md | 19 + docs/type-aliases/TriggerUpdateNewRow.md | 21 + .../TriggerUpdateOrDeleteOldRow.md | 22 + docs/type-aliases/UniqueKey.md | 20 + docs/type-aliases/UpdateField.md | 19 + docs/type-aliases/UpdateInput.md | 34 + docs/type-aliases/UpdateOriginalInput.md | 31 + docs/type-aliases/ValidationRules.md | 58 + docs/type-aliases/ValidatorPlainResult.md | 13 + .../ValidatorStandardSchemaResult.md | 13 + .../ValidatorZodSafeParseResult.md | 13 + docs/type-aliases/Value.md | 19 + docs/type-aliases/ValueRequired.md | 19 + docs/type-aliases/Where.md | 47 + docs/type-aliases/WhyClient.md | 20 + docs/type-aliases/WriteRules.md | 23 + docs/variables/GLOBAL_SHARD.md | 13 + docs/variables/GUEST_ID.md | 14 + docs/variables/ID.md | 14 + docs/variables/MASTER.md | 13 + docs/variables/OMNI_ID.md | 16 + docs/variables/STALE_REPLICA.md | 13 + eslint.base.config.mjs | 675 ++ eslint.config.mjs | 20 + gitbook/README.md | 4 +- index.d.ts | 6 +- index.js | 9 +- internal/build.sh | 11 + internal/clean.sh | 5 +- internal/deploy.sh | 12 +- internal/dev.sh | 14 + internal/docs.sh | 5 + internal/init-db.sh | 14 + internal/lint.sh | 4 + internal/llms.sh | 22 + internal/test.sh | 5 + internal/update-package-json.js | 49 - internal/with-docker-compose-up.postgres.sh | 24 + jest.config.base.js | 13 + jest.config.js | 2 + package.json | 58 +- pg-id.config.sql | 7 + pg.d.ts | 2 +- pg.js | 3 +- pnpm-workspace.yaml | 2 + src/__tests__/types.test.ts | 42 + src/abstract/Batcher.ts | 185 + src/abstract/Client.ts | 215 + src/abstract/ClientError.ts | 75 + src/abstract/Cluster.ts | 660 ++ src/abstract/Island.ts | 342 + src/abstract/Loader.ts | 85 + src/abstract/LocalCache.ts | 200 + src/abstract/Loggers.ts | 80 + src/abstract/Query.ts | 11 + src/abstract/QueryAnnotation.ts | 40 + src/abstract/QueryBase.ts | 47 + src/abstract/QueryPing.ts | 18 + src/abstract/Runner.ts | 86 + src/abstract/Schema.ts | 148 + src/abstract/Shard.ts | 154 + src/abstract/ShardError.ts | 19 + src/abstract/ShardIsNotDiscoverableError.ts | 43 + src/abstract/ShardNamer.ts | 65 + src/abstract/Timeline.ts | 125 + src/abstract/TimelineManager.ts | 60 + src/abstract/__tests__/Cluster.test.ts | 110 + src/abstract/__tests__/Island.test.ts | 126 + src/abstract/__tests__/LocalCache.test.ts | 29 + src/abstract/__tests__/Shard.test.ts | 8 + src/abstract/__tests__/Timeline.test.ts | 46 + .../__snapshots__/Cluster.test.ts.snap | 15 + src/abstract/index.ts | 24 + src/abstract/internal/misc.ts | 3 + src/ent/BaseEnt.ts | 40 + src/ent/Configuration.ts | 144 + src/ent/IDsCache.ts | 24 + src/ent/Inverse.ts | 213 + src/ent/QueryCache.ts | 160 + src/ent/ShardAffinity.ts | 24 + src/ent/ShardLocator.ts | 312 + src/ent/TimelineStorage.ts | 46 + src/ent/Triggers.ts | 396 + src/ent/VC.ts | 655 ++ src/ent/VCCaches.ts | 40 + src/ent/VCFlavor.ts | 34 + src/ent/VCTrace.ts | 48 + src/ent/Validation.ts | 258 + src/ent/__tests__/Ent.composite-pk.test.ts | 110 + src/ent/__tests__/Ent.custom-fields.test.ts | 47 + src/ent/__tests__/Ent.errors.test.ts | 80 + src/ent/__tests__/Ent.generic.test.ts | 852 +++ src/ent/__tests__/Ent.shards.test.ts | 119 + src/ent/__tests__/Ent.timeline.test.ts | 104 + src/ent/__tests__/Inverse.test.ts | 646 ++ src/ent/__tests__/QueryCache.test.ts | 53 + src/ent/__tests__/ShardLocator.test.ts | 123 + src/ent/__tests__/Triggers.test.ts | 496 ++ src/ent/__tests__/VC.test.ts | 188 + src/ent/__tests__/Validation.privacy.test.ts | 544 ++ src/ent/__tests__/Validation.validate.test.ts | 315 + .../__snapshots__/Ent.generic.test.ts.snap | 70 + .../__snapshots__/Ent.shards.test.ts.snap | 16 + .../__snapshots__/Inverse.test.ts.snap | 30 + .../__snapshots__/Triggers.test.ts.snap | 560 ++ .../Validation.privacy.test.ts.snap | 235 + .../Validation.validate.test.ts.snap | 100 + src/ent/__tests__/test-utils.ts | 113 + src/ent/errors/EntAccessError.ts | 70 + src/ent/errors/EntNotFoundError.ts | 32 + src/ent/errors/EntNotInsertableError.ts | 15 + src/ent/errors/EntNotReadableError.ts | 17 + src/ent/errors/EntNotUpdatableError.ts | 21 + src/ent/errors/EntUniqueKeyError.ts | 39 + src/ent/errors/EntValidationError.ts | 49 + .../errors/__tests__/EntAccessError.test.ts | 23 + .../errors/__tests__/EntNotFoundError.test.ts | 12 + .../__tests__/EntNotInsertableError.test.ts | 11 + .../__tests__/EntNotReadableError.test.ts | 13 + .../__tests__/EntNotUpdatableError.test.ts | 13 + .../__tests__/EntValidationError.test.ts | 27 + src/ent/index.ts | 47 + src/ent/mixins/CacheMixin.ts | 163 + src/ent/mixins/ConfigMixin.ts | 236 + src/ent/mixins/HelpersMixin.ts | 306 + src/ent/mixins/PrimitiveMixin.ts | 794 ++ src/ent/mixins/__tests__/CacheMixin.test.ts | 147 + src/ent/predicates/AbstractIs.ts | 55 + src/ent/predicates/CanDeleteOutgoingEdge.ts | 45 + src/ent/predicates/CanReadOutgoingEdge.ts | 46 + src/ent/predicates/CanUpdateOutgoingEdge.ts | 46 + src/ent/predicates/FieldIs.ts | 101 + .../predicates/IncomingEdgeFromVCExists.ts | 85 + src/ent/predicates/Or.ts | 60 + src/ent/predicates/OutgoingEdgePointsToVC.ts | 26 + src/ent/predicates/Predicate.ts | 49 + src/ent/predicates/RowIs.ts | 84 + src/ent/predicates/True.ts | 13 + src/ent/predicates/VCHasFlavor.ts | 18 + .../__tests__/CanReadOutgoingEdge.test.ts | 182 + .../internal/maybeThrowEntValidationError.ts | 48 + src/ent/rules/AllowIf.ts | 30 + src/ent/rules/DenyIf.ts | 32 + src/ent/rules/Require.ts | 19 + src/ent/rules/Rule.ts | 54 + src/ent/rules/evaluate.ts | 148 + src/ent/types.ts | 89 + src/helpers/index.ts | 6 + src/helpers/isBigintStr.ts | 13 + src/helpers/testSpecTypeIntegrity.ts | 46 + src/internal/CachedRefreshedValue.ts | 203 + src/internal/DefaultMap.ts | 11 + src/internal/Ref.ts | 6 + src/internal/Registry.ts | 50 + src/internal/WeakTicker.ts | 89 + .../__tests__/CachedRefreshedValue.test.ts | 331 + src/internal/__tests__/Registry.test.ts | 25 + src/internal/__tests__/WeakTicker.test.ts | 50 + .../__tests__/__snapshots__/misc.test.ts.snap | 39 + src/internal/__tests__/deepEqual.test.ts | 437 ++ src/internal/__tests__/misc.test.ts | 84 + src/internal/__tests__/objectId.test.ts | 11 + src/internal/deepEqual.ts | 145 + src/internal/misc.ts | 447 ++ src/internal/objectId.ts | 13 + src/pg/PgClient.ts | 850 +++ src/pg/PgError.ts | 25 + src/pg/PgQueryCount.ts | 80 + src/pg/PgQueryDelete.ts | 74 + src/pg/PgQueryDeleteWhere.ts | 68 + src/pg/PgQueryExists.ts | 87 + src/pg/PgQueryIDGen.ts | 70 + src/pg/PgQueryInsert.ts | 106 + src/pg/PgQueryLoad.ts | 74 + src/pg/PgQueryLoadBy.ts | 86 + src/pg/PgQuerySelect.ts | 232 + src/pg/PgQuerySelectBy.ts | 132 + src/pg/PgQueryUpdate.ts | 191 + src/pg/PgQueryUpsert.ts | 216 + src/pg/PgRunner.ts | 1188 +++ src/pg/PgSchema.ts | 76 + src/pg/PgShardNamer.ts | 56 + src/pg/PgTimelineStorage.ts | 150 + .../batched-inserts.benchmark.ts | 105 + src/pg/__tests__/PgClient.test.ts | 106 + src/pg/__tests__/PgError.test.ts | 10 + src/pg/__tests__/PgSchema.alias-pk.test.ts | 119 + .../__tests__/PgSchema.async-islands.test.ts | 93 + src/pg/__tests__/PgSchema.cas.test.ts | 330 + .../__tests__/PgSchema.composite-pk.test.ts | 302 + .../PgSchema.connect-timeout.test.ts | 127 + src/pg/__tests__/PgSchema.failover.test.ts | 95 + src/pg/__tests__/PgSchema.generic.test.ts | 1440 ++++ .../PgSchema.islands-reconfig.test.ts | 159 + src/pg/__tests__/PgSchema.node-down.test.ts | 117 + src/pg/__tests__/PgSchema.rediscover.test.ts | 99 + src/pg/__tests__/PgSchema.stress.test.ts | 90 + src/pg/__tests__/PgTimelineStorage.test.ts | 61 + .../PgSchema.alias-pk.test.ts.snap | 159 + .../__snapshots__/PgSchema.cas.test.ts.snap | 77 + .../PgSchema.composite-pk.test.ts.snap | 311 + .../PgSchema.generic.test.ts.snap | 1457 ++++ src/pg/__tests__/test-utils.ts | 428 ++ src/pg/helpers/buildShape.ts | 66 + src/pg/helpers/escapeIdent.ts | 8 + src/pg/helpers/escapeLiteral.ts | 35 + src/pg/index.ts | 28 + .../__tests__/buildHintQueries.test.ts | 113 + .../__tests__/parseCompositeRow.test.ts | 27 + src/pg/internal/buildHintQueries.ts | 62 + src/pg/internal/escapeAny.ts | 25 + src/pg/internal/escapeArray.ts | 22 + src/pg/internal/escapeBoolean.ts | 6 + src/pg/internal/escapeComposite.ts | 27 + src/pg/internal/escapeDate.ts | 10 + src/pg/internal/escapeID.ts | 19 + src/pg/internal/escapeIdentComposite.ts | 18 + src/pg/internal/escapeString.ts | 10 + src/pg/internal/escapeStringify.ts | 12 + src/pg/internal/misc.ts | 184 + src/pg/internal/parseCompositeRow.ts | 93 + src/pg/internal/parseLsn.ts | 7 + src/pg/types/BigIntArrayType.ts | 35 + src/pg/types/ByteaBufferType.ts | 20 + src/pg/types/StringArrayType.ts | 35 + .../types/__tests__/BigIntArrayType.test.ts | 52 + .../types/__tests__/ByteaBufferType.test.ts | 21 + .../types/__tests__/StringArrayType.test.ts | 97 + src/tools/ToolPing.ts | 196 + src/tools/ToolScoreboard.ts | 592 ++ src/tools/index.ts | 6 + src/tools/internal/formatTimeWithMs.ts | 19 + src/tools/internal/highlightIf.ts | 9 + src/types.ts | 428 ++ tsconfig.base.json | 32 + tsconfig.json | 8 + typedoc.config.js | 35 + 426 files changed, 48065 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .prettierrc create mode 100644 .vscode/extensions.json create mode 100644 barrelsby.json create mode 100644 docker-compose.postgres.yml create mode 100644 docs/README.md create mode 100644 docs/_media/logo-berkshire-swash.svg create mode 100644 docs/classes/AllowIf.md create mode 100644 docs/classes/Batcher.md create mode 100644 docs/classes/CanDeleteOutgoingEdge.md create mode 100644 docs/classes/CanReadOutgoingEdge.md create mode 100644 docs/classes/CanUpdateOutgoingEdge.md create mode 100644 docs/classes/Client.md create mode 100644 docs/classes/ClientError.md create mode 100644 docs/classes/Cluster.md create mode 100644 docs/classes/Configuration.md create mode 100644 docs/classes/DenyIf.md create mode 100644 docs/classes/EntAccessError.md create mode 100644 docs/classes/EntNotFoundError.md create mode 100644 docs/classes/EntNotInsertableError.md create mode 100644 docs/classes/EntNotReadableError.md create mode 100644 docs/classes/EntNotUpdatableError.md create mode 100644 docs/classes/EntUniqueKeyError.md create mode 100644 docs/classes/EntValidationError.md create mode 100644 docs/classes/FieldIs.md create mode 100644 docs/classes/FuncToPredicate.md create mode 100644 docs/classes/IDsCache.md create mode 100644 docs/classes/IDsCacheCanReadIncomingEdge.md create mode 100644 docs/classes/IDsCacheDeletable.md create mode 100644 docs/classes/IDsCacheReadable.md create mode 100644 docs/classes/IDsCacheUpdatable.md create mode 100644 docs/classes/IncomingEdgeFromVCExists.md create mode 100644 docs/classes/Inverse.md create mode 100644 docs/classes/Island.md create mode 100644 docs/classes/Loader.md create mode 100644 docs/classes/LocalCache.md create mode 100644 docs/classes/Or.md create mode 100644 docs/classes/OutgoingEdgePointsToVC.md create mode 100644 docs/classes/PgClient.md create mode 100644 docs/classes/PgError.md create mode 100644 docs/classes/PgQueryCount.md create mode 100644 docs/classes/PgQueryDelete.md create mode 100644 docs/classes/PgQueryDeleteWhere.md create mode 100644 docs/classes/PgQueryExists.md create mode 100644 docs/classes/PgQueryIDGen.md create mode 100644 docs/classes/PgQueryInsert.md create mode 100644 docs/classes/PgQueryLoad.md create mode 100644 docs/classes/PgQueryLoadBy.md create mode 100644 docs/classes/PgQuerySelect.md create mode 100644 docs/classes/PgQuerySelectBy.md create mode 100644 docs/classes/PgQueryUpdate.md create mode 100644 docs/classes/PgQueryUpsert.md create mode 100644 docs/classes/PgRunner.md create mode 100644 docs/classes/PgSchema.md create mode 100644 docs/classes/PgShardNamer.md create mode 100644 docs/classes/PgTimelineStorage.md create mode 100644 docs/classes/QueryBase.md create mode 100644 docs/classes/QueryCache.md create mode 100644 docs/classes/QueryPing.md create mode 100644 docs/classes/Require.md create mode 100644 docs/classes/RowIs.md create mode 100644 docs/classes/Rule.md create mode 100644 docs/classes/Runner.md create mode 100644 docs/classes/Schema.md create mode 100644 docs/classes/Shard.md create mode 100644 docs/classes/ShardError.md create mode 100644 docs/classes/ShardIsNotDiscoverableError.md create mode 100644 docs/classes/ShardLocator.md create mode 100644 docs/classes/ShardNamer.md create mode 100644 docs/classes/Timeline.md create mode 100644 docs/classes/TimelineManager.md create mode 100644 docs/classes/TimelineStorage.md create mode 100644 docs/classes/ToolPing.md create mode 100644 docs/classes/ToolScoreboard.md create mode 100644 docs/classes/Triggers.md create mode 100644 docs/classes/True.md create mode 100644 docs/classes/VC.md create mode 100644 docs/classes/VCCaches.md create mode 100644 docs/classes/VCFlavor.md create mode 100644 docs/classes/VCHasFlavor.md create mode 100644 docs/classes/VCTrace.md create mode 100644 docs/classes/VCWithQueryCache.md create mode 100644 docs/classes/VCWithStacks.md create mode 100644 docs/classes/Validation.md create mode 100644 docs/functions/Base64BufferType.md create mode 100644 docs/functions/BaseEnt.md create mode 100644 docs/functions/BigIntArrayType.md create mode 100644 docs/functions/ByteaBufferType.md create mode 100644 docs/functions/CacheMixin.md create mode 100644 docs/functions/ConfigMixin.md create mode 100644 docs/functions/HelpersMixin.md create mode 100644 docs/functions/JSONType.md create mode 100644 docs/functions/PrimitiveMixin.md create mode 100644 docs/functions/StringArrayType.md create mode 100644 docs/functions/buildShape.md create mode 100644 docs/functions/escapeIdent.md create mode 100644 docs/functions/escapeLiteral.md create mode 100644 docs/functions/evaluate.md create mode 100644 docs/functions/isBigintStr.md create mode 100644 docs/functions/testSpecTypeIntegrity.md create mode 100644 docs/globals.md create mode 100644 docs/interfaces/AbstractIs.md create mode 100644 docs/interfaces/ClientConnectedLoggerProps.md create mode 100644 docs/interfaces/ClientConnectionIssue.md create mode 100644 docs/interfaces/ClientEndLoggerProps.md create mode 100644 docs/interfaces/ClientOptions.md create mode 100644 docs/interfaces/ClientPingInput.md create mode 100644 docs/interfaces/ClientQueryLoggerProps.md create mode 100644 docs/interfaces/ClusterOptions.md create mode 100644 docs/interfaces/ConfigClass.md create mode 100644 docs/interfaces/ConfigInstance.md create mode 100644 docs/interfaces/Ent.md create mode 100644 docs/interfaces/EntClass.md create mode 100644 docs/interfaces/EntValidationErrorInfo.md create mode 100644 docs/interfaces/Handler.md create mode 100644 docs/interfaces/HelpersClass.md create mode 100644 docs/interfaces/HelpersInstance.md create mode 100644 docs/interfaces/IslandOptions.md create mode 100644 docs/interfaces/LocalCacheOptions.md create mode 100644 docs/interfaces/Loggers.md create mode 100644 docs/interfaces/PgClientConn.md create mode 100644 docs/interfaces/PgClientOptions.md create mode 100644 docs/interfaces/PgClientSubPoolConfig.md create mode 100644 docs/interfaces/PgTimelineStorageOptions.md create mode 100644 docs/interfaces/Predicate.md create mode 100644 docs/interfaces/PrimitiveInstance.md create mode 100644 docs/interfaces/Query.md create mode 100644 docs/interfaces/QueryAnnotation.md create mode 100644 docs/interfaces/RuleResult.md create mode 100644 docs/interfaces/RunOnShardErrorLoggerProps.md create mode 100644 docs/interfaces/SchemaClass.md create mode 100644 docs/interfaces/ShardNamerOptions.md create mode 100644 docs/interfaces/StandardSchemaV1FailureResult.md create mode 100644 docs/interfaces/SwallowedErrorLoggerProps.md create mode 100644 docs/interfaces/TimelineStorageOptions.md create mode 100644 docs/interfaces/ToolPingOptions.md create mode 100644 docs/interfaces/ToolScoreboardOptions.md create mode 100644 docs/llms-full.md create mode 100644 docs/type-aliases/AfterMutationTrigger.md create mode 100644 docs/type-aliases/AfterUpdateTrigger.md create mode 100644 docs/type-aliases/AnyClass.md create mode 100644 docs/type-aliases/BeforeMutationTrigger.md create mode 100644 docs/type-aliases/BeforeUpdateTrigger.md create mode 100644 docs/type-aliases/ClientErrorKind.md create mode 100644 docs/type-aliases/ClientErrorPostAction.md create mode 100644 docs/type-aliases/ClientRole.md create mode 100644 docs/type-aliases/ClusterIslands.md create mode 100644 docs/type-aliases/CountInput.md create mode 100644 docs/type-aliases/DeleteTrigger.md create mode 100644 docs/type-aliases/DeleteWhereInput.md create mode 100644 docs/type-aliases/DepsBuilder.md create mode 100644 docs/type-aliases/ExistsInput.md create mode 100644 docs/type-aliases/Field.md create mode 100644 docs/type-aliases/FieldAliased.md create mode 100644 docs/type-aliases/FieldIsValidatorPlain.md create mode 100644 docs/type-aliases/FieldIsValidatorStandardSchemaV1.md create mode 100644 docs/type-aliases/FieldIsValidatorZodSafeParse.md create mode 100644 docs/type-aliases/FieldOfIDType.md create mode 100644 docs/type-aliases/FieldOfIDTypeRequired.md create mode 100644 docs/type-aliases/FieldOfPotentialUniqueKey.md create mode 100644 docs/type-aliases/Hints.md create mode 100644 docs/type-aliases/InsertFieldsOptional.md create mode 100644 docs/type-aliases/InsertFieldsRequired.md create mode 100644 docs/type-aliases/InsertInput.md create mode 100644 docs/type-aliases/InsertTrigger.md create mode 100644 docs/type-aliases/JSONValue.md create mode 100644 docs/type-aliases/Literal.md create mode 100644 docs/type-aliases/LoadByInput.md create mode 100644 docs/type-aliases/LoadRule.md create mode 100644 docs/type-aliases/Order.md create mode 100644 docs/type-aliases/PrimitiveClass.md create mode 100644 docs/type-aliases/Row.md create mode 100644 docs/type-aliases/RowIsValidatorPlain.md create mode 100644 docs/type-aliases/RowIsValidatorStandardSchemaV1.md create mode 100644 docs/type-aliases/RowIsValidatorZodSafeParse.md create mode 100644 docs/type-aliases/RowWithID.md create mode 100644 docs/type-aliases/RuleDecision.md create mode 100644 docs/type-aliases/SelectByInput.md create mode 100644 docs/type-aliases/SelectInput.md create mode 100644 docs/type-aliases/SelectInputCustom.md create mode 100644 docs/type-aliases/ShardAffinity.md create mode 100644 docs/type-aliases/Spec.md create mode 100644 docs/type-aliases/SpecType.md create mode 100644 docs/type-aliases/Table.md create mode 100644 docs/type-aliases/TimelineCaughtUpReason.md create mode 100644 docs/type-aliases/TriggerInsertInput.md create mode 100644 docs/type-aliases/TriggerUpdateInput.md create mode 100644 docs/type-aliases/TriggerUpdateNewRow.md create mode 100644 docs/type-aliases/TriggerUpdateOrDeleteOldRow.md create mode 100644 docs/type-aliases/UniqueKey.md create mode 100644 docs/type-aliases/UpdateField.md create mode 100644 docs/type-aliases/UpdateInput.md create mode 100644 docs/type-aliases/UpdateOriginalInput.md create mode 100644 docs/type-aliases/ValidationRules.md create mode 100644 docs/type-aliases/ValidatorPlainResult.md create mode 100644 docs/type-aliases/ValidatorStandardSchemaResult.md create mode 100644 docs/type-aliases/ValidatorZodSafeParseResult.md create mode 100644 docs/type-aliases/Value.md create mode 100644 docs/type-aliases/ValueRequired.md create mode 100644 docs/type-aliases/Where.md create mode 100644 docs/type-aliases/WhyClient.md create mode 100644 docs/type-aliases/WriteRules.md create mode 100644 docs/variables/GLOBAL_SHARD.md create mode 100644 docs/variables/GUEST_ID.md create mode 100644 docs/variables/ID.md create mode 100644 docs/variables/MASTER.md create mode 100644 docs/variables/OMNI_ID.md create mode 100644 docs/variables/STALE_REPLICA.md create mode 100644 eslint.base.config.mjs create mode 100644 eslint.config.mjs create mode 100755 internal/build.sh create mode 100755 internal/dev.sh create mode 100755 internal/docs.sh create mode 100755 internal/init-db.sh create mode 100755 internal/lint.sh create mode 100755 internal/llms.sh create mode 100755 internal/test.sh delete mode 100755 internal/update-package-json.js create mode 100755 internal/with-docker-compose-up.postgres.sh create mode 100644 jest.config.base.js create mode 100644 jest.config.js create mode 100644 pg-id.config.sql create mode 100644 pnpm-workspace.yaml create mode 100644 src/__tests__/types.test.ts create mode 100644 src/abstract/Batcher.ts create mode 100644 src/abstract/Client.ts create mode 100644 src/abstract/ClientError.ts create mode 100644 src/abstract/Cluster.ts create mode 100644 src/abstract/Island.ts create mode 100644 src/abstract/Loader.ts create mode 100644 src/abstract/LocalCache.ts create mode 100644 src/abstract/Loggers.ts create mode 100644 src/abstract/Query.ts create mode 100644 src/abstract/QueryAnnotation.ts create mode 100644 src/abstract/QueryBase.ts create mode 100644 src/abstract/QueryPing.ts create mode 100644 src/abstract/Runner.ts create mode 100644 src/abstract/Schema.ts create mode 100644 src/abstract/Shard.ts create mode 100644 src/abstract/ShardError.ts create mode 100644 src/abstract/ShardIsNotDiscoverableError.ts create mode 100644 src/abstract/ShardNamer.ts create mode 100644 src/abstract/Timeline.ts create mode 100644 src/abstract/TimelineManager.ts create mode 100644 src/abstract/__tests__/Cluster.test.ts create mode 100644 src/abstract/__tests__/Island.test.ts create mode 100644 src/abstract/__tests__/LocalCache.test.ts create mode 100644 src/abstract/__tests__/Shard.test.ts create mode 100644 src/abstract/__tests__/Timeline.test.ts create mode 100644 src/abstract/__tests__/__snapshots__/Cluster.test.ts.snap create mode 100644 src/abstract/index.ts create mode 100644 src/abstract/internal/misc.ts create mode 100644 src/ent/BaseEnt.ts create mode 100644 src/ent/Configuration.ts create mode 100644 src/ent/IDsCache.ts create mode 100644 src/ent/Inverse.ts create mode 100644 src/ent/QueryCache.ts create mode 100644 src/ent/ShardAffinity.ts create mode 100644 src/ent/ShardLocator.ts create mode 100644 src/ent/TimelineStorage.ts create mode 100644 src/ent/Triggers.ts create mode 100644 src/ent/VC.ts create mode 100644 src/ent/VCCaches.ts create mode 100644 src/ent/VCFlavor.ts create mode 100644 src/ent/VCTrace.ts create mode 100644 src/ent/Validation.ts create mode 100644 src/ent/__tests__/Ent.composite-pk.test.ts create mode 100644 src/ent/__tests__/Ent.custom-fields.test.ts create mode 100644 src/ent/__tests__/Ent.errors.test.ts create mode 100644 src/ent/__tests__/Ent.generic.test.ts create mode 100644 src/ent/__tests__/Ent.shards.test.ts create mode 100644 src/ent/__tests__/Ent.timeline.test.ts create mode 100644 src/ent/__tests__/Inverse.test.ts create mode 100644 src/ent/__tests__/QueryCache.test.ts create mode 100644 src/ent/__tests__/ShardLocator.test.ts create mode 100644 src/ent/__tests__/Triggers.test.ts create mode 100644 src/ent/__tests__/VC.test.ts create mode 100644 src/ent/__tests__/Validation.privacy.test.ts create mode 100644 src/ent/__tests__/Validation.validate.test.ts create mode 100644 src/ent/__tests__/__snapshots__/Ent.generic.test.ts.snap create mode 100644 src/ent/__tests__/__snapshots__/Ent.shards.test.ts.snap create mode 100644 src/ent/__tests__/__snapshots__/Inverse.test.ts.snap create mode 100644 src/ent/__tests__/__snapshots__/Triggers.test.ts.snap create mode 100644 src/ent/__tests__/__snapshots__/Validation.privacy.test.ts.snap create mode 100644 src/ent/__tests__/__snapshots__/Validation.validate.test.ts.snap create mode 100644 src/ent/__tests__/test-utils.ts create mode 100644 src/ent/errors/EntAccessError.ts create mode 100644 src/ent/errors/EntNotFoundError.ts create mode 100644 src/ent/errors/EntNotInsertableError.ts create mode 100644 src/ent/errors/EntNotReadableError.ts create mode 100644 src/ent/errors/EntNotUpdatableError.ts create mode 100644 src/ent/errors/EntUniqueKeyError.ts create mode 100644 src/ent/errors/EntValidationError.ts create mode 100644 src/ent/errors/__tests__/EntAccessError.test.ts create mode 100644 src/ent/errors/__tests__/EntNotFoundError.test.ts create mode 100644 src/ent/errors/__tests__/EntNotInsertableError.test.ts create mode 100644 src/ent/errors/__tests__/EntNotReadableError.test.ts create mode 100644 src/ent/errors/__tests__/EntNotUpdatableError.test.ts create mode 100644 src/ent/errors/__tests__/EntValidationError.test.ts create mode 100644 src/ent/index.ts create mode 100644 src/ent/mixins/CacheMixin.ts create mode 100644 src/ent/mixins/ConfigMixin.ts create mode 100644 src/ent/mixins/HelpersMixin.ts create mode 100644 src/ent/mixins/PrimitiveMixin.ts create mode 100644 src/ent/mixins/__tests__/CacheMixin.test.ts create mode 100644 src/ent/predicates/AbstractIs.ts create mode 100644 src/ent/predicates/CanDeleteOutgoingEdge.ts create mode 100644 src/ent/predicates/CanReadOutgoingEdge.ts create mode 100644 src/ent/predicates/CanUpdateOutgoingEdge.ts create mode 100644 src/ent/predicates/FieldIs.ts create mode 100644 src/ent/predicates/IncomingEdgeFromVCExists.ts create mode 100644 src/ent/predicates/Or.ts create mode 100644 src/ent/predicates/OutgoingEdgePointsToVC.ts create mode 100644 src/ent/predicates/Predicate.ts create mode 100644 src/ent/predicates/RowIs.ts create mode 100644 src/ent/predicates/True.ts create mode 100644 src/ent/predicates/VCHasFlavor.ts create mode 100644 src/ent/predicates/__tests__/CanReadOutgoingEdge.test.ts create mode 100644 src/ent/predicates/internal/maybeThrowEntValidationError.ts create mode 100644 src/ent/rules/AllowIf.ts create mode 100644 src/ent/rules/DenyIf.ts create mode 100644 src/ent/rules/Require.ts create mode 100644 src/ent/rules/Rule.ts create mode 100644 src/ent/rules/evaluate.ts create mode 100644 src/ent/types.ts create mode 100644 src/helpers/index.ts create mode 100644 src/helpers/isBigintStr.ts create mode 100644 src/helpers/testSpecTypeIntegrity.ts create mode 100644 src/internal/CachedRefreshedValue.ts create mode 100644 src/internal/DefaultMap.ts create mode 100644 src/internal/Ref.ts create mode 100644 src/internal/Registry.ts create mode 100644 src/internal/WeakTicker.ts create mode 100644 src/internal/__tests__/CachedRefreshedValue.test.ts create mode 100644 src/internal/__tests__/Registry.test.ts create mode 100644 src/internal/__tests__/WeakTicker.test.ts create mode 100644 src/internal/__tests__/__snapshots__/misc.test.ts.snap create mode 100644 src/internal/__tests__/deepEqual.test.ts create mode 100644 src/internal/__tests__/misc.test.ts create mode 100644 src/internal/__tests__/objectId.test.ts create mode 100644 src/internal/deepEqual.ts create mode 100644 src/internal/misc.ts create mode 100644 src/internal/objectId.ts create mode 100644 src/pg/PgClient.ts create mode 100644 src/pg/PgError.ts create mode 100644 src/pg/PgQueryCount.ts create mode 100644 src/pg/PgQueryDelete.ts create mode 100644 src/pg/PgQueryDeleteWhere.ts create mode 100644 src/pg/PgQueryExists.ts create mode 100644 src/pg/PgQueryIDGen.ts create mode 100644 src/pg/PgQueryInsert.ts create mode 100644 src/pg/PgQueryLoad.ts create mode 100644 src/pg/PgQueryLoadBy.ts create mode 100644 src/pg/PgQuerySelect.ts create mode 100644 src/pg/PgQuerySelectBy.ts create mode 100644 src/pg/PgQueryUpdate.ts create mode 100644 src/pg/PgQueryUpsert.ts create mode 100644 src/pg/PgRunner.ts create mode 100644 src/pg/PgSchema.ts create mode 100644 src/pg/PgShardNamer.ts create mode 100644 src/pg/PgTimelineStorage.ts create mode 100644 src/pg/__benchmarks__/batched-inserts.benchmark.ts create mode 100644 src/pg/__tests__/PgClient.test.ts create mode 100644 src/pg/__tests__/PgError.test.ts create mode 100644 src/pg/__tests__/PgSchema.alias-pk.test.ts create mode 100644 src/pg/__tests__/PgSchema.async-islands.test.ts create mode 100644 src/pg/__tests__/PgSchema.cas.test.ts create mode 100644 src/pg/__tests__/PgSchema.composite-pk.test.ts create mode 100644 src/pg/__tests__/PgSchema.connect-timeout.test.ts create mode 100644 src/pg/__tests__/PgSchema.failover.test.ts create mode 100644 src/pg/__tests__/PgSchema.generic.test.ts create mode 100644 src/pg/__tests__/PgSchema.islands-reconfig.test.ts create mode 100644 src/pg/__tests__/PgSchema.node-down.test.ts create mode 100644 src/pg/__tests__/PgSchema.rediscover.test.ts create mode 100644 src/pg/__tests__/PgSchema.stress.test.ts create mode 100644 src/pg/__tests__/PgTimelineStorage.test.ts create mode 100644 src/pg/__tests__/__snapshots__/PgSchema.alias-pk.test.ts.snap create mode 100644 src/pg/__tests__/__snapshots__/PgSchema.cas.test.ts.snap create mode 100644 src/pg/__tests__/__snapshots__/PgSchema.composite-pk.test.ts.snap create mode 100644 src/pg/__tests__/__snapshots__/PgSchema.generic.test.ts.snap create mode 100644 src/pg/__tests__/test-utils.ts create mode 100644 src/pg/helpers/buildShape.ts create mode 100644 src/pg/helpers/escapeIdent.ts create mode 100644 src/pg/helpers/escapeLiteral.ts create mode 100644 src/pg/index.ts create mode 100644 src/pg/internal/__tests__/buildHintQueries.test.ts create mode 100644 src/pg/internal/__tests__/parseCompositeRow.test.ts create mode 100644 src/pg/internal/buildHintQueries.ts create mode 100644 src/pg/internal/escapeAny.ts create mode 100644 src/pg/internal/escapeArray.ts create mode 100644 src/pg/internal/escapeBoolean.ts create mode 100644 src/pg/internal/escapeComposite.ts create mode 100644 src/pg/internal/escapeDate.ts create mode 100644 src/pg/internal/escapeID.ts create mode 100644 src/pg/internal/escapeIdentComposite.ts create mode 100644 src/pg/internal/escapeString.ts create mode 100644 src/pg/internal/escapeStringify.ts create mode 100644 src/pg/internal/misc.ts create mode 100644 src/pg/internal/parseCompositeRow.ts create mode 100644 src/pg/internal/parseLsn.ts create mode 100644 src/pg/types/BigIntArrayType.ts create mode 100644 src/pg/types/ByteaBufferType.ts create mode 100644 src/pg/types/StringArrayType.ts create mode 100644 src/pg/types/__tests__/BigIntArrayType.test.ts create mode 100644 src/pg/types/__tests__/ByteaBufferType.test.ts create mode 100644 src/pg/types/__tests__/StringArrayType.test.ts create mode 100644 src/tools/ToolPing.ts create mode 100644 src/tools/ToolScoreboard.ts create mode 100644 src/tools/index.ts create mode 100644 src/tools/internal/formatTimeWithMs.ts create mode 100644 src/tools/internal/highlightIf.ts create mode 100644 src/types.ts create mode 100644 tsconfig.base.json create mode 100644 tsconfig.json create mode 100644 typedoc.config.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0f16d30 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: "CI Full Run" +on: + pull_request: + branches: + - main + - grok/*/* + push: + branches: + - main +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: ["22.x"] + steps: + - uses: actions/checkout@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - run: npm install -g pnpm --force + - run: pnpm install + - run: pnpm run build + - run: pnpm run lint + - run: pnpm run test diff --git a/.gitignore b/.gitignore index 924fb31..e12f17e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules package-lock.json yarn.lock pnpm-lock.yaml +dist .DS_Store *.log *.tmp diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..f5688b5 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,8 @@ +{ + "overrides": [ + { + "files": ["*.ts", "*.tsx"], + "options": { "parser": "typescript" } + } + ] +} diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..69c81d9 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "mhutchie.git-graph", + "trentrand.git-rebase-shortcuts" + ] +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index a759949..7076e3d 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,6 +1,14 @@ { "version": "2.0.0", "tasks": [ + { + "label": "git grok: push local commits as individual PRs", + "detail": "Install git-grok first: https://github.com/dimikot/git-grok", + "type": "shell", + "command": "git grok", + "problemMatcher": [], + "hide": false + }, { "label": "git rebase --interactive", "detail": "Opens a UI for interactive rebase (install \"Git rebase shortcuts\" extension).", diff --git a/README.md b/README.md index bf1dc37..7fd6d5b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +# Ent Framework + +
+ +The TypeScript library for working with microsharded PostgreSQL databases. + +* [Getting Started and Tutorials](https://ent-framework.net) +* [API documentation](https://github.com/dimikot/ent-framework/blob/main/docs/globals.md) +* [Source code](https://github.com/dimikot/ent-framework/tree/main/src) +* [Ent Framework's Discord](https://discord.gg/QXvN6VTCKS) + +#### Core Features 1. **Graph-like representation of entities.** With Ent Framework, you represent each Ent (a domain object of your business logic) as a TypeScript class with immutable properties. An Ent class instance maps to one row of some table in a relational database (like PostgreSQL). It may look similar to ORM, but has many aspects that traditional ORMs don't have. 2. **Row-level security in a graph (privacy layer).** You manage data as a graph where each node is an Ent instance, and each edge is a field link (think of foreign keys) to other Ents. To be allowed to read (or update/delete) some Ent, you define a set of explicit rules like "user can read EntA if they can read EntB or EntC". And, consequently, in EntB you define its own set of rules, like "user can read EntB if they can read EntD". diff --git a/barrelsby.json b/barrelsby.json new file mode 100644 index 0000000..48dc270 --- /dev/null +++ b/barrelsby.json @@ -0,0 +1,15 @@ +{ + "name": "index", + "directory": [ + "./src/abstract", + "./src/ent", + "./src/helpers", + "./src/pg", + "./src/tools" + ], + "exclude": [ + "ent-framework\/.*\/__", + "ent-framework\/.*\/internal" + ], + "delete": true +} diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml new file mode 100644 index 0000000..6b232a8 --- /dev/null +++ b/docker-compose.postgres.yml @@ -0,0 +1,14 @@ +services: + postgres: + image: postgres:16 + ports: + - "${PGPORT:?err}:${PGPORT}" + environment: + POSTGRES_PASSWORD: postgres + PGDATA: /tmp/postgresql + POSTGRES_INITDB_ARGS: "-c port=$PGPORT -c max_connections=2000 -c synchronous_commit=off -c wal_level=logical" + healthcheck: + test: "PGPORT=$PGPORT pg_isready -U postgres" + interval: 0.3s + timeout: 20s + retries: 10 diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..93ab141 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,33 @@ +**ent-framework** + +*** + +# Ent Framework + +
+ +The TypeScript library for working with microsharded PostgreSQL databases. + +* [Getting Started and Tutorials](https://ent-framework.net) +* [API documentation](https://github.com/dimikot/ent-framework/blob/main/docs/globals.md) +* [Source code](https://github.com/dimikot/ent-framework/tree/main/src) +* [Ent Framework's Discord](https://discord.gg/QXvN6VTCKS) + +#### Core Features + +1. **Graph-like representation of entities.** With Ent Framework, you represent each Ent (a domain object of your business logic) as a TypeScript class with immutable properties. An Ent class instance maps to one row of some table in a relational database (like PostgreSQL). It may look similar to ORM, but has many aspects that traditional ORMs don't have. +2. **Row-level security in a graph (privacy layer).** You manage data as a graph where each node is an Ent instance, and each edge is a field link (think of foreign keys) to other Ents. To be allowed to read (or update/delete) some Ent, you define a set of explicit rules like "user can read EntA if they can read EntB or EntC". And, consequently, in EntB you define its own set of rules, like "user can read EntB if they can read EntD". +3. **Query batching and coalescing.** Ent Framework holistically solves the "N+1 selects" problem commonly known in ORM world. You still write you code as if you work with individual Ents and individual IDs, and the framework magically takes care of sending batched requests (both read and write) to the underlying relational database. You do not work with lists and JOINs anymore. +4. **Microsharding and replication lag tracking support out of the box.** Splitting your database horizontally is like a breeze now: Ent Framework takes care of routing the requests to the proper microshards. When scaling reads, Ent Framework knows whether a replica node is "good enough" for that particular query. It automatically uses the proper replica when possible, falling back to master when not. +5. **Pluggable to your existing relational database.** If your project already uses some ORM or runs raw SQL queries, Ent Framework can be plugged in. +6. **Tens of other features.** Some examples: cross-microshards foreign keys, composite fields, triggers, build-in caching etc. + +#### Installation + +``` +npm add ent-framework +pnpm add ent-framework +yarn add ent-framework +``` + +
diff --git a/docs/_media/logo-berkshire-swash.svg b/docs/_media/logo-berkshire-swash.svg new file mode 100644 index 0000000..6ccce58 --- /dev/null +++ b/docs/_media/logo-berkshire-swash.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/classes/AllowIf.md b/docs/classes/AllowIf.md new file mode 100644 index 0000000..8b2562b --- /dev/null +++ b/docs/classes/AllowIf.md @@ -0,0 +1,77 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / AllowIf + +# Class: AllowIf\ + +Defined in: [src/ent/rules/AllowIf.ts:12](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/AllowIf.ts#L12) + +Returns ALLOW if the predicate succeeds, otherwise SKIP. +- Used mostly for read permission checks. +- This rule may still throw an exception if the exception is a wild one (not + derived from EntAccessError). + +## Extends + +- [`Rule`](Rule.md)\<`TInput`\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` *extends* `object` | + +## Constructors + +### new AllowIf() + +> **new AllowIf**\<`TInput`\>(`predicate`): [`AllowIf`](AllowIf.md)\<`TInput`\> + +Defined in: [src/ent/rules/Rule.ts:43](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Rule.ts#L43) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `predicate` | [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> \| (`vc`, `input`) => `boolean` \| `Promise`\<`boolean`\> | + +#### Returns + +[`AllowIf`](AllowIf.md)\<`TInput`\> + +#### Inherited from + +[`Rule`](Rule.md).[`constructor`](Rule.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `_TAG` | `"AllowIf"` | +| `predicate` | [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> | +| `name` | `string` | + +## Methods + +### evaluate() + +> **evaluate**(`vc`, `input`): `Promise`\<[`RuleResult`](../interfaces/RuleResult.md)\> + +Defined in: [src/ent/rules/AllowIf.ts:15](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/AllowIf.ts#L15) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `input` | `TInput` | + +#### Returns + +`Promise`\<[`RuleResult`](../interfaces/RuleResult.md)\> + +#### Overrides + +[`Rule`](Rule.md).[`evaluate`](Rule.md#evaluate) diff --git a/docs/classes/Batcher.md b/docs/classes/Batcher.md new file mode 100644 index 0000000..4010d00 --- /dev/null +++ b/docs/classes/Batcher.md @@ -0,0 +1,75 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Batcher + +# Class: Batcher\ + +Defined in: [src/abstract/Batcher.ts:19](https://github.com/clickup/ent-framework/blob/master/src/abstract/Batcher.ts#L19) + +Batcher is similar to DataLoader, but with a few important differences: +1. It's strongly typed not only for the output, but for input too. And input + can be arbitrary, not only strings (e.g. rows). +2. It does requests dedupping for all queries (including selects). +3. It's not limited by read-only requests like DataLoader, and thus it + doesn't to any caching. Caching is delegated to some other layer (either + above Batcher or in Runner). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` | +| `TOutput` | + +## Constructors + +### new Batcher() + +> **new Batcher**\<`TInput`, `TOutput`\>(`runner`, `batchDelayMs`, `disableBatching`): [`Batcher`](Batcher.md)\<`TInput`, `TOutput`\> + +Defined in: [src/abstract/Batcher.ts:90](https://github.com/clickup/ent-framework/blob/master/src/abstract/Batcher.ts#L90) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `runner` | [`Runner`](Runner.md)\<`TInput`, `TOutput`\> | +| `batchDelayMs` | `MaybeCallable`\<`number`\> | +| `disableBatching` | `boolean` | + +#### Returns + +[`Batcher`](Batcher.md)\<`TInput`, `TOutput`\> + +## Methods + +### flushQueue() + +> `protected` **flushQueue**(): `Promise`\<`void`\> + +Defined in: [src/abstract/Batcher.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/Batcher.ts#L32) + +#### Returns + +`Promise`\<`void`\> + +*** + +### run() + +> **run**(`input`, `annotation`): `Promise`\<`TOutput`\> + +Defined in: [src/abstract/Batcher.ts:96](https://github.com/clickup/ent-framework/blob/master/src/abstract/Batcher.ts#L96) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | `TInput` | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`TOutput`\> diff --git a/docs/classes/CanDeleteOutgoingEdge.md b/docs/classes/CanDeleteOutgoingEdge.md new file mode 100644 index 0000000..e91bc1a --- /dev/null +++ b/docs/classes/CanDeleteOutgoingEdge.md @@ -0,0 +1,72 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / CanDeleteOutgoingEdge + +# Class: CanDeleteOutgoingEdge\ + +Defined in: [src/ent/predicates/CanDeleteOutgoingEdge.ts:10](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanDeleteOutgoingEdge.ts#L10) + +Checks that an Ent available via a field can be deleted, or Ent doesn't exist +(e.g. Ent is orphaned). See CanReadOutgoingEdge comments for more details. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TField` *extends* `string` | + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<`Record`\<`TField`, `string` \| `null`\>\> + +## Constructors + +### new CanDeleteOutgoingEdge() + +> **new CanDeleteOutgoingEdge**\<`TField`\>(`field`, `toEntClass`): [`CanDeleteOutgoingEdge`](CanDeleteOutgoingEdge.md)\<`TField`\> + +Defined in: [src/ent/predicates/CanDeleteOutgoingEdge.ts:15](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanDeleteOutgoingEdge.ts#L15) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `field` | `TField` | +| `toEntClass` | [`EntClass`](../interfaces/EntClass.md) | + +#### Returns + +[`CanDeleteOutgoingEdge`](CanDeleteOutgoingEdge.md)\<`TField`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | +| `field` | `TField` | +| `toEntClass` | [`EntClass`](../interfaces/EntClass.md) | + +## Methods + +### check() + +> **check**(`vc`, `row`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/CanDeleteOutgoingEdge.ts:22](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanDeleteOutgoingEdge.ts#L22) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | `Record`\<`TField`, `null` \| `string`\> | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/CanReadOutgoingEdge.md b/docs/classes/CanReadOutgoingEdge.md new file mode 100644 index 0000000..f302fa5 --- /dev/null +++ b/docs/classes/CanReadOutgoingEdge.md @@ -0,0 +1,79 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / CanReadOutgoingEdge + +# Class: CanReadOutgoingEdge\ + +Defined in: [src/ent/predicates/CanReadOutgoingEdge.ts:18](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanReadOutgoingEdge.ts#L18) + +Checks that an ent which a field is pointing to is readable: + +EntOur[company_id] ---> EntCompany[id] + +This predicate delegates the readability permission check for the current ent +to another ent with ID equals to the value of our ent's field. + +- field = user_id in the above example +- toEntClass = EntCompany in the above example + +## Type Parameters + +| Type Parameter | +| ------ | +| `TField` *extends* `string` | + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<`Record`\<`TField`, `string` \| `null`\>\> + +## Constructors + +### new CanReadOutgoingEdge() + +> **new CanReadOutgoingEdge**\<`TField`\>(`field`, `toEntClass`): [`CanReadOutgoingEdge`](CanReadOutgoingEdge.md)\<`TField`\> + +Defined in: [src/ent/predicates/CanReadOutgoingEdge.ts:23](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanReadOutgoingEdge.ts#L23) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `field` | `TField` | +| `toEntClass` | [`EntClass`](../interfaces/EntClass.md) | + +#### Returns + +[`CanReadOutgoingEdge`](CanReadOutgoingEdge.md)\<`TField`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | +| `field` | `TField` | +| `toEntClass` | [`EntClass`](../interfaces/EntClass.md) | + +## Methods + +### check() + +> **check**(`vc`, `row`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/CanReadOutgoingEdge.ts:30](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanReadOutgoingEdge.ts#L30) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | `Record`\<`TField`, `null` \| `string`\> | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/CanUpdateOutgoingEdge.md b/docs/classes/CanUpdateOutgoingEdge.md new file mode 100644 index 0000000..5cf14e6 --- /dev/null +++ b/docs/classes/CanUpdateOutgoingEdge.md @@ -0,0 +1,72 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / CanUpdateOutgoingEdge + +# Class: CanUpdateOutgoingEdge\ + +Defined in: [src/ent/predicates/CanUpdateOutgoingEdge.ts:10](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanUpdateOutgoingEdge.ts#L10) + +Checks that an Ent available via a field is updatable. See +CanReadOutgoingEdge comments for more details. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TField` *extends* `string` | + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<`Record`\<`TField`, `string` \| `null`\>\> + +## Constructors + +### new CanUpdateOutgoingEdge() + +> **new CanUpdateOutgoingEdge**\<`TField`\>(`field`, `toEntClass`): [`CanUpdateOutgoingEdge`](CanUpdateOutgoingEdge.md)\<`TField`\> + +Defined in: [src/ent/predicates/CanUpdateOutgoingEdge.ts:15](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanUpdateOutgoingEdge.ts#L15) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `field` | `TField` | +| `toEntClass` | [`EntClass`](../interfaces/EntClass.md) | + +#### Returns + +[`CanUpdateOutgoingEdge`](CanUpdateOutgoingEdge.md)\<`TField`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | +| `field` | `TField` | +| `toEntClass` | [`EntClass`](../interfaces/EntClass.md) | + +## Methods + +### check() + +> **check**(`vc`, `row`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/CanUpdateOutgoingEdge.ts:22](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/CanUpdateOutgoingEdge.ts#L22) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | `Record`\<`TField`, `null` \| `string`\> | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/Client.md b/docs/classes/Client.md new file mode 100644 index 0000000..3ee8048 --- /dev/null +++ b/docs/classes/Client.md @@ -0,0 +1,273 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Client + +# Class: `abstract` Client + +Defined in: [src/abstract/Client.ts:67](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L67) + +Client is a Shard name aware abstraction which sends an actual query and +tracks the master/replica timeline. The concrete query sending implementation +(including required arguments) is up to the derived classes. + +## Extended by + +- [`PgClient`](PgClient.md) + +## Constructors + +### new Client() + +> **new Client**(`options`): [`Client`](Client.md) + +Defined in: [src/abstract/Client.ts:158](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L158) + +Initializes an instance of Client. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`ClientOptions`](../interfaces/ClientOptions.md) | + +#### Returns + +[`Client`](Client.md) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`ClientOptions`](../interfaces/ClientOptions.md)\>\> | Default values for the constructor options. | +| `options` | `Required`\<[`ClientOptions`](../interfaces/ClientOptions.md)\> | Client configuration options. | +| `createdAt` | `Date` | Date when this Client instance was constructed. | +| `shardName` | `string` | Each Client may be bound to some Shard, so the queries executed via it will be namespaced to this Shard. E.g. in relational databases, Shard name may be a namespace (or schema) name (or "public" if the Client wasn't created by withShard() method). | +| `timelineManager` | [`TimelineManager`](TimelineManager.md) | Tracks the master/replica replication timeline position. Shared across all the Clients within the same Island. | + +## Methods + +### address() + +> `abstract` **address**(): `string` + +Defined in: [src/abstract/Client.ts:98](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L98) + +Represents the full destination address this Client is working with. +Depending on the implementation, it may include hostname, port number, +database name, shard name etc. It is required that the address is stable +enough to be able to cache some destination database related metadata (e.g. +shardNos) based on that address. + +#### Returns + +`string` + +*** + +### end() + +> `abstract` **end**(): `Promise`\<`void`\> + +Defined in: [src/abstract/Client.ts:105](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L105) + +Gracefully closes the connections to let the caller destroy the Client. The +pending queries are awaited to finish before returning. The Client becomes +unusable after calling this method: you should not send queries to it. + +#### Returns + +`Promise`\<`void`\> + +*** + +### isEnded() + +> `abstract` **isEnded**(): `boolean` + +Defined in: [src/abstract/Client.ts:110](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L110) + +Returns true if the Client is ended and can't be used anymore. + +#### Returns + +`boolean` + +*** + +### shardNos() + +> `abstract` **shardNos**(): `Promise`\ + +Defined in: [src/abstract/Client.ts:116](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L116) + +Returns all Shard numbers discoverable via the connection to the Client's +database. + +#### Returns + +`Promise`\ + +*** + +### ping() + +> `abstract` **ping**(`input`): `Promise`\<`void`\> + +Defined in: [src/abstract/Client.ts:122](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L122) + +Sends a read or write test query to the server. Tells the server to sit and +wait for at least the provided number of milliseconds. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`ClientPingInput`](../interfaces/ClientPingInput.md) | + +#### Returns + +`Promise`\<`void`\> + +*** + +### withShard() + +> `abstract` **withShard**(`no`): `this` + +Defined in: [src/abstract/Client.ts:128](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L128) + +Creates a new Client which is namespaced to the provided Shard number. The +new Client will share the same connection pool with the parent's Client. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `no` | `number` | + +#### Returns + +`this` + +*** + +### role() + +> `abstract` **role**(): [`ClientRole`](../type-aliases/ClientRole.md) + +Defined in: [src/abstract/Client.ts:135](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L135) + +Returns the Client's role reported after the last successful query. Master +and replica roles may switch online unpredictably, without reconnecting, so +we only know the role after a query. + +#### Returns + +[`ClientRole`](../type-aliases/ClientRole.md) + +*** + +### connectionIssue() + +> `abstract` **connectionIssue**(): `null` \| [`ClientConnectionIssue`](../interfaces/ClientConnectionIssue.md) + +Defined in: [src/abstract/Client.ts:143](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L143) + +Returns a non-nullable value if the Client couldn't connect to the server +(or it could, but the load balancer reported the remote server as not +working), so it should ideally be removed from the list of active replicas +until e.g. the next discovery query to it (or any query) succeeds. + +#### Returns + +`null` \| [`ClientConnectionIssue`](../interfaces/ClientConnectionIssue.md) + +*** + +### logSwallowedError() + +> `protected` **logSwallowedError**(`props`): `void` + +Defined in: [src/abstract/Client.ts:148](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L148) + +Calls swallowedErrorLogger() doing some preliminary amendment. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `props` | [`SwallowedErrorLoggerProps`](../interfaces/SwallowedErrorLoggerProps.md) | + +#### Returns + +`void` + +*** + +### batcher() + +> **batcher**\<`TInput`, `TOutput`, `TTable`\>(`_QueryClass`, `_schema`, `_additionalShape`, `disableBatching`, `runnerCreator`): [`Batcher`](Batcher.md)\<`TInput`, `TOutput`\> + +Defined in: [src/abstract/Client.ts:189](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L189) + +Batcher is per-Client per-query-type +per-table-name-and-shape-and-disableBatching: + +- Per-Client means that batchers are removed as soon as the Client is + removed, i.e. the Client owns all the batchers for all tables. +- Per-query-type means that the batcher for a SELECT query is different + from the batcher for an INSERT query (obviously). +- Per-table-name-and-shape-and-disableBatching means that each table has + its own set of batchers (obviously). Also, some queries may be complex + (like UPDATE), so the batcher also depends on the "shape" - the list of + fields we're updating. Plus, for some inputs, we want to disable batching + at all - that produces a separate Batcher instance. + +Also, for every Batcher, there is exactly one Runner (which knows how to +build the actual query in the context of the current Client). Batchers are +generic (like DataLoader, but more general), and Runners are very custom to +the query (and are private to these queries). + +All that means that in a 1000-Shard 20-table Cluster we'll eventually have +1000x20x8 Batchers/Runners (assuming we have 8 different operations). + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TInput` | +| `TOutput` | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `_QueryClass` | `Function` | +| `_schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `_additionalShape` | `string` | +| `disableBatching` | `boolean` | +| `runnerCreator` | () => [`Runner`](Runner.md)\<`TInput`, `TOutput`\> | + +#### Returns + +[`Batcher`](Batcher.md)\<`TInput`, `TOutput`\> + +*** + +### prewarm() + +> **prewarm**(): `void` + +Defined in: [src/abstract/Client.ts:214](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L214) + +A convenience method to put connections prewarming logic to. The idea is to +keep the needed number of open connections and also, in each connection, +minimize the time which the very 1st query will take (e.g. pre-cache +full-text dictionaries). + +#### Returns + +`void` diff --git a/docs/classes/ClientError.md b/docs/classes/ClientError.md new file mode 100644 index 0000000..a4f48f4 --- /dev/null +++ b/docs/classes/ClientError.md @@ -0,0 +1,57 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientError + +# Class: ClientError + +Defined in: [src/abstract/ClientError.ts:43](https://github.com/clickup/ent-framework/blob/master/src/abstract/ClientError.ts#L43) + +Encapsulates the error thrown when running a Client query. The object also +carries suggestions, what to do next. + +## Extends + +- `Error` + +## Extended by + +- [`PgError`](PgError.md) + +## Constructors + +### new ClientError() + +> **new ClientError**(`cause`, `where`, `postAction`, `kind`, `abbreviation`, `comment`?): [`ClientError`](ClientError.md) + +Defined in: [src/abstract/ClientError.ts:44](https://github.com/clickup/ent-framework/blob/master/src/abstract/ClientError.ts#L44) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `cause` | `MaybeError` | +| `where` | `string` | +| `postAction` | [`ClientErrorPostAction`](../type-aliases/ClientErrorPostAction.md) | +| `kind` | [`ClientErrorKind`](../type-aliases/ClientErrorKind.md) | +| `abbreviation` | `string` | +| `comment`? | `string` | + +#### Returns + +[`ClientError`](ClientError.md) + +#### Overrides + +`Error.constructor` + +## Properties + +| Property | Type | +| ------ | ------ | +| `cause` | `MaybeError` | +| `postAction` | [`ClientErrorPostAction`](../type-aliases/ClientErrorPostAction.md) | +| `kind` | [`ClientErrorKind`](../type-aliases/ClientErrorKind.md) | +| `abbreviation` | `string` | +| `comment?` | `string` | diff --git a/docs/classes/Cluster.md b/docs/classes/Cluster.md new file mode 100644 index 0000000..7437319 --- /dev/null +++ b/docs/classes/Cluster.md @@ -0,0 +1,246 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Cluster + +# Class: Cluster\ + +Defined in: [src/abstract/Cluster.ts:129](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L129) + +Cluster is a collection of Islands and an orchestration of shardNo -> Island +resolution. + +It's unknown beforehand, which Island some particular Shard belongs to; the +resolution is done asynchronously and lazily. + +Shard 0 is a special "global" Shard. + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TClient` *extends* [`Client`](Client.md) | - | +| `TNode` | `DesperateAny` | + +## Constructors + +### new Cluster() + +> **new Cluster**\<`TClient`, `TNode`\>(`options`): [`Cluster`](Cluster.md)\<`TClient`, `TNode`\> + +Defined in: [src/abstract/Cluster.ts:172](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L172) + +Initializes the Cluster, but doesn't send any queries yet, even discovery +queries (also, no implicit prewarming). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`ClusterOptions`](../interfaces/ClusterOptions.md)\<`TClient`, `TNode`\> | + +#### Returns + +[`Cluster`](Cluster.md)\<`TClient`, `TNode`\> + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`ClusterOptions`](../interfaces/ClusterOptions.md)\<[`Client`](Client.md), `never`\>\>\> | Default values for the constructor options. | +| `options` | `Required`\<[`ClusterOptions`](../interfaces/ClusterOptions.md)\<`TClient`, `TNode`\>\> | Cluster configuration options. | + +## Methods + +### prewarm() + +> **prewarm**(`randomizedDelayMs`, `onInitialPrewarm`?): `void` + +Defined in: [src/abstract/Cluster.ts:303](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L303) + +Signals the Cluster to keep the Clients pre-warmed, e.g. open. (It's up to +the particular Client's implementation, what does a "pre-warmed Client" +mean; typically, it's keeping some minimal number of pooled connections.) + +Except when `randomizedDelayMs` is passed as 0, the actual prewarm (and +Islands discovery) queries will run with a randomized delay between N/2 and +N ms. It is better to operate in such mode: if multiple Node processes +start simultaneously in the cluster, then the randomization helps to avoid +new connections burst (new connections establishment is expensive for e.g. +pgbouncer or when DB is accessed over SSL). + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `randomizedDelayMs` | `number` | `5000` | +| `onInitialPrewarm`? | (`delayMs`) => `void` | `undefined` | + +#### Returns + +`void` + +*** + +### globalShard() + +> **globalShard**(): [`Shard`](Shard.md)\<`TClient`\> + +Defined in: [src/abstract/Cluster.ts:334](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L334) + +Returns a global Shard of the Cluster. This method is made synchronous +intentionally, to defer the I/O and possible errors to the moment of the +actual query. + +#### Returns + +[`Shard`](Shard.md)\<`TClient`\> + +*** + +### nonGlobalShards() + +> **nonGlobalShards**(): `Promise`\[]\> + +Defined in: [src/abstract/Cluster.ts:341](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L341) + +Returns all currently known (discovered) non-global Shards in the Cluster. + +#### Returns + +`Promise`\[]\> + +*** + +### shard() + +> **shard**(`id`): [`Shard`](Shard.md)\<`TClient`\> + +Defined in: [src/abstract/Cluster.ts:361](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L361) + +Returns Shard of a particular id. This method is made synchronous +intentionally, to defer the I/O and possible errors to the moment of the +actual query. + +Why is it important? Because Shards may go up and down temporarily at +random moments of time. Imagine we made this method async and asserted that +the Shard is actually available at the moment when the method is called. +What would happen if the Shard object was stored somewhere as "successful" +by the caller, then the Island went down, and then a query is sent to the +Shard in, say, 20 seconds? We'd get an absolutely different exception, at +the moment of the query. We don't want this to happen: we want all of the +exceptions to be thrown with a consistent call stack (e.g. at the moment of +the query), no matter whether it was an immediate call or a deferred one. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | + +#### Returns + +[`Shard`](Shard.md)\<`TClient`\> + +*** + +### shardByNo() + +> **shardByNo**(`shardNo`): [`Shard`](Shard.md)\<`TClient`\> + +Defined in: [src/abstract/Cluster.ts:376](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L376) + +Returns a Shard if we know its number. The idea: for each Shard number +(even for non-discovered yet Shards), we keep the corresponding Shard +object in a Memoize cache, so Shards with the same number always resolve +into the same Shard object. Then, an actual Island locating process happens +when the caller wants to get a Client of that Shard (and it throws if such +Shard hasn't been discovered actually). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `shardNo` | `number` | + +#### Returns + +[`Shard`](Shard.md)\<`TClient`\> + +*** + +### randomShard() + +> **randomShard**(`seed`?): `Promise`\<[`Shard`](Shard.md)\<`TClient`\>\> + +Defined in: [src/abstract/Cluster.ts:384](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L384) + +Returns a random Shard among the ones which are currently known +(discovered) in the Cluster. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `seed`? | `object` | + +#### Returns + +`Promise`\<[`Shard`](Shard.md)\<`TClient`\>\> + +*** + +### island() + +> **island**(`islandNo`): `Promise`\<[`Island`](Island.md)\<`TClient`\>\> + +Defined in: [src/abstract/Cluster.ts:408](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L408) + +Returns an Island by its number. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `islandNo` | `number` | + +#### Returns + +`Promise`\<[`Island`](Island.md)\<`TClient`\>\> + +*** + +### islands() + +> **islands**(): `Promise`\<[`Island`](Island.md)\<`TClient`\>[]\> + +Defined in: [src/abstract/Cluster.ts:419](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L419) + +Returns all Islands in the Cluster. + +#### Returns + +`Promise`\<[`Island`](Island.md)\<`TClient`\>[]\> + +*** + +### rediscover() + +> **rediscover**(`what`?): `Promise`\<`void`\> + +Defined in: [src/abstract/Cluster.ts:429](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L429) + +Triggers shards rediscovery and finishes as soon as it's done. To be used +in unit tests mostly, because in real life, it's enough to just modify the +cluster configuration. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `what`? | `"islands"` \| `"shards"` | + +#### Returns + +`Promise`\<`void`\> diff --git a/docs/classes/Configuration.md b/docs/classes/Configuration.md new file mode 100644 index 0000000..f49d6ad --- /dev/null +++ b/docs/classes/Configuration.md @@ -0,0 +1,70 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Configuration + +# Class: Configuration\ + +Defined in: [src/ent/Configuration.ts:37](https://github.com/clickup/ent-framework/blob/master/src/ent/Configuration.ts#L37) + +Strongly typed configuration framework to force TS auto-infer privacy +callbacks arguments types (which are not Ents, but row-like inputs). + +Motivation: +1. We MUST resolve privacyXyz rules below lazily, at actual operation; + otherwise in case of cyclic Ent dependencies between EntA and EntB, one of + them will be magically undefined. +2. We can’t define these parameter as BaseEnt arguments: privacy rules may + refer the derived Ent itself and other Ents and thus produce cyclic + dependencies. TS doesn't allow to work with such cyclic dependencies + during the class is defining. +3. Configuration can’t be just returned from a virtual method, because in TS, + type inference in return values is poor: + https://github.com/Microsoft/TypeScript/issues/31273 + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new Configuration() + +> **new Configuration**\<`TTable`\>(`cfg`): [`Configuration`](Configuration.md)\<`TTable`\> + +Defined in: [src/ent/Configuration.ts:141](https://github.com/clickup/ent-framework/blob/master/src/ent/Configuration.ts#L141) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `cfg` | [`Configuration`](Configuration.md)\<`TTable`\> | + +#### Returns + +[`Configuration`](Configuration.md)\<`TTable`\> + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `shardAffinity` | [`ShardAffinity`](../type-aliases/ShardAffinity.md)\<[`FieldOfIDType`](../type-aliases/FieldOfIDType.md)\<`TTable`\>\> | Defines how to locate a Shard at Ent insert time. See ShardAffinity for more details. 1. GLOBAL_SHARD: places the Ent in the global Shard (0). 2. `[]`: places the Ent in a random Shard. The "randomness" of the "random Shard" is deterministic by the Ent's unique key at the moment of insertion (if it's defined; otherwise completely random). This helps two racy insert operations running concurrently to choose the same Shard for the Ent to be created in, so only one of them will win, instead of both winning and mistakenly creating the Ent duplicates. I.e. having the same value in unique key forces the engine to target the same "random" Shard. 3. `["field1", "field2", ...]`: places the Ent in the Shard that is pointed to by the value in field1 (if it's null, then field2 etc.). A special treatment is applied if a fieldN value in (3) points to the global Shard. In such a case, the Shard for the current Ent is chosen deterministic-randomly at insert time, as if [] is passed. This allows the Ent to refer other "owning" Ents of different types, some of which may be located in the global Shard. Keep in mind that, to locate such an Ent pointing to another Ent in the global Shard, an inverse for fieldN must be defined in most of the cases. | +| `inverses?` | `{ [k in string]?: { name: string; type: string } }` | Inverses allow cross-Shard foreign keys & cross-Shard selection. If a field points to an Ent in another Shard, and we're e.g. selecting by a value in this field, inverses allow to locate Shard(s) of the Ent. | +| `privacyTenantPrincipalField?` | [`InsertFieldsRequired`](../type-aliases/InsertFieldsRequired.md)\<`TTable`\> & `string` | If defined, forces all Ents of this class to have the value of that field equal to VC's principal at load time. This is a very 1st unavoidable check in the privacy rules chain, thus it's bullet-proof. | +| `privacyInferPrincipal` | `null` \| `string` \| (`vc`, `row`) => `null` \| `string` \| `Promise`\<`null` \| `string` \| [`Ent`](../interfaces/Ent.md)\<\{\}\>\> | An attempt to load this Ent using an omni VC will "lower" that VC to the principal returned. Omni VC is always lowered. 1. If an Ent is returned, the lowered principal will be Ent#vc.principal. It is a way to delegate principal inference to another Ent. 2. If a string is returned, then it's treated as a principal ID. 3. If a null is returned, then a guest principal will be used. 4. Returning an omni principal or VC will result in a run-time error. | +| `privacyLoad` | [`LoadRule`](../type-aliases/LoadRule.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>\>[] | Privacy rules checked on every row loaded from the DB. | +| `privacyInsert` | [`WriteRules`](../type-aliases/WriteRules.md)\<[`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>\> | Privacy rules checked before a row is inserted to the DB. - It the list is empty, then only omni VC can insert; it's typically a good option for Ents representing e.g. a user. - If no update/delete rules are defined, then privacyInsert rules are also run on update/delete by default. - Unless empty, the rules must include at least one Require() predicate, they can't entirely consist of AllowIf(). This is because for write rules (privacyInsert, privacyUpdate, privacyDelete) it's important to make sure that ALL rules permit the operation, not only one of them allows it; this is what Require() is exactly for. | +| `privacyUpdate?` | [`WriteRules`](../type-aliases/WriteRules.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>\> | Privacy rules checked before a row is updated in the DB. - If not defined, privacyInsert rules are used. - The rules must include at least one Require() predicate. | +| `privacyDelete?` | [`WriteRules`](../type-aliases/WriteRules.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>\> | Privacy rules checked before a row is deleted in the DB. - If not defined, privacyInsert rules are used. - The rules must include at least one Require() predicate. | +| `validators?` | [`AbstractIs`](../interfaces/AbstractIs.md)\<[`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>\>[] | Custom field values validators run before any insert/update. | +| `beforeInsert?` | [`InsertTrigger`](../type-aliases/InsertTrigger.md)\<`TTable`\>[] | Triggers run before every insert. | +| `beforeUpdate?` | ([`BeforeUpdateTrigger`](../type-aliases/BeforeUpdateTrigger.md)\<`TTable`\> \| \[[`DepsBuilder`](../type-aliases/DepsBuilder.md)\<`TTable`\>, [`BeforeUpdateTrigger`](../type-aliases/BeforeUpdateTrigger.md)\<`TTable`\>\])[] | Triggers run before every update. | +| `beforeDelete?` | [`DeleteTrigger`](../type-aliases/DeleteTrigger.md)\<`TTable`\>[] | Triggers run before every delete. | +| `beforeMutation?` | ([`BeforeMutationTrigger`](../type-aliases/BeforeMutationTrigger.md)\<`TTable`\> \| \[[`DepsBuilder`](../type-aliases/DepsBuilder.md)\<`TTable`\>, [`BeforeMutationTrigger`](../type-aliases/BeforeMutationTrigger.md)\<`TTable`\>\])[] | Triggers run before every insert/update/delete. Each trigger may also be passed as "React useEffect-like" tuple where the callback is executed only if the deps are modified. | +| `afterInsert?` | [`InsertTrigger`](../type-aliases/InsertTrigger.md)\<`TTable`\>[] | Triggers run after every delete. | +| `afterUpdate?` | ([`AfterUpdateTrigger`](../type-aliases/AfterUpdateTrigger.md)\<`TTable`\> \| \[[`DepsBuilder`](../type-aliases/DepsBuilder.md)\<`TTable`\>, [`AfterUpdateTrigger`](../type-aliases/AfterUpdateTrigger.md)\<`TTable`\>\])[] | Triggers run after every update. | +| `afterDelete?` | [`DeleteTrigger`](../type-aliases/DeleteTrigger.md)\<`TTable`\>[] | Triggers run after every delete. | +| `afterMutation?` | ([`AfterMutationTrigger`](../type-aliases/AfterMutationTrigger.md)\<`TTable`\> \| \[[`DepsBuilder`](../type-aliases/DepsBuilder.md)\<`TTable`\>, [`AfterMutationTrigger`](../type-aliases/AfterMutationTrigger.md)\<`TTable`\>\])[] | Triggers run after every insert/update/delete. Each trigger may also be passed as "React useEffect-like" tuple where the callback is executed only if the deps are modified. | diff --git a/docs/classes/DenyIf.md b/docs/classes/DenyIf.md new file mode 100644 index 0000000..fb34919 --- /dev/null +++ b/docs/classes/DenyIf.md @@ -0,0 +1,79 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / DenyIf + +# Class: DenyIf\ + +Defined in: [src/ent/rules/DenyIf.ts:14](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/DenyIf.ts#L14) + +Returns DENY if the predicate succeeds, otherwise SKIP. +- Used mostly to early block some read/write access. +- EntAccessError exception will be treated as a DENY signal (so it will abort + processing immediately). +- This rule may still throw an exception if the exception is a wild one (not + derived from EntAccessError). + +## Extends + +- [`Rule`](Rule.md)\<`TInput`\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` *extends* `object` | + +## Constructors + +### new DenyIf() + +> **new DenyIf**\<`TInput`\>(`predicate`): [`DenyIf`](DenyIf.md)\<`TInput`\> + +Defined in: [src/ent/rules/Rule.ts:43](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Rule.ts#L43) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `predicate` | [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> \| (`vc`, `input`) => `boolean` \| `Promise`\<`boolean`\> | + +#### Returns + +[`DenyIf`](DenyIf.md)\<`TInput`\> + +#### Inherited from + +[`Rule`](Rule.md).[`constructor`](Rule.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `_TAG` | `"DenyIf"` | +| `predicate` | [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> | +| `name` | `string` | + +## Methods + +### evaluate() + +> **evaluate**(`vc`, `input`): `Promise`\<[`RuleResult`](../interfaces/RuleResult.md)\> + +Defined in: [src/ent/rules/DenyIf.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/DenyIf.ts#L17) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `input` | `TInput` | + +#### Returns + +`Promise`\<[`RuleResult`](../interfaces/RuleResult.md)\> + +#### Overrides + +[`Rule`](Rule.md).[`evaluate`](Rule.md#evaluate) diff --git a/docs/classes/EntAccessError.md b/docs/classes/EntAccessError.md new file mode 100644 index 0000000..6883ab3 --- /dev/null +++ b/docs/classes/EntAccessError.md @@ -0,0 +1,71 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntAccessError + +# Class: EntAccessError + +Defined in: [src/ent/errors/EntAccessError.ts:22](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntAccessError.ts#L22) + +A base class for errors that trigger the validation framework to process them +as a DENY/SKIP. Invariants in derived classes: the error message should be +safe to pass to the client (it must not have any private information; a good +example is EntValidationError), plus the message alone should be descriptive +enough to extract information from it. If `cause` is passed, it becomes a +part of the message, with the above assumptions. + +## Extends + +- `Error` + +## Extended by + +- [`EntNotFoundError`](EntNotFoundError.md) +- [`EntNotInsertableError`](EntNotInsertableError.md) +- [`EntNotReadableError`](EntNotReadableError.md) +- [`EntNotUpdatableError`](EntNotUpdatableError.md) +- [`EntValidationError`](EntValidationError.md) + +## Constructors + +### new EntAccessError() + +> **new EntAccessError**(`entName`, `message`, `cause`): [`EntAccessError`](EntAccessError.md) + +Defined in: [src/ent/errors/EntAccessError.ts:25](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntAccessError.ts#L25) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `entName` | `string` | `undefined` | +| `message` | `string` | `undefined` | +| `cause` | `unknown` | `null` | + +#### Returns + +[`EntAccessError`](EntAccessError.md) + +#### Overrides + +`Error.constructor` + +## Properties + +| Property | Type | +| ------ | ------ | +| `cause` | `null` \| `string` \| `Error` | +| `entName` | `string` | + +## Methods + +### toStandardSchemaV1() + +> **toStandardSchemaV1**(): [`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +Defined in: [src/ent/errors/EntAccessError.ts:52](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntAccessError.ts#L52) + +#### Returns + +[`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) diff --git a/docs/classes/EntNotFoundError.md b/docs/classes/EntNotFoundError.md new file mode 100644 index 0000000..0ad01b8 --- /dev/null +++ b/docs/classes/EntNotFoundError.md @@ -0,0 +1,65 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntNotFoundError + +# Class: EntNotFoundError + +Defined in: [src/ent/errors/EntNotFoundError.ts:9](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntNotFoundError.ts#L9) + +Error: non-existing ID in the database (failed loadX() call), or non-existing +Ent (failed loadByX() call). Notice that `where` data is intentionally NOT +considered as private and may be delivered to the client. + +## Extends + +- [`EntAccessError`](EntAccessError.md) + +## Constructors + +### new EntNotFoundError() + +> **new EntNotFoundError**(`entName`, `where`, `cause`): [`EntNotFoundError`](EntNotFoundError.md) + +Defined in: [src/ent/errors/EntNotFoundError.ts:10](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntNotFoundError.ts#L10) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `entName` | `string` | `undefined` | +| `where` | `Record`\<`string`, `unknown`\> | `undefined` | +| `cause` | `unknown` | `null` | + +#### Returns + +[`EntNotFoundError`](EntNotFoundError.md) + +#### Overrides + +[`EntAccessError`](EntAccessError.md).[`constructor`](EntAccessError.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `cause` | `null` \| `string` \| `Error` | +| `entName` | `string` | +| `where` | `Record`\<`string`, `unknown`\> | + +## Methods + +### toStandardSchemaV1() + +> **toStandardSchemaV1**(): [`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +Defined in: [src/ent/errors/EntAccessError.ts:52](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntAccessError.ts#L52) + +#### Returns + +[`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +#### Inherited from + +[`EntAccessError`](EntAccessError.md).[`toStandardSchemaV1`](EntAccessError.md#tostandardschemav1) diff --git a/docs/classes/EntNotInsertableError.md b/docs/classes/EntNotInsertableError.md new file mode 100644 index 0000000..d2ef432 --- /dev/null +++ b/docs/classes/EntNotInsertableError.md @@ -0,0 +1,65 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntNotInsertableError + +# Class: EntNotInsertableError + +Defined in: [src/ent/errors/EntNotInsertableError.ts:6](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntNotInsertableError.ts#L6) + +Error: thrown when an Ent cannot be inserted due to privacy reasons. + +## Extends + +- [`EntAccessError`](EntAccessError.md) + +## Constructors + +### new EntNotInsertableError() + +> **new EntNotInsertableError**(`entName`, `vc`, `row`, `cause`): [`EntNotInsertableError`](EntNotInsertableError.md) + +Defined in: [src/ent/errors/EntNotInsertableError.ts:7](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntNotInsertableError.ts#L7) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `entName` | `string` | `undefined` | +| `vc` | `string` | `undefined` | +| `row` | `object` | `undefined` | +| `cause` | `unknown` | `null` | + +#### Returns + +[`EntNotInsertableError`](EntNotInsertableError.md) + +#### Overrides + +[`EntAccessError`](EntAccessError.md).[`constructor`](EntAccessError.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `cause` | `null` \| `string` \| `Error` | +| `entName` | `string` | +| `vc` | `string` | +| `row` | `object` | + +## Methods + +### toStandardSchemaV1() + +> **toStandardSchemaV1**(): [`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +Defined in: [src/ent/errors/EntAccessError.ts:52](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntAccessError.ts#L52) + +#### Returns + +[`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +#### Inherited from + +[`EntAccessError`](EntAccessError.md).[`toStandardSchemaV1`](EntAccessError.md#tostandardschemav1) diff --git a/docs/classes/EntNotReadableError.md b/docs/classes/EntNotReadableError.md new file mode 100644 index 0000000..43c6e21 --- /dev/null +++ b/docs/classes/EntNotReadableError.md @@ -0,0 +1,65 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntNotReadableError + +# Class: EntNotReadableError + +Defined in: [src/ent/errors/EntNotReadableError.ts:8](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntNotReadableError.ts#L8) + +Error: thrown when an Ent cannot be read due to privacy reasons. + +## Extends + +- [`EntAccessError`](EntAccessError.md) + +## Constructors + +### new EntNotReadableError() + +> **new EntNotReadableError**(`entName`, `vc`, `row`, `cause`): [`EntNotReadableError`](EntNotReadableError.md) + +Defined in: [src/ent/errors/EntNotReadableError.ts:9](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntNotReadableError.ts#L9) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `entName` | `string` | `undefined` | +| `vc` | `string` | `undefined` | +| `row` | [`RowWithID`](../type-aliases/RowWithID.md) | `undefined` | +| `cause` | `unknown` | `null` | + +#### Returns + +[`EntNotReadableError`](EntNotReadableError.md) + +#### Overrides + +[`EntAccessError`](EntAccessError.md).[`constructor`](EntAccessError.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `cause` | `null` \| `string` \| `Error` | +| `entName` | `string` | +| `vc` | `string` | +| `row` | [`RowWithID`](../type-aliases/RowWithID.md) | + +## Methods + +### toStandardSchemaV1() + +> **toStandardSchemaV1**(): [`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +Defined in: [src/ent/errors/EntAccessError.ts:52](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntAccessError.ts#L52) + +#### Returns + +[`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +#### Inherited from + +[`EntAccessError`](EntAccessError.md).[`toStandardSchemaV1`](EntAccessError.md#tostandardschemav1) diff --git a/docs/classes/EntNotUpdatableError.md b/docs/classes/EntNotUpdatableError.md new file mode 100644 index 0000000..4a69659 --- /dev/null +++ b/docs/classes/EntNotUpdatableError.md @@ -0,0 +1,65 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntNotUpdatableError + +# Class: EntNotUpdatableError + +Defined in: [src/ent/errors/EntNotUpdatableError.ts:8](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntNotUpdatableError.ts#L8) + +Error: thrown when an Ent cannot be updated or deleted due to privacy reasons. + +## Extends + +- [`EntAccessError`](EntAccessError.md) + +## Constructors + +### new EntNotUpdatableError() + +> **new EntNotUpdatableError**(`entName`, `vc`, `row`, `cause`): [`EntNotUpdatableError`](EntNotUpdatableError.md) + +Defined in: [src/ent/errors/EntNotUpdatableError.ts:9](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntNotUpdatableError.ts#L9) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `entName` | `string` | `undefined` | +| `vc` | `string` | `undefined` | +| `row` | [`RowWithID`](../type-aliases/RowWithID.md) | `undefined` | +| `cause` | `unknown` | `null` | + +#### Returns + +[`EntNotUpdatableError`](EntNotUpdatableError.md) + +#### Overrides + +[`EntAccessError`](EntAccessError.md).[`constructor`](EntAccessError.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `cause` | `null` \| `string` \| `Error` | +| `entName` | `string` | +| `vc` | `string` | +| `row` | [`RowWithID`](../type-aliases/RowWithID.md) | + +## Methods + +### toStandardSchemaV1() + +> **toStandardSchemaV1**(): [`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +Defined in: [src/ent/errors/EntAccessError.ts:52](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntAccessError.ts#L52) + +#### Returns + +[`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +#### Inherited from + +[`EntAccessError`](EntAccessError.md).[`toStandardSchemaV1`](EntAccessError.md#tostandardschemav1) diff --git a/docs/classes/EntUniqueKeyError.md b/docs/classes/EntUniqueKeyError.md new file mode 100644 index 0000000..dc4e346 --- /dev/null +++ b/docs/classes/EntUniqueKeyError.md @@ -0,0 +1,73 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntUniqueKeyError + +# Class: EntUniqueKeyError + +Defined in: [src/ent/errors/EntUniqueKeyError.ts:7](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntUniqueKeyError.ts#L7) + +Error: while inserting or updating, DB unique key was violated, +so the Ent was not mutated. + +## Extends + +- `Error` + +## Constructors + +### new EntUniqueKeyError() + +> **new EntUniqueKeyError**(`entName`, `input`): [`EntUniqueKeyError`](EntUniqueKeyError.md) + +Defined in: [src/ent/errors/EntUniqueKeyError.ts:8](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntUniqueKeyError.ts#L8) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `entName` | `string` | +| `input` | `unknown` | + +#### Returns + +[`EntUniqueKeyError`](EntUniqueKeyError.md) + +#### Overrides + +`Error.constructor` + +## Properties + +| Property | Type | +| ------ | ------ | +| `entName` | `string` | +| `input` | `unknown` | + +## Methods + +### ignore() + +> `static` **ignore**\<`T`\>(`promise`): `Promise`\<`undefined` \| `T`\> + +Defined in: [src/ent/errors/EntUniqueKeyError.ts:28](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntUniqueKeyError.ts#L28) + +Returns a promise of T on success, and undefined in case unique key +violation happened during the promise resolution. + +#### Type Parameters + +| Type Parameter | +| ------ | +| `T` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `promise` | `Promise`\<`T`\> | + +#### Returns + +`Promise`\<`undefined` \| `T`\> diff --git a/docs/classes/EntValidationError.md b/docs/classes/EntValidationError.md new file mode 100644 index 0000000..5d10c8c --- /dev/null +++ b/docs/classes/EntValidationError.md @@ -0,0 +1,66 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntValidationError + +# Class: EntValidationError + +Defined in: [src/ent/errors/EntValidationError.ts:8](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntValidationError.ts#L8) + +Error: thrown after all validators are executed, and some of them think that +the row is invalid. + +## Extends + +- [`EntAccessError`](EntAccessError.md) + +## Constructors + +### new EntValidationError() + +> **new EntValidationError**(`entName`, `errors`): [`EntValidationError`](EntValidationError.md) + +Defined in: [src/ent/errors/EntValidationError.ts:9](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntValidationError.ts#L9) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `entName` | `string` | +| `errors` | readonly [`EntValidationErrorInfo`](../interfaces/EntValidationErrorInfo.md)[] | + +#### Returns + +[`EntValidationError`](EntValidationError.md) + +#### Overrides + +[`EntAccessError`](EntAccessError.md).[`constructor`](EntAccessError.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `cause` | `null` \| `string` \| `Error` | +| `entName` | `string` | +| `errors` | readonly [`EntValidationErrorInfo`](../interfaces/EntValidationErrorInfo.md)[] | + +## Methods + +### toStandardSchemaV1() + +> **toStandardSchemaV1**(): [`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +Defined in: [src/ent/errors/EntValidationError.ts:27](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntValidationError.ts#L27) + +Converts the payload to a Standard Schema V1 compatible error result. See +https://standardschema.dev. + +#### Returns + +[`StandardSchemaV1FailureResult`](../interfaces/StandardSchemaV1FailureResult.md) + +#### Overrides + +[`EntAccessError`](EntAccessError.md).[`toStandardSchemaV1`](EntAccessError.md#tostandardschemav1) diff --git a/docs/classes/FieldIs.md b/docs/classes/FieldIs.md new file mode 100644 index 0000000..3c80e4d --- /dev/null +++ b/docs/classes/FieldIs.md @@ -0,0 +1,101 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FieldIs + +# Class: FieldIs\ + +Defined in: [src/ent/predicates/FieldIs.ts:43](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/FieldIs.ts#L43) + +Checks that the validator function returns true for the value in some field. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TField` *extends* `string` | +| `TRow` *extends* `Partial`\<`Record`\<`TField`, `unknown`\>\> | + +## Implements + +- [`AbstractIs`](../interfaces/AbstractIs.md)\<`TRow`\> + +## Constructors + +### new FieldIs() + +> **new FieldIs**\<`TField`, `TRow`\>(`field`, `validator`, `message`): [`FieldIs`](FieldIs.md)\<`TField`, `TRow`\> + +Defined in: [src/ent/predicates/FieldIs.ts:55](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/FieldIs.ts#L55) + +Manual validator. Implies that we can trust the fieldValue TS type. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `field` | `TField` | +| `validator` | [`FieldIsValidatorPlain`](../type-aliases/FieldIsValidatorPlain.md)\<`TField`, `TRow`\> | +| `message` | `string` | + +#### Returns + +[`FieldIs`](FieldIs.md)\<`TField`, `TRow`\> + +### new FieldIs() + +> **new FieldIs**\<`TField`, `TRow`\>(`field`, `validator`): [`FieldIs`](FieldIs.md)\<`TField`, `TRow`\> + +Defined in: [src/ent/predicates/FieldIs.ts:65](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/FieldIs.ts#L65) + +Rich validator, like Standard Schema (https://standardschema.dev) or Zod. +No implications are made on the fieldValue type. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `field` | `TField` | +| `validator` | [`FieldIsValidatorZodSafeParse`](../type-aliases/FieldIsValidatorZodSafeParse.md)\<`TRow`\> \| [`FieldIsValidatorStandardSchemaV1`](../type-aliases/FieldIsValidatorStandardSchemaV1.md)\<`TRow`\> | + +#### Returns + +[`FieldIs`](FieldIs.md)\<`TField`, `TRow`\> + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `name` | `string` | - | +| `field` | `TField` | The field this validation predicate is related to (null means that it applies to the entire Ent). | +| `message` | `null` \| `string` | In case the predicate returns false or doesn't provide error messages by throwing EntValidationError, this message will be used. When message is null, it means that we expect the validator to return detailed information about each field errored (e.g. ValidatorStandardSchemaResult). | +| `validator` | [`FieldIsValidatorPlain`](../type-aliases/FieldIsValidatorPlain.md)\<`TField`, `TRow`\> \| [`FieldIsValidatorZodSafeParse`](../type-aliases/FieldIsValidatorZodSafeParse.md)\<`TRow`\> \| [`FieldIsValidatorStandardSchemaV1`](../type-aliases/FieldIsValidatorStandardSchemaV1.md)\<`TRow`\> | - | + +## Methods + +### check() + +> **check**(`vc`, `row`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/FieldIs.ts:92](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/FieldIs.ts#L92) + +Returns true if validation succeeds. Returns false if it wants the client +to use this.message as a validation failure response. Throws an instance of +EntValidationError when it needs to deliver the detailed error messages +about multiple fields. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | `TRow` | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`AbstractIs`](../interfaces/AbstractIs.md).[`check`](../interfaces/AbstractIs.md#check) diff --git a/docs/classes/FuncToPredicate.md b/docs/classes/FuncToPredicate.md new file mode 100644 index 0000000..43d7685 --- /dev/null +++ b/docs/classes/FuncToPredicate.md @@ -0,0 +1,72 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FuncToPredicate + +# Class: FuncToPredicate\ + +Defined in: [src/ent/predicates/Predicate.ts:29](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L29) + +Sometimes, instead of passing a well-known predicate like OutgoingEdgePointsToVC +or CanUpdateOutgoingEdge, we want to pass just a function which accepts a row +and returns true or false. This class represents a Predicate which delegates +its work to such a function. The name of the function becomes the name of the +predicate. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` | + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> + +## Constructors + +### new FuncToPredicate() + +> **new FuncToPredicate**\<`TInput`\>(`func`): [`FuncToPredicate`](FuncToPredicate.md)\<`TInput`\> + +Defined in: [src/ent/predicates/Predicate.ts:32](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L32) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `func` | (`vc`, `input`) => `boolean` \| `Promise`\<`boolean`\> | + +#### Returns + +[`FuncToPredicate`](FuncToPredicate.md)\<`TInput`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | + +## Methods + +### check() + +> **check**(`vc`, `input`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/Predicate.ts:38](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L38) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `input` | `TInput` | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/IDsCache.md b/docs/classes/IDsCache.md new file mode 100644 index 0000000..ef2ff8e --- /dev/null +++ b/docs/classes/IDsCache.md @@ -0,0 +1,84 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / IDsCache + +# Class: `abstract` IDsCache + +Defined in: [src/ent/IDsCache.ts:10](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L10) + +## Extended by + +- [`IDsCacheReadable`](IDsCacheReadable.md) +- [`IDsCacheUpdatable`](IDsCacheUpdatable.md) +- [`IDsCacheDeletable`](IDsCacheDeletable.md) +- [`IDsCacheCanReadIncomingEdge`](IDsCacheCanReadIncomingEdge.md) + +## Constructors + +### new IDsCache() + +> **new IDsCache**(): [`IDsCache`](IDsCache.md) + +#### Returns + +[`IDsCache`](IDsCache.md) + +## Methods + +### has() + +> **has**(`Ent`, `id`): `boolean` + +Defined in: [src/ent/IDsCache.ts:13](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L13) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`boolean` + +*** + +### add() + +> **add**(`Ent`, `id`, `value`): `void` + +Defined in: [src/ent/IDsCache.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L17) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `Ent` | `EntClassAlike` | `undefined` | +| `id` | `string` | `undefined` | +| `value` | `boolean` | `true` | + +#### Returns + +`void` + +*** + +### get() + +> **get**(`Ent`, `id`): `undefined` \| `boolean` + +Defined in: [src/ent/IDsCache.ts:21](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L21) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`undefined` \| `boolean` diff --git a/docs/classes/IDsCacheCanReadIncomingEdge.md b/docs/classes/IDsCacheCanReadIncomingEdge.md new file mode 100644 index 0000000..28da809 --- /dev/null +++ b/docs/classes/IDsCacheCanReadIncomingEdge.md @@ -0,0 +1,97 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / IDsCacheCanReadIncomingEdge + +# Class: IDsCacheCanReadIncomingEdge + +Defined in: [src/ent/predicates/Predicate.ts:49](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L49) + +## Extends + +- [`IDsCache`](IDsCache.md) + +## Constructors + +### new IDsCacheCanReadIncomingEdge() + +> **new IDsCacheCanReadIncomingEdge**(): [`IDsCacheCanReadIncomingEdge`](IDsCacheCanReadIncomingEdge.md) + +#### Returns + +[`IDsCacheCanReadIncomingEdge`](IDsCacheCanReadIncomingEdge.md) + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`constructor`](IDsCache.md#constructors) + +## Methods + +### has() + +> **has**(`Ent`, `id`): `boolean` + +Defined in: [src/ent/IDsCache.ts:13](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L13) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`boolean` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`has`](IDsCache.md#has) + +*** + +### add() + +> **add**(`Ent`, `id`, `value`): `void` + +Defined in: [src/ent/IDsCache.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L17) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `Ent` | `EntClassAlike` | `undefined` | +| `id` | `string` | `undefined` | +| `value` | `boolean` | `true` | + +#### Returns + +`void` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`add`](IDsCache.md#add) + +*** + +### get() + +> **get**(`Ent`, `id`): `undefined` \| `boolean` + +Defined in: [src/ent/IDsCache.ts:21](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L21) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`undefined` \| `boolean` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`get`](IDsCache.md#get) diff --git a/docs/classes/IDsCacheDeletable.md b/docs/classes/IDsCacheDeletable.md new file mode 100644 index 0000000..f533ef3 --- /dev/null +++ b/docs/classes/IDsCacheDeletable.md @@ -0,0 +1,97 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / IDsCacheDeletable + +# Class: IDsCacheDeletable + +Defined in: [src/ent/predicates/Predicate.ts:48](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L48) + +## Extends + +- [`IDsCache`](IDsCache.md) + +## Constructors + +### new IDsCacheDeletable() + +> **new IDsCacheDeletable**(): [`IDsCacheDeletable`](IDsCacheDeletable.md) + +#### Returns + +[`IDsCacheDeletable`](IDsCacheDeletable.md) + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`constructor`](IDsCache.md#constructors) + +## Methods + +### has() + +> **has**(`Ent`, `id`): `boolean` + +Defined in: [src/ent/IDsCache.ts:13](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L13) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`boolean` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`has`](IDsCache.md#has) + +*** + +### add() + +> **add**(`Ent`, `id`, `value`): `void` + +Defined in: [src/ent/IDsCache.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L17) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `Ent` | `EntClassAlike` | `undefined` | +| `id` | `string` | `undefined` | +| `value` | `boolean` | `true` | + +#### Returns + +`void` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`add`](IDsCache.md#add) + +*** + +### get() + +> **get**(`Ent`, `id`): `undefined` \| `boolean` + +Defined in: [src/ent/IDsCache.ts:21](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L21) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`undefined` \| `boolean` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`get`](IDsCache.md#get) diff --git a/docs/classes/IDsCacheReadable.md b/docs/classes/IDsCacheReadable.md new file mode 100644 index 0000000..f0c4bde --- /dev/null +++ b/docs/classes/IDsCacheReadable.md @@ -0,0 +1,97 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / IDsCacheReadable + +# Class: IDsCacheReadable + +Defined in: [src/ent/predicates/Predicate.ts:46](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L46) + +## Extends + +- [`IDsCache`](IDsCache.md) + +## Constructors + +### new IDsCacheReadable() + +> **new IDsCacheReadable**(): [`IDsCacheReadable`](IDsCacheReadable.md) + +#### Returns + +[`IDsCacheReadable`](IDsCacheReadable.md) + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`constructor`](IDsCache.md#constructors) + +## Methods + +### has() + +> **has**(`Ent`, `id`): `boolean` + +Defined in: [src/ent/IDsCache.ts:13](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L13) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`boolean` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`has`](IDsCache.md#has) + +*** + +### add() + +> **add**(`Ent`, `id`, `value`): `void` + +Defined in: [src/ent/IDsCache.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L17) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `Ent` | `EntClassAlike` | `undefined` | +| `id` | `string` | `undefined` | +| `value` | `boolean` | `true` | + +#### Returns + +`void` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`add`](IDsCache.md#add) + +*** + +### get() + +> **get**(`Ent`, `id`): `undefined` \| `boolean` + +Defined in: [src/ent/IDsCache.ts:21](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L21) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`undefined` \| `boolean` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`get`](IDsCache.md#get) diff --git a/docs/classes/IDsCacheUpdatable.md b/docs/classes/IDsCacheUpdatable.md new file mode 100644 index 0000000..8e71c69 --- /dev/null +++ b/docs/classes/IDsCacheUpdatable.md @@ -0,0 +1,97 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / IDsCacheUpdatable + +# Class: IDsCacheUpdatable + +Defined in: [src/ent/predicates/Predicate.ts:47](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L47) + +## Extends + +- [`IDsCache`](IDsCache.md) + +## Constructors + +### new IDsCacheUpdatable() + +> **new IDsCacheUpdatable**(): [`IDsCacheUpdatable`](IDsCacheUpdatable.md) + +#### Returns + +[`IDsCacheUpdatable`](IDsCacheUpdatable.md) + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`constructor`](IDsCache.md#constructors) + +## Methods + +### has() + +> **has**(`Ent`, `id`): `boolean` + +Defined in: [src/ent/IDsCache.ts:13](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L13) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`boolean` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`has`](IDsCache.md#has) + +*** + +### add() + +> **add**(`Ent`, `id`, `value`): `void` + +Defined in: [src/ent/IDsCache.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L17) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `Ent` | `EntClassAlike` | `undefined` | +| `id` | `string` | `undefined` | +| `value` | `boolean` | `true` | + +#### Returns + +`void` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`add`](IDsCache.md#add) + +*** + +### get() + +> **get**(`Ent`, `id`): `undefined` \| `boolean` + +Defined in: [src/ent/IDsCache.ts:21](https://github.com/clickup/ent-framework/blob/master/src/ent/IDsCache.ts#L21) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Ent` | `EntClassAlike` | +| `id` | `string` | + +#### Returns + +`undefined` \| `boolean` + +#### Inherited from + +[`IDsCache`](IDsCache.md).[`get`](IDsCache.md#get) diff --git a/docs/classes/IncomingEdgeFromVCExists.md b/docs/classes/IncomingEdgeFromVCExists.md new file mode 100644 index 0000000..64b1fec --- /dev/null +++ b/docs/classes/IncomingEdgeFromVCExists.md @@ -0,0 +1,86 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / IncomingEdgeFromVCExists + +# Class: IncomingEdgeFromVCExists\ + +Defined in: [src/ent/predicates/IncomingEdgeFromVCExists.ts:23](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/IncomingEdgeFromVCExists.ts#L23) + +An ent may represent not necessarily a node in the graph, but also an edge +between two nodes. Consider EntMember in the below example: + +vc.principal <--- EntMember[user_id, company_id] ---> EntCompany + +This predicate verifies that for a e.g. given EntCompany row and a given VC, +an EntMember row exists (and optionally matches some criterion) in the +database. + +- entEdgeVCField = user_id in the above example +- entEdgeFKField = company_id in the above example +- if an EntMember object exists, it must also match entEdgeFilter() + +## Type Parameters + +| Type Parameter | +| ------ | +| `TEdgeTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<[`RowWithID`](../type-aliases/RowWithID.md)\> + +## Constructors + +### new IncomingEdgeFromVCExists() + +> **new IncomingEdgeFromVCExists**\<`TEdgeTable`\>(`EntEdge`, `entEdgeVCField`, `entEdgeFKField`, `entEdgeFilter`?): [`IncomingEdgeFromVCExists`](IncomingEdgeFromVCExists.md)\<`TEdgeTable`\> + +Defined in: [src/ent/predicates/IncomingEdgeFromVCExists.ts:29](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/IncomingEdgeFromVCExists.ts#L29) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `EntEdge` | [`EntClass`](../interfaces/EntClass.md)\<`TEdgeTable`\> | +| `entEdgeVCField` | `"id"` \| [`Field`](../type-aliases/Field.md)\<`TEdgeTable`\> | +| `entEdgeFKField` | `"id"` \| [`Field`](../type-aliases/Field.md)\<`TEdgeTable`\> | +| `entEdgeFilter`? | (`ent`) => `boolean` | + +#### Returns + +[`IncomingEdgeFromVCExists`](IncomingEdgeFromVCExists.md)\<`TEdgeTable`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | +| `EntEdge` | [`EntClass`](../interfaces/EntClass.md)\<`TEdgeTable`\> | +| `entEdgeVCField` | `"id"` \| [`Field`](../type-aliases/Field.md)\<`TEdgeTable`\> | +| `entEdgeFKField` | `"id"` \| [`Field`](../type-aliases/Field.md)\<`TEdgeTable`\> | +| `entEdgeFilter?` | (`ent`: [`Row`](../type-aliases/Row.md)\<`TEdgeTable`\>) => `boolean` | + +## Methods + +### check() + +> **check**(`vc`, `row`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/IncomingEdgeFromVCExists.ts:46](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/IncomingEdgeFromVCExists.ts#L46) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | [`RowWithID`](../type-aliases/RowWithID.md) | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/Inverse.md b/docs/classes/Inverse.md new file mode 100644 index 0000000..45236f6 --- /dev/null +++ b/docs/classes/Inverse.md @@ -0,0 +1,143 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Inverse + +# Class: Inverse\ + +Defined in: [src/ent/Inverse.ts:29](https://github.com/clickup/ent-framework/blob/master/src/ent/Inverse.ts#L29) + +Represents an Inverse assoc manager which knows how to modify/query Inverses. +Parameter `name` is the Inverse's schema name (in relational databases, most +likely a table name), and `type` holds both the name of the "parent" entity +and the field name of the child (e.g. "org2users" when a field "org_id" in +EntUser refers an EntOrg row). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TClient` *extends* [`Client`](Client.md) | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new Inverse() + +> **new Inverse**\<`TClient`, `TTable`\>(`__namedParameters`): [`Inverse`](Inverse.md)\<`TClient`, `TTable`\> + +Defined in: [src/ent/Inverse.ts:37](https://github.com/clickup/ent-framework/blob/master/src/ent/Inverse.ts#L37) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `__namedParameters` | \{ `cluster`: [`Cluster`](Cluster.md)\<`TClient`, `any`\>; `shardAffinity`: [`ShardAffinity`](../type-aliases/ShardAffinity.md)\<`string`\>; `id2Schema`: [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\>; `id2Field`: [`FieldOfIDTypeRequired`](../type-aliases/FieldOfIDTypeRequired.md)\<`TTable`\>; `name`: `string`; `type`: `string`; \} | +| `__namedParameters.cluster` | [`Cluster`](Cluster.md)\<`TClient`, `any`\> | +| `__namedParameters.shardAffinity` | [`ShardAffinity`](../type-aliases/ShardAffinity.md)\<`string`\> | +| `__namedParameters.id2Schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `__namedParameters.id2Field` | [`FieldOfIDTypeRequired`](../type-aliases/FieldOfIDTypeRequired.md)\<`TTable`\> | +| `__namedParameters.name` | `string` | +| `__namedParameters.type` | `string` | + +#### Returns + +[`Inverse`](Inverse.md)\<`TClient`, `TTable`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `id2Field` | [`FieldOfIDTypeRequired`](../type-aliases/FieldOfIDTypeRequired.md)\<`TTable`\> | +| `type` | `string` | + +## Methods + +### beforeInsert() + +> **beforeInsert**(`vc`, `id1`, `id2`): `Promise`\<`boolean`\> + +Defined in: [src/ent/Inverse.ts:64](https://github.com/clickup/ent-framework/blob/master/src/ent/Inverse.ts#L64) + +Runs before a row with a pre-generated id2 was inserted to the main schema. +Returns true if the Inverse row was actually inserted in the DB. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `id1` | `null` \| `string` | +| `id2` | `string` | + +#### Returns + +`Promise`\<`boolean`\> + +*** + +### afterUpdate() + +> **afterUpdate**(`vc`, `id1`, `id2`, `oldID1`): `Promise`\<`void`\> + +Defined in: [src/ent/Inverse.ts:88](https://github.com/clickup/ent-framework/blob/master/src/ent/Inverse.ts#L88) + +Runs after a row was updated in the main schema. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `id1` | `null` \| `string` | +| `id2` | `string` | +| `oldID1` | `null` \| `string` | + +#### Returns + +`Promise`\<`void`\> + +*** + +### afterDelete() + +> **afterDelete**(`vc`, `id1`, `id2`): `Promise`\<`void`\> + +Defined in: [src/ent/Inverse.ts:107](https://github.com/clickup/ent-framework/blob/master/src/ent/Inverse.ts#L107) + +Runs after a row was deleted in the main schema. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `id1` | `null` \| `string` | +| `id2` | `string` | + +#### Returns + +`Promise`\<`void`\> + +*** + +### id2s() + +> **id2s**(`vc`, `id1`): `Promise`\<`string`[]\> + +Defined in: [src/ent/Inverse.ts:130](https://github.com/clickup/ent-framework/blob/master/src/ent/Inverse.ts#L130) + +Returns all id2s by a particular (id1, type) pair. The number of resulting +rows is limited to not overload the database. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `id1` | `null` \| `string` | + +#### Returns + +`Promise`\<`string`[]\> diff --git a/docs/classes/Island.md b/docs/classes/Island.md new file mode 100644 index 0000000..845c035 --- /dev/null +++ b/docs/classes/Island.md @@ -0,0 +1,177 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Island + +# Class: Island\ + +Defined in: [src/abstract/Island.ts:94](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L94) + +Island is a moderately short-lived collection of DB connections (represented +as Clients) that contains a single master Client and any number of replicas. + +- In normal situations, you don't likely need to work with Islands directly, + you can rely on higher level abstractions which support automatic + rediscovery and retries: Ent (or lower level Shard and Schema). +- Islands are helpful mostly when working with cross-Shards logic. +- Island is somewhat temporary: if the Cluster is reconfigured in real-time, + then its Island objects may be recycled and re-created, and the + corresponding Clients may be ended. This also applies to any given Client + instance. Don't retain and reuse those objects for too long. The reliable + abstractions (resilient to disconnects, shards migration, failover etc.) + start from Shard level. +- There is no guarantee that the data returned by shards(), master() or + replica() will be up to date. Shards may be just migrated to another + Island. Master may become a replica, or vice versa. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TClient` *extends* [`Client`](Client.md) | + +## Constructors + +### new Island() + +> **new Island**\<`TClient`\>(`options`): [`Island`](Island.md)\<`TClient`\> + +Defined in: [src/abstract/Island.ts:121](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L121) + +Initializes the Island by copying the Client references into it. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`IslandOptions`](../interfaces/IslandOptions.md)\<`TClient`\> | + +#### Returns + +[`Island`](Island.md)\<`TClient`\> + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`IslandOptions`](../interfaces/IslandOptions.md)\<[`Client`](Client.md)\>\>\> | Default values for the constructor options. | +| `options` | `Required`\<[`IslandOptions`](../interfaces/IslandOptions.md)\<`TClient`\>\> | Island configuration options. | + +## Accessors + +### no + +#### Get Signature + +> **get** **no**(): `number` + +Defined in: [src/abstract/Island.ts:133](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L133) + +Island number. + +##### Returns + +`number` + +*** + +### clients + +#### Get Signature + +> **get** **clients**(): readonly `TClient`[] + +Defined in: [src/abstract/Island.ts:140](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L140) + +The list of Clients in this Island. No assumptions about the order. + +##### Returns + +readonly `TClient`[] + +## Methods + +### rediscover() + +> **rediscover**(): `Promise`\<[`SwallowedErrorLoggerProps`](../interfaces/SwallowedErrorLoggerProps.md)[]\> + +Defined in: [src/abstract/Island.ts:157](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L157) + +Queries for Shards on the best available Client (preferably master, then +replicas) and stores the result internally, available for the further +shards() call. +- If some Clients are unavailable, tries its best to infer the data from + other Clients. +- The method queries ALL clients in parallel, because the caller logic + anyways needs to know, who's master and who's replica, as a side effect + of the very 1st query after the Client creation. We infer that as a piggy + back after calling Client#shardNos(). +- In case we could not discover shards, returns the list of errors happened + during the discovery. + +#### Returns + +`Promise`\<[`SwallowedErrorLoggerProps`](../interfaces/SwallowedErrorLoggerProps.md)[]\> + +*** + +### shards() + +> **shards**(): [`Shard`](Shard.md)\<`TClient`\>[] + +Defined in: [src/abstract/Island.ts:224](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L224) + +Returns the currently best-known Shards on this Island. This method is +needed only when working with cross-Shards logic; in normal situations, it +is not called much. + +#### Returns + +[`Shard`](Shard.md)\<`TClient`\>[] + +*** + +### master() + +> **master**(): `TClient` + +Defined in: [src/abstract/Island.ts:252](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L252) + +Returns the currently best-known master Client among the Clients of this +Island. + +- If all masters are unhealthy, we still return one of them and prefer not + to fall back on a replica, because otherwise, we'll see non-obvious + errors in logs ("can't write in a read-only Client" or so) and suspect + that there is a bug in logic, although there is really no bug, it's just + the master node went down. It's way better to throw a straightforward + error like "Client is down". +- If we can't find a master, but there is a list of Clients with unknown + roles, prefer returning one of them vs. any known replica, since there is + a chance that among those unknown Clients, there will be a master. +- In case all Clients are read-only (replicas), still returns the 1st of + them, assuming that it's better to throw at the caller side on a failed + write (at worst) rather than here. It is not common to have an Island + without a master Client, that happens only temporarily during + failover/switchover, so the caller will likely rediscover and find a new + master on a next retry. + +#### Returns + +`TClient` + +*** + +### replica() + +> **replica**(): `TClient` + +Defined in: [src/abstract/Island.ts:279](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L279) + +Returns a currently best-known random replica Client. In case there are no +replicas, returns the master Client. + +#### Returns + +`TClient` diff --git a/docs/classes/Loader.md b/docs/classes/Loader.md new file mode 100644 index 0000000..632e0e6 --- /dev/null +++ b/docs/classes/Loader.md @@ -0,0 +1,75 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Loader + +# Class: Loader\ + +Defined in: [src/abstract/Loader.ts:42](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loader.ts#L42) + +Loader allows to batch single-item requests into batches. It uses a different +architecture than Facebook's DataLoader: + +- it's more developers-friendly: multi-parameter loadings, you may implement + automatic deduplication of requests at onCollect stage, no requirement to + serialize/deserialize requests into string keys; +- strong-typed load() and handler arguments. + +To create your own specific loader: +1. Define a handler class with onCollect/onReturn/onFlush methods. +2. In onCollect, accumulate the incoming requests in the handler object's + private property. +3. In onFlush, process what you accumulated so far and save to another + handler object's private property (and by adding, say, delay(50) in the + beginning of onFlush, you may group the requests coming within the 1st 50 + ms). +3. In onReturn, extract the result corresponding to the request and return + it, so the caller will receive it seamlessly as a load() return value. + +In the future, Batcher may be refactored to use Loader as the underlying +engine, but for now they're separate (Batcher is much more domain logic +specific and Loader is completely abstract). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TLoadArgs` *extends* `unknown`[] | +| `TReturn` | + +## Constructors + +### new Loader() + +> **new Loader**\<`TLoadArgs`, `TReturn`\>(`handlerCreator`): [`Loader`](Loader.md)\<`TLoadArgs`, `TReturn`\> + +Defined in: [src/abstract/Loader.ts:45](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loader.ts#L45) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `handlerCreator` | () => [`Handler`](../interfaces/Handler.md)\<`TLoadArgs`, `TReturn`\> | + +#### Returns + +[`Loader`](Loader.md)\<`TLoadArgs`, `TReturn`\> + +## Methods + +### load() + +> **load**(...`args`): `Promise`\<`TReturn`\> + +Defined in: [src/abstract/Loader.ts:47](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loader.ts#L47) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| ...`args` | `TLoadArgs` | + +#### Returns + +`Promise`\<`TReturn`\> diff --git a/docs/classes/LocalCache.md b/docs/classes/LocalCache.md new file mode 100644 index 0000000..4faa66d --- /dev/null +++ b/docs/classes/LocalCache.md @@ -0,0 +1,109 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / LocalCache + +# Class: LocalCache\ + +Defined in: [src/abstract/LocalCache.ts:45](https://github.com/clickup/ent-framework/blob/master/src/abstract/LocalCache.ts#L45) + +A simple key-value cache stored on the local machine. + +- The expectation is that there will be not too many keys stored, since the + background cleanup process running time to time is O(numKeysStored). +- Guarantees corruption-free writes to the keys from multiple processes + running concurrently. +- The values which are not requested longer than approximately `expirationMs` + are auto-removed. +- Each key is stored in an individual file under `dir`. Some temporary files + may also appear in that directory, but eventually, they will be cleaned up, + even if they get stuck for some time. + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TValue` *extends* `object` | `never` | + +## Constructors + +### new LocalCache() + +> **new LocalCache**\<`TValue`\>(`options`): [`LocalCache`](LocalCache.md)\<`TValue`\> + +Defined in: [src/abstract/LocalCache.ts:64](https://github.com/clickup/ent-framework/blob/master/src/abstract/LocalCache.ts#L64) + +Initializes the instance. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`LocalCacheOptions`](../interfaces/LocalCacheOptions.md) | + +#### Returns + +[`LocalCache`](LocalCache.md)\<`TValue`\> + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`LocalCacheOptions`](../interfaces/LocalCacheOptions.md)\>\> | Default values for the constructor options. | +| `options` | `Required`\<[`LocalCacheOptions`](../interfaces/LocalCacheOptions.md)\> | LocalCache configuration options. | + +## Methods + +### end() + +> **end**(): `void` + +Defined in: [src/abstract/LocalCache.ts:78](https://github.com/clickup/ent-framework/blob/master/src/abstract/LocalCache.ts#L78) + +Ends the instance lifecycle (e.g. garbage recheck interval). + +#### Returns + +`void` + +*** + +### get() + +> **get**(`key`): `Promise`\<`null` \| `TValue`\> + +Defined in: [src/abstract/LocalCache.ts:86](https://github.com/clickup/ent-framework/blob/master/src/abstract/LocalCache.ts#L86) + +Returns the value for the given key, or null if the key does not exist. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `key` | `string` | + +#### Returns + +`Promise`\<`null` \| `TValue`\> + +*** + +### set() + +> **set**(`key`, `value`): `Promise`\<`void`\> + +Defined in: [src/abstract/LocalCache.ts:105](https://github.com/clickup/ent-framework/blob/master/src/abstract/LocalCache.ts#L105) + +Sets the value for the given key. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `key` | `string` | +| `value` | `TValue` | + +#### Returns + +`Promise`\<`void`\> diff --git a/docs/classes/Or.md b/docs/classes/Or.md new file mode 100644 index 0000000..2840800 --- /dev/null +++ b/docs/classes/Or.md @@ -0,0 +1,69 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Or + +# Class: Or\ + +Defined in: [src/ent/predicates/Or.ts:10](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Or.ts#L10) + +Checks that at least one of the children predicates succeed. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` | + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> + +## Constructors + +### new Or() + +> **new Or**\<`TInput`\>(...`predicates`): [`Or`](Or.md)\<`TInput`\> + +Defined in: [src/ent/predicates/Or.ts:14](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Or.ts#L14) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| ...`predicates` | readonly ([`Predicate`](../interfaces/Predicate.md)\<`TInput`\> \| (`vc`, `input`) => `boolean` \| `Promise`\<`boolean`\>)[] | + +#### Returns + +[`Or`](Or.md)\<`TInput`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | +| `predicates` | readonly [`Predicate`](../interfaces/Predicate.md)\<`TInput`\>[] | + +## Methods + +### check() + +> **check**(`vc`, `input`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/Or.ts:27](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Or.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `input` | `TInput` | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/OutgoingEdgePointsToVC.md b/docs/classes/OutgoingEdgePointsToVC.md new file mode 100644 index 0000000..b858369 --- /dev/null +++ b/docs/classes/OutgoingEdgePointsToVC.md @@ -0,0 +1,71 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / OutgoingEdgePointsToVC + +# Class: OutgoingEdgePointsToVC\ + +Defined in: [src/ent/predicates/OutgoingEdgePointsToVC.ts:9](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/OutgoingEdgePointsToVC.ts#L9) + +Checks that the field's value is the same as VC's principal: + +EntOur[user_id] ---> vc.principal + +## Type Parameters + +| Type Parameter | +| ------ | +| `TField` *extends* `string` | + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<`Record`\<`TField`, `string` \| `null`\>\> + +## Constructors + +### new OutgoingEdgePointsToVC() + +> **new OutgoingEdgePointsToVC**\<`TField`\>(`field`): [`OutgoingEdgePointsToVC`](OutgoingEdgePointsToVC.md)\<`TField`\> + +Defined in: [src/ent/predicates/OutgoingEdgePointsToVC.ts:14](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/OutgoingEdgePointsToVC.ts#L14) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `field` | `TField` | + +#### Returns + +[`OutgoingEdgePointsToVC`](OutgoingEdgePointsToVC.md)\<`TField`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | +| `field` | `TField` | + +## Methods + +### check() + +> **check**(`vc`, `row`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/OutgoingEdgePointsToVC.ts:18](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/OutgoingEdgePointsToVC.ts#L18) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | `Record`\<`TField`, `null` \| `string`\> | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/PgClient.md b/docs/classes/PgClient.md new file mode 100644 index 0000000..bde5e54 --- /dev/null +++ b/docs/classes/PgClient.md @@ -0,0 +1,425 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgClient + +# Class: PgClient\ + +Defined in: [src/pg/PgClient.ts:128](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L128) + +An abstract PostgreSQL Client. Includes connection pooling logic. + +Since the class is cloneable internally (using the prototype substitution +technique, see withShard()), the contract of this class is that ALL its +derived classes may only have readonly immediate properties. Use Ref helper +if you need some mutable properties. + +## Extends + +- [`Client`](Client.md) + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TPool` *extends* `pg.Pool` | `pg.Pool` | + +## Constructors + +### new PgClient() + +> **new PgClient**\<`TPool`\>(`options`): [`PgClient`](PgClient.md)\<`TPool`\> + +Defined in: [src/pg/PgClient.ts:190](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L190) + +Initializes an instance of PgClient. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`PgClientOptions`](../interfaces/PgClientOptions.md)\<`TPool`\> | + +#### Returns + +[`PgClient`](PgClient.md)\<`TPool`\> + +#### Overrides + +[`Client`](Client.md).[`constructor`](Client.md#constructors) + +## Properties + +| Property | Type | Default value | Description | +| ------ | ------ | ------ | ------ | +| `createdAt` | `Date` | `undefined` | Date when this Client instance was constructed. | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`PgClientOptions`](../interfaces/PgClientOptions.md)\<`Pool`\>\>\> | `undefined` | Default values for the constructor options. | +| `options` | `Required`\<[`PgClientOptions`](../interfaces/PgClientOptions.md)\<`TPool`\>\> | `undefined` | PgClient configuration options. | +| `shardName` | `string` | `"public"` | Name of the shard associated to this Client. | +| `timelineManager` | [`TimelineManager`](TimelineManager.md) | `undefined` | An active TimelineManager for this particular Client. | + +## Methods + +### batcher() + +> **batcher**\<`TInput`, `TOutput`, `TTable`\>(`_QueryClass`, `_schema`, `_additionalShape`, `disableBatching`, `runnerCreator`): [`Batcher`](Batcher.md)\<`TInput`, `TOutput`\> + +Defined in: [src/abstract/Client.ts:189](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L189) + +Batcher is per-Client per-query-type +per-table-name-and-shape-and-disableBatching: + +- Per-Client means that batchers are removed as soon as the Client is + removed, i.e. the Client owns all the batchers for all tables. +- Per-query-type means that the batcher for a SELECT query is different + from the batcher for an INSERT query (obviously). +- Per-table-name-and-shape-and-disableBatching means that each table has + its own set of batchers (obviously). Also, some queries may be complex + (like UPDATE), so the batcher also depends on the "shape" - the list of + fields we're updating. Plus, for some inputs, we want to disable batching + at all - that produces a separate Batcher instance. + +Also, for every Batcher, there is exactly one Runner (which knows how to +build the actual query in the context of the current Client). Batchers are +generic (like DataLoader, but more general), and Runners are very custom to +the query (and are private to these queries). + +All that means that in a 1000-Shard 20-table Cluster we'll eventually have +1000x20x8 Batchers/Runners (assuming we have 8 different operations). + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TInput` | +| `TOutput` | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `_QueryClass` | `Function` | +| `_schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `_additionalShape` | `string` | +| `disableBatching` | `boolean` | +| `runnerCreator` | () => [`Runner`](Runner.md)\<`TInput`, `TOutput`\> | + +#### Returns + +[`Batcher`](Batcher.md)\<`TInput`, `TOutput`\> + +#### Inherited from + +[`Client`](Client.md).[`batcher`](Client.md#batcher) + +*** + +### logSwallowedError() + +> `protected` **logSwallowedError**(`props`): `void` + +Defined in: [src/pg/PgClient.ts:181](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L181) + +Calls swallowedErrorLogger() doing some preliminary amendment. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `props` | [`SwallowedErrorLoggerProps`](../interfaces/SwallowedErrorLoggerProps.md) | + +#### Returns + +`void` + +#### Overrides + +[`Client`](Client.md).[`logSwallowedError`](Client.md#logswallowederror) + +*** + +### address() + +> **address**(): `string` + +Defined in: [src/pg/PgClient.ts:239](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L239) + +Represents the full destination address this Client is working with. +Depending on the implementation, it may include hostname, port number, +database name, shard name etc. It is required that the address is stable +enough to be able to cache some destination database related metadata (e.g. +shardNos) based on that address. + +#### Returns + +`string` + +#### Overrides + +[`Client`](Client.md).[`address`](Client.md#address) + +*** + +### end() + +> **end**(): `Promise`\<`void`\> + +Defined in: [src/pg/PgClient.ts:256](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L256) + +Gracefully closes all the connections of this Client to let the caller +destroy it. The pending queries are awaited to finish before returning. The +Client becomes unusable right after calling this method (even before the +connections are drained): you should not send queries to it. + +#### Returns + +`Promise`\<`void`\> + +#### Overrides + +[`Client`](Client.md).[`end`](Client.md#end) + +*** + +### isEnded() + +> **isEnded**(): `boolean` + +Defined in: [src/pg/PgClient.ts:270](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L270) + +Returns true if the Client is ended and can't be used anymore. + +#### Returns + +`boolean` + +#### Overrides + +[`Client`](Client.md).[`isEnded`](Client.md#isended) + +*** + +### shardNos() + +> **shardNos**(): `Promise`\ + +Defined in: [src/pg/PgClient.ts:278](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L278) + +Returns all Shard numbers discoverable via the connection to the Client's +database. + +#### Returns + +`Promise`\ + +#### Overrides + +[`Client`](Client.md).[`shardNos`](Client.md#shardnos) + +*** + +### ping() + +> **ping**(`__namedParameters`): `Promise`\<`void`\> + +Defined in: [src/pg/PgClient.ts:305](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L305) + +Sends a read or write test query to the server. Tells the server to sit and +wait for at least the provided number of milliseconds. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `__namedParameters` | [`ClientPingInput`](../interfaces/ClientPingInput.md) | + +#### Returns + +`Promise`\<`void`\> + +#### Overrides + +[`Client`](Client.md).[`ping`](Client.md#ping) + +*** + +### withShard() + +> **withShard**(`no`): `this` + +Defined in: [src/pg/PgClient.ts:327](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L327) + +Creates a new Client which is namespaced to the provided Shard number. The +new Client will share the same connection pool with the parent's Client. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `no` | `number` | + +#### Returns + +`this` + +#### Overrides + +[`Client`](Client.md).[`withShard`](Client.md#withshard) + +*** + +### role() + +> **role**(): [`ClientRole`](../type-aliases/ClientRole.md) + +Defined in: [src/pg/PgClient.ts:345](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L345) + +Returns the Client's role reported after the last successful query. Master +and replica roles may switch online unpredictably, without reconnecting, so +we only know the role after a query. + +#### Returns + +[`ClientRole`](../type-aliases/ClientRole.md) + +#### Overrides + +[`Client`](Client.md).[`role`](Client.md#role) + +*** + +### connectionIssue() + +> **connectionIssue**(): `null` \| [`ClientConnectionIssue`](../interfaces/ClientConnectionIssue.md) + +Defined in: [src/pg/PgClient.ts:355](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L355) + +Returns a non-nullable value if the Client couldn't connect to the server +(or it could, but the load balancer reported the remote server as not +working), so it should ideally be removed from the list of active replicas +until e.g. the next discovery query to it (or any query) succeeds. + +#### Returns + +`null` \| [`ClientConnectionIssue`](../interfaces/ClientConnectionIssue.md) + +#### Overrides + +[`Client`](Client.md).[`connectionIssue`](Client.md#connectionissue) + +*** + +### prewarm() + +> **prewarm**(): `void` + +Defined in: [src/pg/PgClient.ts:365](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L365) + +A convenience method to put connections prewarming logic to. The idea is to +keep the needed number of open connections and also, in each connection, +minimize the time which the very 1st query will take (e.g. pre-cache +full-text dictionaries). + +#### Returns + +`void` + +#### Overrides + +[`Client`](Client.md).[`prewarm`](Client.md#prewarm) + +*** + +### pool() + +> **pool**(`subPoolConfig`?): `TPool` + +Defined in: [src/pg/PgClient.ts:431](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L431) + +Returns a default pool (when subPoolConfig is not passed), or a "sub-pool" +(a named low-level Pool implementation compatible to node-postgres). The +method is specific to the current class and is not a part of +database-agnostic Client API. +- Sub-pools are lazily created and memoized by the provided name. They may + differ by config options (like statement_timeout or max connections). +- Sub-pools inherit the properties from default PgClientOptions.config. +- It is implied (but not enforced) that all sub-pools use the same physical + database, because otherwise it makes not a lot of sense. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `subPoolConfig`? | [`PgClientSubPoolConfig`](../interfaces/PgClientSubPoolConfig.md) | + +#### Returns + +`TPool` + +*** + +### acquireConn() + +> **acquireConn**(`subPoolConfig`?): `Promise`\<[`PgClientConn`](../interfaces/PgClientConn.md)\<`TPool`\>\> + +Defined in: [src/pg/PgClient.ts:493](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L493) + +Called when the Client needs a connection in the default pool (when +subPoolConfig is not passed), or in a sub-pool (see pool() method) to run a +query against. Implies than the caller MUST call release() method on the +returned object. The difference from pool().connect() is that when calling +release() on a result of acquireConn(), it additionally closes the +connection automatically if was OPENED (not queried!) more than +maxConnLifetimeMs ago (node-postgres Pool doesn't have this feature) The +method is specific to the current class and is not a part of +database-agnostic Client API. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `subPoolConfig`? | [`PgClientSubPoolConfig`](../interfaces/PgClientSubPoolConfig.md) | + +#### Returns + +`Promise`\<[`PgClientConn`](../interfaces/PgClientConn.md)\<`TPool`\>\> + +*** + +### query() + +> **query**\<`TRow`\>(`__namedParameters`): `Promise`\<`TRow`[]\> + +Defined in: [src/pg/PgClient.ts:517](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L517) + +Sends a query (internally, a multi-query) through the default Pool (if +subPoolConfig is not passed), or through a named sub-pool (see pool() +method). After the query finishes, we should expect that role() returns the +actual master/replica role. The method is specific to the current class and +is not a part of database-agnostic Client API. + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TRow` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `__namedParameters` | \{ `query`: [`Literal`](../type-aliases/Literal.md); `hints`: [`Hints`](../type-aliases/Hints.md); `isWrite`: `boolean`; `annotations`: [`QueryAnnotation`](../interfaces/QueryAnnotation.md)[]; `op`: `string`; `table`: `string`; `batchFactor`: `number`; `subPoolConfig`: [`PgClientSubPoolConfig`](../interfaces/PgClientSubPoolConfig.md); \} | +| `__namedParameters.query` | [`Literal`](../type-aliases/Literal.md) | +| `__namedParameters.hints`? | [`Hints`](../type-aliases/Hints.md) | +| `__namedParameters.isWrite` | `boolean` | +| `__namedParameters.annotations` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md)[] | +| `__namedParameters.op` | `string` | +| `__namedParameters.table` | `string` | +| `__namedParameters.batchFactor`? | `number` | +| `__namedParameters.subPoolConfig`? | [`PgClientSubPoolConfig`](../interfaces/PgClientSubPoolConfig.md) | + +#### Returns + +`Promise`\<`TRow`[]\> diff --git a/docs/classes/PgError.md b/docs/classes/PgError.md new file mode 100644 index 0000000..203e07c --- /dev/null +++ b/docs/classes/PgError.md @@ -0,0 +1,71 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgError + +# Class: PgError + +Defined in: [src/pg/PgError.ts:3](https://github.com/clickup/ent-framework/blob/master/src/pg/PgError.ts#L3) + +Encapsulates the error thrown when running a Client query. The object also +carries suggestions, what to do next. + +## Extends + +- [`ClientError`](ClientError.md) + +## Constructors + +### new PgError() + +> **new PgError**(`cause`, `where`, `sql`, `table`): [`PgError`](PgError.md) + +Defined in: [src/pg/PgError.ts:4](https://github.com/clickup/ent-framework/blob/master/src/pg/PgError.ts#L4) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `cause` | `undefined` \| `null` \| \{\} | +| `where` | `string` | +| `sql` | `string` | +| `table` | `string` | + +#### Returns + +[`PgError`](PgError.md) + +#### Overrides + +[`ClientError`](ClientError.md).[`constructor`](ClientError.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `cause` | `MaybeError` | +| `postAction` | [`ClientErrorPostAction`](../type-aliases/ClientErrorPostAction.md) | +| `kind` | [`ClientErrorKind`](../type-aliases/ClientErrorKind.md) | +| `abbreviation` | `string` | +| `comment?` | `string` | +| `sql` | `string` | +| `table` | `string` | + +## Methods + +### isFKError() + +> **isFKError**(`fkName`?): `boolean` + +Defined in: [src/pg/PgError.ts:19](https://github.com/clickup/ent-framework/blob/master/src/pg/PgError.ts#L19) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `fkName`? | `string` | + +#### Returns + +`boolean` diff --git a/docs/classes/PgQueryCount.md b/docs/classes/PgQueryCount.md new file mode 100644 index 0000000..eed8b41 --- /dev/null +++ b/docs/classes/PgQueryCount.md @@ -0,0 +1,96 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryCount + +# Class: PgQueryCount\ + +Defined in: [src/pg/PgQueryCount.ts:8](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryCount.ts#L8) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, [`CountInput`](../type-aliases/CountInput.md)\<`TTable`\>, `number`, [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new PgQueryCount() + +> **new PgQueryCount**\<`TTable`\>(`schema`, `input`): [`PgQueryCount`](PgQueryCount.md)\<`TTable`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`CountInput`](../type-aliases/CountInput.md)\<`TTable`\> | + +#### Returns + +[`PgQueryCount`](PgQueryCount.md)\<`TTable`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`CountInput`](../type-aliases/CountInput.md)\<`TTable`\> | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`number`\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`number`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQueryDelete.md b/docs/classes/PgQueryDelete.md new file mode 100644 index 0000000..c5ef8ad --- /dev/null +++ b/docs/classes/PgQueryDelete.md @@ -0,0 +1,96 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryDelete + +# Class: PgQueryDelete\ + +Defined in: [src/pg/PgQueryDelete.ts:9](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryDelete.ts#L9) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, `string`, `boolean`, [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new PgQueryDelete() + +> **new PgQueryDelete**\<`TTable`\>(`schema`, `input`): [`PgQueryDelete`](PgQueryDelete.md)\<`TTable`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | `string` | + +#### Returns + +[`PgQueryDelete`](PgQueryDelete.md)\<`TTable`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | `string` | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`boolean`\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`boolean`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQueryDeleteWhere.md b/docs/classes/PgQueryDeleteWhere.md new file mode 100644 index 0000000..46f939d --- /dev/null +++ b/docs/classes/PgQueryDeleteWhere.md @@ -0,0 +1,96 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryDeleteWhere + +# Class: PgQueryDeleteWhere\ + +Defined in: [src/pg/PgQueryDeleteWhere.ts:10](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryDeleteWhere.ts#L10) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, [`DeleteWhereInput`](../type-aliases/DeleteWhereInput.md)\<`TTable`\>, `string`[], [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new PgQueryDeleteWhere() + +> **new PgQueryDeleteWhere**\<`TTable`\>(`schema`, `input`): [`PgQueryDeleteWhere`](PgQueryDeleteWhere.md)\<`TTable`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`DeleteWhereInput`](../type-aliases/DeleteWhereInput.md)\<`TTable`\> | + +#### Returns + +[`PgQueryDeleteWhere`](PgQueryDeleteWhere.md)\<`TTable`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`DeleteWhereInput`](../type-aliases/DeleteWhereInput.md)\<`TTable`\> | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`string`[]\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`string`[]\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQueryExists.md b/docs/classes/PgQueryExists.md new file mode 100644 index 0000000..59c0331 --- /dev/null +++ b/docs/classes/PgQueryExists.md @@ -0,0 +1,96 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryExists + +# Class: PgQueryExists\ + +Defined in: [src/pg/PgQueryExists.ts:8](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryExists.ts#L8) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, [`ExistsInput`](../type-aliases/ExistsInput.md)\<`TTable`\>, `boolean`, [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new PgQueryExists() + +> **new PgQueryExists**\<`TTable`\>(`schema`, `input`): [`PgQueryExists`](PgQueryExists.md)\<`TTable`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`ExistsInput`](../type-aliases/ExistsInput.md)\<`TTable`\> | + +#### Returns + +[`PgQueryExists`](PgQueryExists.md)\<`TTable`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`ExistsInput`](../type-aliases/ExistsInput.md)\<`TTable`\> | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`boolean`\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`boolean`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQueryIDGen.md b/docs/classes/PgQueryIDGen.md new file mode 100644 index 0000000..3df5903 --- /dev/null +++ b/docs/classes/PgQueryIDGen.md @@ -0,0 +1,96 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryIDGen + +# Class: PgQueryIDGen\ + +Defined in: [src/pg/PgQueryIDGen.ts:10](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryIDGen.ts#L10) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, `void`, `string`, [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new PgQueryIDGen() + +> **new PgQueryIDGen**\<`TTable`\>(`schema`, `input`): [`PgQueryIDGen`](PgQueryIDGen.md)\<`TTable`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | `void` | + +#### Returns + +[`PgQueryIDGen`](PgQueryIDGen.md)\<`TTable`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | `void` | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`string`\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`string`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQueryInsert.md b/docs/classes/PgQueryInsert.md new file mode 100644 index 0000000..431f23f --- /dev/null +++ b/docs/classes/PgQueryInsert.md @@ -0,0 +1,96 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryInsert + +# Class: PgQueryInsert\ + +Defined in: [src/pg/PgQueryInsert.ts:9](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryInsert.ts#L9) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>, `string` \| `null`, [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new PgQueryInsert() + +> **new PgQueryInsert**\<`TTable`\>(`schema`, `input`): [`PgQueryInsert`](PgQueryInsert.md)\<`TTable`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +[`PgQueryInsert`](PgQueryInsert.md)\<`TTable`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`null` \| `string`\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`null` \| `string`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQueryLoad.md b/docs/classes/PgQueryLoad.md new file mode 100644 index 0000000..9796e02 --- /dev/null +++ b/docs/classes/PgQueryLoad.md @@ -0,0 +1,96 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryLoad + +# Class: PgQueryLoad\ + +Defined in: [src/pg/PgQueryLoad.ts:9](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryLoad.ts#L9) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, `string`, [`Row`](../type-aliases/Row.md)\<`TTable`\> \| `null`, [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new PgQueryLoad() + +> **new PgQueryLoad**\<`TTable`\>(`schema`, `input`): [`PgQueryLoad`](PgQueryLoad.md)\<`TTable`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | `string` | + +#### Returns + +[`PgQueryLoad`](PgQueryLoad.md)\<`TTable`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | `string` | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQueryLoadBy.md b/docs/classes/PgQueryLoadBy.md new file mode 100644 index 0000000..f7811f3 --- /dev/null +++ b/docs/classes/PgQueryLoadBy.md @@ -0,0 +1,97 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryLoadBy + +# Class: PgQueryLoadBy\ + +Defined in: [src/pg/PgQueryLoadBy.ts:8](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryLoadBy.ts#L8) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TUniqueKey`\>, [`Row`](../type-aliases/Row.md)\<`TTable`\> \| `null`, [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | + +## Constructors + +### new PgQueryLoadBy() + +> **new PgQueryLoadBy**\<`TTable`, `TUniqueKey`\>(`schema`, `input`): [`PgQueryLoadBy`](PgQueryLoadBy.md)\<`TTable`, `TUniqueKey`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TUniqueKey`\> | + +#### Returns + +[`PgQueryLoadBy`](PgQueryLoadBy.md)\<`TTable`, `TUniqueKey`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TUniqueKey`\> | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQuerySelect.md b/docs/classes/PgQuerySelect.md new file mode 100644 index 0000000..5875075 --- /dev/null +++ b/docs/classes/PgQuerySelect.md @@ -0,0 +1,96 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQuerySelect + +# Class: PgQuerySelect\ + +Defined in: [src/pg/PgQuerySelect.ts:35](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQuerySelect.ts#L35) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extends + +- [`QueryBase`](QueryBase.md)\<`TTable`, [`SelectInput`](../type-aliases/SelectInput.md)\<`TTable`\>, [`Row`](../type-aliases/Row.md)\<`TTable`\>[], [`PgClient`](PgClient.md)\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new PgQuerySelect() + +> **new PgQuerySelect**\<`TTable`\>(`schema`, `input`): [`PgQuerySelect`](PgQuerySelect.md)\<`TTable`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`SelectInput`](../type-aliases/SelectInput.md)\<`TTable`\> | + +#### Returns + +[`PgQuerySelect`](PgQuerySelect.md)\<`TTable`\> + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`SelectInput`](../type-aliases/SelectInput.md)\<`TTable`\> | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Inherited from + +[`QueryBase`](QueryBase.md).[`IS_WRITE`](QueryBase.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +Defined in: [src/pg/PgQuerySelect.ts:44](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQuerySelect.ts#L44) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +#### Overrides + +[`QueryBase`](QueryBase.md).[`run`](QueryBase.md#run) diff --git a/docs/classes/PgQuerySelectBy.md b/docs/classes/PgQuerySelectBy.md new file mode 100644 index 0000000..84eb34f --- /dev/null +++ b/docs/classes/PgQuerySelectBy.md @@ -0,0 +1,73 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQuerySelectBy + +# Class: PgQuerySelectBy\ + +Defined in: [src/pg/PgQuerySelectBy.ts:14](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQuerySelectBy.ts#L14) + +A very lean interface for a Query. In practice each query is so different +that this interface is the only common part of them all. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | + +## Implements + +- [`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +## Constructors + +### new PgQuerySelectBy() + +> **new PgQuerySelectBy**\<`TTable`, `TUniqueKey`\>(`schema`, `input`): [`PgQuerySelectBy`](PgQuerySelectBy.md)\<`TTable`, `TUniqueKey`\> + +Defined in: [src/pg/PgQuerySelectBy.ts:20](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQuerySelectBy.ts#L20) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`SelectByInput`](../type-aliases/SelectByInput.md)\<`TTable`, `TUniqueKey`\> | + +#### Returns + +[`PgQuerySelectBy`](PgQuerySelectBy.md)\<`TTable`, `TUniqueKey`\> + +## Properties + +| Property | Type | Default value | +| ------ | ------ | ------ | +| `IS_WRITE` | `false` | `false` | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | `undefined` | +| `input` | [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TuplePrefixes`\<`TUniqueKey`\>\> | `undefined` | + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +Defined in: [src/pg/PgQuerySelectBy.ts:25](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQuerySelectBy.ts#L25) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +#### Implementation of + +[`Query`](../interfaces/Query.md).[`run`](../interfaces/Query.md#run) diff --git a/docs/classes/PgQueryUpdate.md b/docs/classes/PgQueryUpdate.md new file mode 100644 index 0000000..d01696f --- /dev/null +++ b/docs/classes/PgQueryUpdate.md @@ -0,0 +1,73 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryUpdate + +# Class: PgQueryUpdate\ + +Defined in: [src/pg/PgQueryUpdate.ts:10](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryUpdate.ts#L10) + +A very lean interface for a Query. In practice each query is so different +that this interface is the only common part of them all. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Implements + +- [`Query`](../interfaces/Query.md)\<`boolean`\> + +## Constructors + +### new PgQueryUpdate() + +> **new PgQueryUpdate**\<`TTable`\>(`schema`, `id`, `input`): [`PgQueryUpdate`](PgQueryUpdate.md)\<`TTable`\> + +Defined in: [src/pg/PgQueryUpdate.ts:15](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryUpdate.ts#L15) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `id` | `string` | +| `input` | [`UpdateInput`](../type-aliases/UpdateInput.md)\<`TTable`\> | + +#### Returns + +[`PgQueryUpdate`](PgQueryUpdate.md)\<`TTable`\> + +## Properties + +| Property | Type | Default value | +| ------ | ------ | ------ | +| `input` | \{ \[K in string \| number \| symbol\]?: Value\ \} & `object` & `object` | `undefined` | +| `IS_WRITE` | `true` | `true` | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | `undefined` | + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`boolean`\> + +Defined in: [src/pg/PgQueryUpdate.ts:25](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryUpdate.ts#L25) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Query`](../interfaces/Query.md).[`run`](../interfaces/Query.md#run) diff --git a/docs/classes/PgQueryUpsert.md b/docs/classes/PgQueryUpsert.md new file mode 100644 index 0000000..00b58f8 --- /dev/null +++ b/docs/classes/PgQueryUpsert.md @@ -0,0 +1,72 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgQueryUpsert + +# Class: PgQueryUpsert\ + +Defined in: [src/pg/PgQueryUpsert.ts:15](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryUpsert.ts#L15) + +A very lean interface for a Query. In practice each query is so different +that this interface is the only common part of them all. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Implements + +- [`Query`](../interfaces/Query.md)\<`string`\> + +## Constructors + +### new PgQueryUpsert() + +> **new PgQueryUpsert**\<`TTable`\>(`schema`, `input`): [`PgQueryUpsert`](PgQueryUpsert.md)\<`TTable`\> + +Defined in: [src/pg/PgQueryUpsert.ts:18](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryUpsert.ts#L18) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +[`PgQueryUpsert`](PgQueryUpsert.md)\<`TTable`\> + +## Properties + +| Property | Type | Default value | +| ------ | ------ | ------ | +| `IS_WRITE` | `true` | `true` | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | `undefined` | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | `undefined` | + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`string`\> + +Defined in: [src/pg/PgQueryUpsert.ts:23](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQueryUpsert.ts#L23) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`string`\> + +#### Implementation of + +[`Query`](../interfaces/Query.md).[`run`](../interfaces/Query.md#run) diff --git a/docs/classes/PgRunner.md b/docs/classes/PgRunner.md new file mode 100644 index 0000000..dc4829d --- /dev/null +++ b/docs/classes/PgRunner.md @@ -0,0 +1,706 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgRunner + +# Class: `abstract` PgRunner\ + +Defined in: [src/pg/PgRunner.ts:52](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L52) + +A convenient pile of helper methods usable by most of PgQuery* classes. In +some sense it's an anti-pattern, but still reduces the boilerplate. + +PgRunner is also responsible for stringifying the values passed to the +queries and parsing values returned from the DB according to the field types +specs. + +## Extends + +- [`Runner`](Runner.md)\<`TInput`, `TOutput`\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TInput` | +| `TOutput` | + +## Constructors + +### new PgRunner() + +> **new PgRunner**\<`TTable`, `TInput`, `TOutput`\>(`schema`, `client`): [`PgRunner`](PgRunner.md)\<`TTable`, `TInput`, `TOutput`\> + +Defined in: [src/pg/PgRunner.ts:547](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L547) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `client` | [`PgClient`](PgClient.md)\<`Pool`\> | + +#### Returns + +[`PgRunner`](PgRunner.md)\<`TTable`, `TInput`, `TOutput`\> + +#### Overrides + +[`Runner`](Runner.md).[`constructor`](Runner.md#constructors) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `IS_WRITE` | `boolean` | If true, it's a write operation. | +| `op` | `string` | Operation name for logging purposes. | +| `maxBatchSize` | `number` | Maximum batch size for this type of operations. | +| `default` | `TOutput` | In case undefined is returned from batching, this value will be returned instead. | +| `name` | `string` | - | +| `constructor` | *typeof* [`PgRunner`](PgRunner.md) | The initial value of Object.prototype.constructor is the standard built-in Object constructor. | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | - | + +## Methods + +### runSingle() + +> `abstract` **runSingle**(`input`, `annotations`): `Promise`\<`undefined` \| `TOutput`\> + +Defined in: [src/abstract/Runner.ts:30](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L30) + +Method runSingle is to e.g. produce simple DB requests when we have only +one input to process, not many. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | `TInput` | +| `annotations` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md)[] | + +#### Returns + +`Promise`\<`undefined` \| `TOutput`\> + +#### Inherited from + +[`Runner`](Runner.md).[`runSingle`](Runner.md#runsingle) + +*** + +### runBatch()? + +> `abstract` `optional` **runBatch**(`inputs`, `annotations`): `Promise`\<`Map`\<`string`, `TOutput`\>\> + +Defined in: [src/abstract/Runner.ts:39](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L39) + +Typically issues complex queries with magic. If the method is not defined, +then the runner doesn't support batching, so only runSingle() will be used. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `inputs` | `Map`\<`string`, `TInput`\> | +| `annotations` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md)[] | + +#### Returns + +`Promise`\<`Map`\<`string`, `TOutput`\>\> + +#### Inherited from + +[`Runner`](Runner.md).[`runBatch`](Runner.md#runbatch) + +*** + +### key() + +> **key**(`_input`): `string` + +Defined in: [src/abstract/Runner.ts:77](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L77) + +Returns a batch-dedupping key for the input. By default, no dedupping is +performed (i.e. all inputs are processed individually and not collapsed +into one input; e.g. this is needed for inserts). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `_input` | `TInput` | + +#### Returns + +`string` + +#### Inherited from + +[`Runner`](Runner.md).[`key`](Runner.md#key) + +*** + +### clientQuery() + +> `protected` **clientQuery**\<`TOutput`\>(`sql`, `annotations`, `batchFactor`, `hints`?): `Promise`\<`TOutput`[]\> + +Defined in: [src/pg/PgRunner.ts:66](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L66) + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TOutput` *extends* `object` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `sql` | `string` | +| `annotations` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md)[] | +| `batchFactor` | `number` | +| `hints`? | [`Hints`](../type-aliases/Hints.md) | + +#### Returns + +`Promise`\<`TOutput`[]\> + +*** + +### fmt() + +> `protected` **fmt**(`template`, `args`): `string` + +Defined in: [src/pg/PgRunner.ts:105](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L105) + +Formats prefixes/suffixes of various compound SQL clauses. Don't use on +performance-critical path! + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `template` | `string` | +| `args` | \{ `fields`: [`FieldAliased`](../type-aliases/FieldAliased.md)\<`TTable`\>[]; `autos`: `Partial`\<`Record`\<`string`, `string`\>\>; `normalize`: `boolean`; \} | +| `args.fields`? | [`FieldAliased`](../type-aliases/FieldAliased.md)\<`TTable`\>[] | +| `args.autos`? | `Partial`\<`Record`\<`string`, `string`\>\> | +| `args.normalize`? | `boolean` | + +#### Returns + +`string` + +*** + +### escapeValue() + +> `protected` **escapeValue**(`field`, `value`): `string` + +Defined in: [src/pg/PgRunner.ts:197](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L197) + +Escapes a value at runtime using the codegen functions created above. We +use escapers table and the codegen for the following reasons: +1. We want to be sure that we know in advance, how to escape all table + fields (and not fail at runtime). +2. We want to make createEscapeCode() the single source of truth about + fields escaping, even at runtime. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `field` | [`Field`](../type-aliases/Field.md)\<`TTable`\> | +| `value` | `unknown` | + +#### Returns + +`string` + +*** + +### escapeField() + +> `protected` **escapeField**(`info`, `__namedParameters`): `string` + +Defined in: [src/pg/PgRunner.ts:213](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L213) + +Escapes field name identifier. +- In case it's a composite primary key, returns its `ROW(f1,f2,...)` + representation. +- A field may be aliased, e.g. if `{ field: "abc", alias: "$cas.abc" }` is + passed, then the returned value will be `"$cas.abc"`. Basically, `field` + name is used only to verify that such field is presented in the schema. +- If `autos` is passed, and the field is in it, returns the value of the + `autos` record for the field. This is used to inject an + autoInsert/autoUpdate SQL expression at the position of some field. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `info` | [`FieldAliased`](../type-aliases/FieldAliased.md)\<`TTable`\> | +| `__namedParameters` | \{ `withTable`: `string`; `normalize`: `boolean`; \} | +| `__namedParameters.withTable`? | `string` | +| `__namedParameters.normalize`? | `boolean` | + +#### Returns + +`string` + +*** + +### createWithBuilder() + +> `protected` **createWithBuilder**(`__namedParameters`): `object` + +Defined in: [src/pg/PgRunner.ts:251](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L251) + +Returns a newly created JS function which, when called with a row set, +returns the following SQL clause: + +``` +WITH rows(id, a, b, _key) AS (VALUES + ((NULL::tbl).id, (NULL::tbl).a, (NULL::tbl).b, 'k0'), + ('123', 'xyz', 'nn', 'kSome'), + ('456', 'abc', 'nn', 'kOther'), + ... +) +{suffix} +``` + +For composite primary key, its parts (fields) are always prepended. The set +of columns is passed in specs. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `__namedParameters` | \{ `fields`: readonly [`FieldAliased`](../type-aliases/FieldAliased.md)\<`TTable`\>[]; `skipSorting`: `boolean`; `suffix`: `string`; \} | +| `__namedParameters.fields` | readonly [`FieldAliased`](../type-aliases/FieldAliased.md)\<`TTable`\>[] | +| `__namedParameters.skipSorting`? | `boolean` | +| `__namedParameters.suffix` | `string` | + +#### Returns + +`object` + +##### prefix + +> **prefix**: `string` + +##### func() + +> **func**: (`entries`) => `string` + +###### Parameters + +| Parameter | Type | +| ------ | ------ | +| `entries` | `Iterable`\<\[`string`, `object`\], `any`, `any`\> | + +###### Returns + +`string` + +##### suffix + +> **suffix**: `string` + +*** + +### createValuesBuilder() + +> `protected` **createValuesBuilder**\<`TInput`\>(`__namedParameters`): `object` + +Defined in: [src/pg/PgRunner.ts:331](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L331) + +Returns a newly created JS function which, when called with a row set, +returns the following SQL clause: + +When called with withKey=true: + +``` +('123', 'xyz', 'nn', 'kSome'), +('456', 'abc', 'nn', 'kOther'), +``` + +When called with withKey=true, but with empty `fields`: + +``` +('kSome'), +('kOther'), +``` + +When called with withKey=false: + +``` +('123', 'xyz', 'nn'), +('456', 'abc', 'nn'), +``` + +The list of column names is passed in `fields`. + +When the builder func is called, the actual values for some field in a row +is extracted from the same-named prop of the row, but if `{ field, alias }` +object is passed in `fields` array, then the value is extracted from the +`alias` sub-prop of the row. This is used to e.g. access `row.$cas.blah` +value for a field named blah (in this case, `alias="$cas"`). + +Notice that either a simple primary key or a composite primary key columns +are always prepended to the list of values since it makes no sense to +generate VALUES clause without exact identification of the destination. + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TInput` *extends* `object` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `__namedParameters` | \{ `prefix`: `string`; `indent`: `string`; `fields`: readonly [`FieldAliased`](../type-aliases/FieldAliased.md)\<`TTable`\>[]; `withKey`: `boolean`; `skipSorting`: `boolean`; `suffix`: `string`; \} | +| `__namedParameters.prefix` | `string` | +| `__namedParameters.indent` | `string` | +| `__namedParameters.fields` | readonly [`FieldAliased`](../type-aliases/FieldAliased.md)\<`TTable`\>[] | +| `__namedParameters.withKey`? | `boolean` | +| `__namedParameters.skipSorting`? | `boolean` | +| `__namedParameters.suffix` | `string` | + +#### Returns + +`object` + +##### prefix + +> **prefix**: `string` + +##### func() + +> **func**: (`entries`) => `string` + +###### Parameters + +| Parameter | Type | +| ------ | ------ | +| `entries` | `Iterable`\<\[`string`, `TInput`\], `any`, `any`\> | + +###### Returns + +`string` + +##### suffix + +> **suffix**: `string` + +*** + +### createUpdateKVsBuilder() + +> `protected` **createUpdateKVsBuilder**(`fields`): (`input`, `literal`?) => `string` + +Defined in: [src/pg/PgRunner.ts:404](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L404) + +Returns a newly created JS function which, when called with an object, +returns the following SQL clause: + +id='123', a='xyz', b='nnn' [, {literal}] + +The set of columns is passed in specs, all other columns are ignored. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `fields` | [`Field`](../type-aliases/Field.md)\<`TTable`\>[] | + +#### Returns + +`Function` + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | `object` | +| `literal`? | [`Literal`](../type-aliases/Literal.md) | + +##### Returns + +`string` + +*** + +### createOneOfBuilder() + +> `protected` **createOneOfBuilder**(`field`, `fieldValCode`): (`values`) => `string` + +Defined in: [src/pg/PgRunner.ts:432](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L432) + +Prefers to do utilize createAnyBuilder() if it can (i.e. build +a=ANY('{...}') clause). Otherwise, builds an IN(...) clause. + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `field` | [`Field`](../type-aliases/Field.md)\<`TTable`\> | `undefined` | +| `fieldValCode` | `string` | `"$value"` | + +#### Returns + +`Function` + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| `values` | `Iterable`\<`unknown`, `any`, `any`\> | + +##### Returns + +`string` + +*** + +### createWhereBuildersFieldsEq() + +> `protected` **createWhereBuildersFieldsEq**\<`TInput`\>(`args`): `object` + +Defined in: [src/pg/PgRunner.ts:466](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L466) + +Given a list of fields, returns two builders: + +1. "Optimized": a newly created JS function which, when called with a row + set, returns one the following SQL clauses: + +``` +WHERE (field1, field2) IN(VALUES + ((NULL::tbl).field1, (NULL::tbl).field2), + ('aa', 'bb'), + ('cc', 'dd')) + +or + +WHERE (field1='a' AND field2='b' AND field3 IN('a', 'b', 'c', ...)) OR (...) + ^^^^^^^^^^prefix^^^^^^^^^ ^^^^^^^^ins^^^^^^^ +``` + +2. "Plain": the last one builder mentioned above (good to always use for + non-batched queries for instance). + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TInput` *extends* `object` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `args` | \{ `prefix`: `string`; `fields`: readonly [`Field`](../type-aliases/Field.md)\<`TTable`\>[]; `suffix`: `string`; \} | +| `args.prefix` | `string` | +| `args.fields` | readonly [`Field`](../type-aliases/Field.md)\<`TTable`\>[] | +| `args.suffix` | `string` | + +#### Returns + +`object` + +##### plain + +> **plain**: `object` + +###### plain.prefix + +> **prefix**: `string` + +###### plain.func() + +> **func**: (`inputs`) => `string` + +###### Parameters + +| Parameter | Type | +| ------ | ------ | +| `inputs` | `Iterable`\<\[`string`, `TInput`\], `any`, `any`\> | + +###### Returns + +`string` + +###### plain.suffix + +> **suffix**: `string` + +##### optimized + +> **optimized**: `object` + +###### optimized.prefix + +> **prefix**: `string` + +###### optimized.func() + +> **func**: (`inputs`) => `string` + +###### Parameters + +| Parameter | Type | +| ------ | ------ | +| `inputs` | `Iterable`\<\[`string`, `TInput`\], `any`, `any`\> | + +###### Returns + +`string` + +###### optimized.suffix + +> **suffix**: `string` + +*** + +### createWhereBuilder() + +> `protected` **createWhereBuilder**(`__namedParameters`): `object` + +Defined in: [src/pg/PgRunner.ts:501](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L501) + +Returns a newly created JS function which, when called with a Where object, +returns the generated SQL WHERE clause. + +- The building is relatively expensive, since it traverses the Where object + at run-time and doesn't know the shape beforehand. +- If the Where object is undefined, skips the entire WHERE clause. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `__namedParameters` | \{ `prefix`: `string`; `suffix`: `string`; \} | +| `__namedParameters.prefix` | `string` | +| `__namedParameters.suffix` | `string` | + +#### Returns + +`object` + +##### prefix + +> **prefix**: `string` + +##### func() + +> **func**: (`where`) => `string` + +###### Parameters + +| Parameter | Type | +| ------ | ------ | +| `where` | [`Where`](../type-aliases/Where.md)\<`TTable`\> | + +###### Returns + +`string` + +##### suffix + +> **suffix**: `string` + +*** + +### addPK() + +> `protected` **addPK**(`fields`, `mode`): `string`[] + +Defined in: [src/pg/PgRunner.ts:536](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L536) + +Prepends or appends a primary key to the list of fields. In case the +primary key is plain (i.e. "id" field), it's just added as a field; +otherwise, the unique key fields are added. + +For INSERT/UPSERT operations, we want to append the primary key, since it's +often types pre-generated as a random-looking value. In many places, we +sort batched lists of rows before e.g. inserting them, so we order them by +their natural data order which prevents deadlocks on unique key conflict +when multiple concurrent transactions try to insert the same set of rows in +different order ("while inserting index tuple"). + +For UPDATE operations though, we want to prepend the primary key, to make +sure we run batched updates in the same order in multiple concurrent +transactions. This lowers the chances of deadlocks too. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `fields` | readonly [`Field`](../type-aliases/Field.md)\<`TTable`\>[] | +| `mode` | `"prepend"` \| `"append"` | + +#### Returns + +`string`[] + +*** + +### delayForSingleQueryRetryOnError() + +> **delayForSingleQueryRetryOnError**(`e`): `number` \| `"immediate_retry"` \| `"no_retry"` + +Defined in: [src/pg/PgRunner.ts:569](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L569) + +If the single query's error needs to be retried (e.g. it's a deadlock +error), returns the number of milliseconds to wait before retrying. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `e` | `unknown` | + +#### Returns + +`number` \| `"immediate_retry"` \| `"no_retry"` + +#### Overrides + +[`Runner`](Runner.md).[`delayForSingleQueryRetryOnError`](Runner.md#delayforsinglequeryretryonerror) + +*** + +### shouldDebatchOnError() + +> **shouldDebatchOnError**(`e`): `boolean` + +Defined in: [src/pg/PgRunner.ts:582](https://github.com/clickup/ent-framework/blob/master/src/pg/PgRunner.ts#L582) + +If this method returns true for an error object, the batch is split back +into sub-queries, they are executed individually, and then the response of +each query is delivered to each caller individually. Used mostly for e.g. +batch-deadlock errors or for FK constraint errors when it makes sense to +retry other members of the batch and not fail it entirely hurting other +innocent queries. + +We can do this, because we know that if some transaction is aborted, it's +always safe to retry it. (If we're not sure about the transaction, e.g. the +Client doesn't support transactions at all, then the method should return +false.) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `e` | `unknown` | + +#### Returns + +`boolean` + +#### Overrides + +[`Runner`](Runner.md).[`shouldDebatchOnError`](Runner.md#shoulddebatchonerror) diff --git a/docs/classes/PgSchema.md b/docs/classes/PgSchema.md new file mode 100644 index 0000000..d860deb --- /dev/null +++ b/docs/classes/PgSchema.md @@ -0,0 +1,333 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgSchema + +# Class: PgSchema\ + +Defined in: [src/pg/PgSchema.ts:27](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L27) + +Schema is like a "table" in some database (sharded, but it's beyond the scope +of Schema). It is also a factory of Query: it knows how to build runnable +Query objects. This 2nd role is database engine specific (e.g. there might be +PgSchema, RedisSchema etc.): such composition simplifies the code and lowers +the number of abstractions. + +The set of supported Queries is opinionated and is crafted carefully to +support the minimal possible list of primitives, but at the same time, be not +too limited in the queries the DB engine can execute. + +## Extends + +- [`Schema`](Schema.md)\<`TTable`, `TUniqueKey`\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | + +## Constructors + +### new PgSchema() + +> **new PgSchema**\<`TTable`, `TUniqueKey`\>(`name`, `table`, `uniqueKey`): [`PgSchema`](PgSchema.md)\<`TTable`, `TUniqueKey`\> + +Defined in: [src/abstract/Schema.ts:116](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L116) + +Used in e.g. inverses. This casts this.constructor to SchemaClass with all +static methods and `new` semantic (TS doesn't do it by default; for TS, +x.constructor is Function). + +#### Parameters + +| Parameter | Type | Description | +| ------ | ------ | ------ | +| `name` | `string` | For relational databases, it's likely a table name. | +| `table` | `TTable` | Structure of the table. | +| `uniqueKey` | `TUniqueKey` | Fields which the native unique key consists of (if any). | + +#### Returns + +[`PgSchema`](PgSchema.md)\<`TTable`, `TUniqueKey`\> + +#### Inherited from + +[`Schema`](Schema.md).[`constructor`](Schema.md#constructors) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `constructor` | [`SchemaClass`](../interfaces/SchemaClass.md) | Used in e.g. inverses. This casts this.constructor to SchemaClass with all static methods and `new` semantic (TS doesn't do it by default; for TS, x.constructor is Function). | +| `name` | `string` | For relational databases, it's likely a table name. | +| `table` | `TTable` | Structure of the table. | +| `uniqueKey` | `TUniqueKey` | Fields which the native unique key consists of (if any). | + +## Methods + +### idGen() + +> **idGen**(): [`Query`](../interfaces/Query.md)\<`string`\> + +Defined in: [src/pg/PgSchema.ts:31](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L31) + +Generates a new ID for the row. Used when e.g. there is a beforeInsert +trigger on the Ent which needs to know the ID beforehand. + +#### Returns + +[`Query`](../interfaces/Query.md)\<`string`\> + +#### Overrides + +[`Schema`](Schema.md).[`idGen`](Schema.md#idgen) + +*** + +### insert() + +> **insert**(`input`): [`Query`](../interfaces/Query.md)\<`null` \| `string`\> + +Defined in: [src/pg/PgSchema.ts:35](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L35) + +Creates a new row. Returns null if the row violates some unique key +constraint, otherwise returns the row ID. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`null` \| `string`\> + +#### Overrides + +[`Schema`](Schema.md).[`insert`](Schema.md#insert) + +*** + +### upsert() + +> **upsert**(`input`): [`Query`](../interfaces/Query.md)\<`string`\> + +Defined in: [src/pg/PgSchema.ts:39](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L39) + +Upserts a row. Always returns the row ID. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`string`\> + +#### Overrides + +[`Schema`](Schema.md).[`upsert`](Schema.md#upsert) + +*** + +### update() + +> **update**(`id`, `input`): [`Query`](../interfaces/Query.md)\<`boolean`\> + +Defined in: [src/pg/PgSchema.ts:43](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L43) + +Updates one single row by its ID. Returns true if it actually existed. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | +| `input` | [`UpdateInput`](../type-aliases/UpdateInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`boolean`\> + +#### Overrides + +[`Schema`](Schema.md).[`update`](Schema.md#update) + +*** + +### delete() + +> **delete**(`id`): [`Query`](../interfaces/Query.md)\<`boolean`\> + +Defined in: [src/pg/PgSchema.ts:47](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L47) + +Deletes a row by id. Returns true if it actually existed. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`boolean`\> + +#### Overrides + +[`Schema`](Schema.md).[`delete`](Schema.md#delete) + +*** + +### load() + +> **load**(`id`): [`Query`](../interfaces/Query.md)\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +Defined in: [src/pg/PgSchema.ts:51](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L51) + +"Load" family of methods means that we load exactly one row. This one +returns a row by its ID or null if it's not found. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +#### Overrides + +[`Schema`](Schema.md).[`load`](Schema.md#load) + +*** + +### loadBy() + +> **loadBy**(`input`): [`Query`](../interfaces/Query.md)\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +Defined in: [src/pg/PgSchema.ts:55](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L55) + +Loads one single row by its unique key ("by" denotes that it's based on an +unique key, not on an ID). Returns null if it's not found. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TUniqueKey`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +#### Overrides + +[`Schema`](Schema.md).[`loadBy`](Schema.md#loadby) + +*** + +### selectBy() + +> **selectBy**(`input`): [`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +Defined in: [src/pg/PgSchema.ts:59](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L59) + +"Select" family of methods means that we load multiple rows ("by" denotes +that it's based on an unique key, not on an arbitrary query). This one +returns all rows whose unique key prefix matches the input. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`SelectByInput`](../type-aliases/SelectByInput.md)\<`TTable`, `TUniqueKey`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +#### Overrides + +[`Schema`](Schema.md).[`selectBy`](Schema.md#selectby) + +*** + +### select() + +> **select**(`input`): [`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +Defined in: [src/pg/PgSchema.ts:65](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L65) + +Returns all rows matching an arbitrary query. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`SelectInput`](../type-aliases/SelectInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +#### Overrides + +[`Schema`](Schema.md).[`select`](Schema.md#select) + +*** + +### count() + +> **count**(`input`): [`Query`](../interfaces/Query.md)\<`number`\> + +Defined in: [src/pg/PgSchema.ts:69](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L69) + +Returns the number of rows matching an arbitrary query. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`CountInput`](../type-aliases/CountInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`number`\> + +#### Overrides + +[`Schema`](Schema.md).[`count`](Schema.md#count) + +*** + +### exists() + +> **exists**(`input`): [`Query`](../interfaces/Query.md)\<`boolean`\> + +Defined in: [src/pg/PgSchema.ts:73](https://github.com/clickup/ent-framework/blob/master/src/pg/PgSchema.ts#L73) + +An optimized version of count() for the cases where we only need to know +whether at least one row exists, and don't need a precise count. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`ExistsInput`](../type-aliases/ExistsInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`boolean`\> + +#### Overrides + +[`Schema`](Schema.md).[`exists`](Schema.md#exists) diff --git a/docs/classes/PgShardNamer.md b/docs/classes/PgShardNamer.md new file mode 100644 index 0000000..c4960a6 --- /dev/null +++ b/docs/classes/PgShardNamer.md @@ -0,0 +1,123 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgShardNamer + +# Class: PgShardNamer + +Defined in: [src/pg/PgShardNamer.ts:9](https://github.com/clickup/ent-framework/blob/master/src/pg/PgShardNamer.ts#L9) + +ShardNamer implementation for PG. + +## Extends + +- [`ShardNamer`](ShardNamer.md) + +## Constructors + +### new PgShardNamer() + +> **new PgShardNamer**(`options`): [`PgShardNamer`](PgShardNamer.md) + +Defined in: [src/abstract/ShardNamer.ts:34](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L34) + +Initializes an instance of ShardNamer. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`ShardNamerOptions`](../interfaces/ShardNamerOptions.md) | + +#### Returns + +[`PgShardNamer`](PgShardNamer.md) + +#### Inherited from + +[`ShardNamer`](ShardNamer.md).[`constructor`](ShardNamer.md#constructors) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `shardNoPadLen` | `number` | Number of decimal digits in an ID allocated for Shard number. Calculated dynamically based on `ShardNamerOptions#nameFormat` (e.g. for "sh%04d", it will be 4 since it expands to "sh0012"). | +| `options` | [`ShardNamerOptions`](../interfaces/ShardNamerOptions.md) | - | + +## Methods + +### shardNoByName() + +> **shardNoByName**(`name`): `null` \| `number` + +Defined in: [src/abstract/ShardNamer.ts:46](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L46) + +Converts a Shard name to Shard number. Returns null if it's not a correct +Shard name. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `name` | `string` | + +#### Returns + +`null` \| `number` + +#### Inherited from + +[`ShardNamer`](ShardNamer.md).[`shardNoByName`](ShardNamer.md#shardnobyname) + +*** + +### shardNameByNo() + +> **shardNameByNo**(`no`): `string` + +Defined in: [src/abstract/ShardNamer.ts:58](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L58) + +Builds the Shard name (e.g. for PG, "schema name") by Shard number using +`ShardNamerOptions#nameFormat`. + +E.g. nameFormat="sh%04d" generates names like "sh0042". + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `no` | `number` | + +#### Returns + +`string` + +#### Inherited from + +[`ShardNamer`](ShardNamer.md).[`shardNameByNo`](ShardNamer.md#shardnamebyno) + +*** + +### shardNoByID() + +> **shardNoByID**(`id`): `number` + +Defined in: [src/pg/PgShardNamer.ts:14](https://github.com/clickup/ent-framework/blob/master/src/pg/PgShardNamer.ts#L14) + +Synchronously extracts Shard number from an ID. Can also extract from PG +composite rows (to support composite IDs). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | + +#### Returns + +`number` + +#### Overrides + +[`ShardNamer`](ShardNamer.md).[`shardNoByID`](ShardNamer.md#shardnobyid) diff --git a/docs/classes/PgTimelineStorage.md b/docs/classes/PgTimelineStorage.md new file mode 100644 index 0000000..e7eceab --- /dev/null +++ b/docs/classes/PgTimelineStorage.md @@ -0,0 +1,119 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgTimelineStorage + +# Class: PgTimelineStorage + +Defined in: [src/pg/PgTimelineStorage.ts:46](https://github.com/clickup/ent-framework/blob/master/src/pg/PgTimelineStorage.ts#L46) + +An append-only (with compaction) timeline storage for PG. The timelines are +always appended to the table, but from time to time, when the number of +chunks per principal exceeds the limit, the timelines are read back, +compacted and written back as a single row. This is race condition safe, +since timelines merging is an associative and idempotent operation, i.e. +(T1+T2)+T3 == T1+(T2+T3); in the worst case, we'll just have slightly +suboptimal timeline rows. + +The expected table schema is: +``` +CREATE UNLOGGED TABLE timelines( + id bigserial PRIMARY KEY, + principal text NOT NULL, + data text NOT NULL, + created_at timestamptz NOT NULL +); +CREATE INDEX timelines_principal ON timelines (principal); +``` + +Notes: +1. Index on `principal` must be non-unique, since there may be multiple + records with the same value. +2. The `id` field should have sequential auto-increment, since it's used for + garbage collection. +3. The table must exist in all microshards (including global shard). + +## Extends + +- [`TimelineStorage`](TimelineStorage.md) + +## Constructors + +### new PgTimelineStorage() + +> **new PgTimelineStorage**(`options`): [`PgTimelineStorage`](PgTimelineStorage.md) + +Defined in: [src/pg/PgTimelineStorage.ts:62](https://github.com/clickup/ent-framework/blob/master/src/pg/PgTimelineStorage.ts#L62) + +Initializes an instance of PgTimelineStorage. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`PgTimelineStorageOptions`](../interfaces/PgTimelineStorageOptions.md) | + +#### Returns + +[`PgTimelineStorage`](PgTimelineStorage.md) + +#### Overrides + +[`TimelineStorage`](TimelineStorage.md).[`constructor`](TimelineStorage.md#constructors) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`PgTimelineStorageOptions`](../interfaces/PgTimelineStorageOptions.md)\>\> | Default values for the constructor options. | +| `options` | `Required`\<[`PgTimelineStorageOptions`](../interfaces/PgTimelineStorageOptions.md)\> | PgTimelineStorage configuration options. | + +## Methods + +### load() + +> **load**(`principal`): `Promise`\<`string`[]\> + +Defined in: [src/pg/PgTimelineStorage.ts:72](https://github.com/clickup/ent-framework/blob/master/src/pg/PgTimelineStorage.ts#L72) + +Loads the timelines from the storage for a given principal. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `principal` | `string` | + +#### Returns + +`Promise`\<`string`[]\> + +#### Overrides + +[`TimelineStorage`](TimelineStorage.md).[`load`](TimelineStorage.md#load) + +*** + +### save() + +> **save**(`principal`, `dataStr`): `Promise`\<`void`\> + +Defined in: [src/pg/PgTimelineStorage.ts:81](https://github.com/clickup/ent-framework/blob/master/src/pg/PgTimelineStorage.ts#L81) + +Saves the timelines in the storage for a given principal. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `principal` | `string` | +| `dataStr` | `string` | + +#### Returns + +`Promise`\<`void`\> + +#### Overrides + +[`TimelineStorage`](TimelineStorage.md).[`save`](TimelineStorage.md#save) diff --git a/docs/classes/QueryBase.md b/docs/classes/QueryBase.md new file mode 100644 index 0000000..e220a68 --- /dev/null +++ b/docs/classes/QueryBase.md @@ -0,0 +1,107 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / QueryBase + +# Class: `abstract` QueryBase\ + +Defined in: [src/abstract/QueryBase.ts:15](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L15) + +A convenient base class for most (but not all) of the queries, where the +Runner instance is the same for different query input shapes. If the query +doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we +have separate Runner instances for separate set of updated fields), a Query +is used directly instead. + +## Extended by + +- [`PgQueryCount`](PgQueryCount.md) +- [`PgQueryDelete`](PgQueryDelete.md) +- [`PgQueryDeleteWhere`](PgQueryDeleteWhere.md) +- [`PgQueryExists`](PgQueryExists.md) +- [`PgQueryIDGen`](PgQueryIDGen.md) +- [`PgQueryInsert`](PgQueryInsert.md) +- [`PgQueryLoad`](PgQueryLoad.md) +- [`PgQueryLoadBy`](PgQueryLoadBy.md) +- [`PgQuerySelect`](PgQuerySelect.md) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TInput` | +| `TOutput` | +| `TClient` *extends* [`Client`](Client.md) | + +## Implements + +- [`Query`](../interfaces/Query.md)\<`TOutput`\> + +## Constructors + +### new QueryBase() + +> **new QueryBase**\<`TTable`, `TInput`, `TOutput`, `TClient`\>(`schema`, `input`): [`QueryBase`](QueryBase.md)\<`TTable`, `TInput`, `TOutput`, `TClient`\> + +Defined in: [src/abstract/QueryBase.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L27) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | `TInput` | + +#### Returns + +[`QueryBase`](QueryBase.md)\<`TTable`, `TInput`, `TOutput`, `TClient`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `schema` | [`Schema`](Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `input` | `TInput` | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryBase.ts:32](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L32) + +##### Returns + +`boolean` + +#### Implementation of + +[`Query`](../interfaces/Query.md).[`IS_WRITE`](../interfaces/Query.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`TOutput`\> + +Defined in: [src/abstract/QueryBase.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryBase.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | `TClient` | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`TOutput`\> + +#### Implementation of + +[`Query`](../interfaces/Query.md).[`run`](../interfaces/Query.md#run) diff --git a/docs/classes/QueryCache.md b/docs/classes/QueryCache.md new file mode 100644 index 0000000..2e651c4 --- /dev/null +++ b/docs/classes/QueryCache.md @@ -0,0 +1,147 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / QueryCache + +# Class: QueryCache + +Defined in: [src/ent/QueryCache.ts:26](https://github.com/clickup/ent-framework/blob/master/src/ent/QueryCache.ts#L26) + +Caches Ents loaded by a particular VC. I.e. the same query running for the +same VC twice will quickly return the same Ents. This is typically enabled on +web servers only, to deliver the fastest UI response. + +## Constructors + +### new QueryCache() + +> **new QueryCache**(`vc`): [`QueryCache`](QueryCache.md) + +Defined in: [src/ent/QueryCache.ts:38](https://github.com/clickup/ent-framework/blob/master/src/ent/QueryCache.ts#L38) + +Creates the QueryCache object. It enable caching only if VCWithQueryCache +was manually added to the VC by the user, otherwise caching is a no-op. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | + +#### Returns + +[`QueryCache`](QueryCache.md) + +## Properties + +| Property | Type | +| ------ | ------ | +| `whyOff?` | `string` | + +## Methods + +### set() + +> **set**(`EntClass`, `op`, `key`, `value`): `this` + +Defined in: [src/ent/QueryCache.ts:59](https://github.com/clickup/ent-framework/blob/master/src/ent/QueryCache.ts#L59) + +Saves a Promise to the cache slot for `op`. If this Promise rejects, the +slot will automatically be cleared (we don't cache rejected Promises to not +have a risk of caching a transient DB error). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `EntClass` | [`AnyClass`](../type-aliases/AnyClass.md) | +| `op` | `"loadNullable"` \| `"loadByNullable"` \| `"selectBy"` \| `"select"` \| `"count"` \| `"exists"` | +| `key` | `string` | +| `value` | `undefined` \| `Promise`\<`unknown`\> | + +#### Returns + +`this` + +*** + +### delete() + +> **delete**(`EntClass`, `ops`, `key`?): `this` + +Defined in: [src/ent/QueryCache.ts:95](https://github.com/clickup/ent-framework/blob/master/src/ent/QueryCache.ts#L95) + +Deletes cache slots or keys for an Ent. If key is null, skips the deletion. +If key is undefined (i.e. not passed), then deletes all slots. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `EntClass` | [`AnyClass`](../type-aliases/AnyClass.md) | +| `ops` | readonly (`"loadNullable"` \| `"loadByNullable"` \| `"selectBy"` \| `"select"` \| `"count"` \| `"exists"`)[] | +| `key`? | `null` \| `string` | + +#### Returns + +`this` + +*** + +### get() + +> **get**\<`TValue`\>(`EntClass`, `op`, `key`): `undefined` \| `Promise`\<`TValue`\> + +Defined in: [src/ent/QueryCache.ts:121](https://github.com/clickup/ent-framework/blob/master/src/ent/QueryCache.ts#L121) + +This method is non-async on intent. We store Promises in the cache, not end +values, because we want the code to join awaiting an ongoing operation in +case it's inflight already. + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TValue` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `EntClass` | [`AnyClass`](../type-aliases/AnyClass.md) | +| `op` | `"loadNullable"` \| `"loadByNullable"` \| `"selectBy"` \| `"select"` \| `"count"` \| `"exists"` | +| `key` | `string` | + +#### Returns + +`undefined` \| `Promise`\<`TValue`\> + +*** + +### through() + +> **through**\<`TValue`\>(`EntClass`, `op`, `key`, `creator`): `Promise`\<`TValue`\> + +Defined in: [src/ent/QueryCache.ts:137](https://github.com/clickup/ent-framework/blob/master/src/ent/QueryCache.ts#L137) + +Read-through caching pattern. + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TValue` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `EntClass` | [`AnyClass`](../type-aliases/AnyClass.md) | +| `op` | `"loadNullable"` \| `"loadByNullable"` \| `"selectBy"` \| `"select"` \| `"count"` \| `"exists"` | +| `key` | `string` | +| `creator` | () => `Promise`\<`TValue`\> | + +#### Returns + +`Promise`\<`TValue`\> diff --git a/docs/classes/QueryPing.md b/docs/classes/QueryPing.md new file mode 100644 index 0000000..e4ad7ab --- /dev/null +++ b/docs/classes/QueryPing.md @@ -0,0 +1,80 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / QueryPing + +# Class: QueryPing + +Defined in: [src/abstract/QueryPing.ts:8](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryPing.ts#L8) + +A helper Query which delegates to Client#ping(execTimeMs). + +## Implements + +- [`Query`](../interfaces/Query.md)\<`void`\> + +## Constructors + +### new QueryPing() + +> **new QueryPing**(`input`): [`QueryPing`](QueryPing.md) + +Defined in: [src/abstract/QueryPing.ts:9](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryPing.ts#L9) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | `Omit`\<[`ClientPingInput`](../interfaces/ClientPingInput.md), `"annotation"`\> | + +#### Returns + +[`QueryPing`](QueryPing.md) + +## Properties + +| Property | Type | +| ------ | ------ | +| `input` | `Omit`\<[`ClientPingInput`](../interfaces/ClientPingInput.md), `"annotation"`\> | + +## Accessors + +### IS\_WRITE + +#### Get Signature + +> **get** **IS\_WRITE**(): `boolean` + +Defined in: [src/abstract/QueryPing.ts:11](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryPing.ts#L11) + +##### Returns + +`boolean` + +#### Implementation of + +[`Query`](../interfaces/Query.md).[`IS_WRITE`](../interfaces/Query.md#is_write) + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`void`\> + +Defined in: [src/abstract/QueryPing.ts:15](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryPing.ts#L15) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`Client`](Client.md) | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | + +#### Returns + +`Promise`\<`void`\> + +#### Implementation of + +[`Query`](../interfaces/Query.md).[`run`](../interfaces/Query.md#run) diff --git a/docs/classes/Require.md b/docs/classes/Require.md new file mode 100644 index 0000000..a0faae1 --- /dev/null +++ b/docs/classes/Require.md @@ -0,0 +1,77 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Require + +# Class: Require\ + +Defined in: [src/ent/rules/Require.ts:11](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Require.ts#L11) + +Returns TOLERATE if the predicate succeeds, otherwise DENY. +- Used mostly for write permission checks. +- This rule may still throw an exception if it's a wild one (i.e. not derived + from EntAccessError). + +## Extends + +- [`Rule`](Rule.md)\<`TInput`\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` *extends* `object` | + +## Constructors + +### new Require() + +> **new Require**\<`TInput`\>(`predicate`): [`Require`](Require.md)\<`TInput`\> + +Defined in: [src/ent/rules/Rule.ts:43](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Rule.ts#L43) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `predicate` | [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> \| (`vc`, `input`) => `boolean` \| `Promise`\<`boolean`\> | + +#### Returns + +[`Require`](Require.md)\<`TInput`\> + +#### Inherited from + +[`Rule`](Rule.md).[`constructor`](Rule.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `_TAG` | `"Require"` | +| `predicate` | [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> | +| `name` | `string` | + +## Methods + +### evaluate() + +> **evaluate**(`vc`, `input`): `Promise`\<[`RuleResult`](../interfaces/RuleResult.md)\> + +Defined in: [src/ent/rules/Require.ts:14](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Require.ts#L14) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `input` | `TInput` | + +#### Returns + +`Promise`\<[`RuleResult`](../interfaces/RuleResult.md)\> + +#### Overrides + +[`Rule`](Rule.md).[`evaluate`](Rule.md#evaluate) diff --git a/docs/classes/RowIs.md b/docs/classes/RowIs.md new file mode 100644 index 0000000..2405797 --- /dev/null +++ b/docs/classes/RowIs.md @@ -0,0 +1,97 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / RowIs + +# Class: RowIs\ + +Defined in: [src/ent/predicates/RowIs.ts:37](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/RowIs.ts#L37) + +Checks that the validator function returns true for the entire row. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TRow` | + +## Implements + +- [`AbstractIs`](../interfaces/AbstractIs.md)\<`TRow`\> + +## Constructors + +### new RowIs() + +> **new RowIs**\<`TRow`\>(`validator`, `message`): [`RowIs`](RowIs.md)\<`TRow`\> + +Defined in: [src/ent/predicates/RowIs.ts:46](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/RowIs.ts#L46) + +Manual validator. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `validator` | [`RowIsValidatorPlain`](../type-aliases/RowIsValidatorPlain.md)\<`TRow`\> | +| `message` | `string` | + +#### Returns + +[`RowIs`](RowIs.md)\<`TRow`\> + +### new RowIs() + +> **new RowIs**\<`TRow`\>(`validator`): [`RowIs`](RowIs.md)\<`TRow`\> + +Defined in: [src/ent/predicates/RowIs.ts:51](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/RowIs.ts#L51) + +Rich validator, like Standard Schema (https://standardschema.dev) or Zod. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `validator` | [`RowIsValidatorZodSafeParse`](../type-aliases/RowIsValidatorZodSafeParse.md)\<`TRow`\> \| [`RowIsValidatorStandardSchemaV1`](../type-aliases/RowIsValidatorStandardSchemaV1.md)\<`TRow`\> | + +#### Returns + +[`RowIs`](RowIs.md)\<`TRow`\> + +## Properties + +| Property | Type | Default value | Description | +| ------ | ------ | ------ | ------ | +| `name` | `string` | `undefined` | - | +| `field` | `null` | `null` | The field this validation predicate is related to (null means that it applies to the entire Ent). | +| `message` | `null` \| `string` | `undefined` | In case the predicate returns false or doesn't provide error messages by throwing EntValidationError, this message will be used. When message is null, it means that we expect the validator to return detailed information about each field errored (e.g. ValidatorStandardSchemaResult). | +| `validator` | [`RowIsValidatorPlain`](../type-aliases/RowIsValidatorPlain.md)\<`TRow`\> \| [`RowIsValidatorZodSafeParse`](../type-aliases/RowIsValidatorZodSafeParse.md)\<`TRow`\> \| [`RowIsValidatorStandardSchemaV1`](../type-aliases/RowIsValidatorStandardSchemaV1.md)\<`TRow`\> | `undefined` | - | + +## Methods + +### check() + +> **check**(`vc`, `row`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/RowIs.ts:75](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/RowIs.ts#L75) + +Returns true if validation succeeds. Returns false if it wants the client +to use this.message as a validation failure response. Throws an instance of +EntValidationError when it needs to deliver the detailed error messages +about multiple fields. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | `TRow` | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`AbstractIs`](../interfaces/AbstractIs.md).[`check`](../interfaces/AbstractIs.md#check) diff --git a/docs/classes/Rule.md b/docs/classes/Rule.md new file mode 100644 index 0000000..0845168 --- /dev/null +++ b/docs/classes/Rule.md @@ -0,0 +1,80 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Rule + +# Class: `abstract` Rule\ + +Defined in: [src/ent/rules/Rule.ts:37](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Rule.ts#L37) + +A base class which can e.g. accept not only a predicate, but also a plain JS +lambda function as a predicate. Also has a logic of "glueing" the rule name +with the predicate name. + +Each Rule must either: +- throw (or return DENY) if it disallows access immediately, +- return ALLOW if the access is granted (so no other rules will run), +- return TOLERATE if it's okay with the row, but wants others' votes too, +- or return SKIP to fully delegate the decision to the next rule. + +See more comments in rules.ts. + +Each rule carries a predicate which it calls and then decides, how to +interpret the result. + +## Extended by + +- [`AllowIf`](AllowIf.md) +- [`DenyIf`](DenyIf.md) +- [`Require`](Require.md) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` | + +## Constructors + +### new Rule() + +> **new Rule**\<`TInput`\>(`predicate`): [`Rule`](Rule.md)\<`TInput`\> + +Defined in: [src/ent/rules/Rule.ts:43](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Rule.ts#L43) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `predicate` | [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> \| (`vc`, `input`) => `boolean` \| `Promise`\<`boolean`\> | + +#### Returns + +[`Rule`](Rule.md)\<`TInput`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `predicate` | [`Predicate`](../interfaces/Predicate.md)\<`TInput`\> | +| `name` | `string` | + +## Methods + +### evaluate() + +> `abstract` **evaluate**(`vc`, `input`): `Promise`\<[`RuleResult`](../interfaces/RuleResult.md)\> + +Defined in: [src/ent/rules/Rule.ts:41](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Rule.ts#L41) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `input` | `TInput` | + +#### Returns + +`Promise`\<[`RuleResult`](../interfaces/RuleResult.md)\> diff --git a/docs/classes/Runner.md b/docs/classes/Runner.md new file mode 100644 index 0000000..e5bc224 --- /dev/null +++ b/docs/classes/Runner.md @@ -0,0 +1,170 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Runner + +# Class: `abstract` Runner\ + +Defined in: [src/abstract/Runner.ts:9](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L9) + +Knows how to translate individual strongly typed requests into DB language +and how to parse the result back. + +## Extended by + +- [`PgRunner`](PgRunner.md) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` | +| `TOutput` | + +## Constructors + +### new Runner() + +> **new Runner**\<`TInput`, `TOutput`\>(`name`): [`Runner`](Runner.md)\<`TInput`, `TOutput`\> + +Defined in: [src/abstract/Runner.ts:70](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L70) + +Parameter `name` is typically a table name. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `name` | `string` | + +#### Returns + +[`Runner`](Runner.md)\<`TInput`, `TOutput`\> + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `IS_WRITE` | `boolean` | If true, it's a write operation. | +| `op` | `string` | Operation name for logging purposes. | +| `maxBatchSize` | `number` | Maximum batch size for this type of operations. | +| `default` | `TOutput` | In case undefined is returned from batching, this value will be returned instead. | +| `name` | `string` | - | + +## Methods + +### runSingle() + +> `abstract` **runSingle**(`input`, `annotations`): `Promise`\<`undefined` \| `TOutput`\> + +Defined in: [src/abstract/Runner.ts:30](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L30) + +Method runSingle is to e.g. produce simple DB requests when we have only +one input to process, not many. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | `TInput` | +| `annotations` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md)[] | + +#### Returns + +`Promise`\<`undefined` \| `TOutput`\> + +*** + +### runBatch()? + +> `abstract` `optional` **runBatch**(`inputs`, `annotations`): `Promise`\<`Map`\<`string`, `TOutput`\>\> + +Defined in: [src/abstract/Runner.ts:39](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L39) + +Typically issues complex queries with magic. If the method is not defined, +then the runner doesn't support batching, so only runSingle() will be used. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `inputs` | `Map`\<`string`, `TInput`\> | +| `annotations` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md)[] | + +#### Returns + +`Promise`\<`Map`\<`string`, `TOutput`\>\> + +*** + +### delayForSingleQueryRetryOnError() + +> `abstract` **delayForSingleQueryRetryOnError**(`error`): `number` \| `"immediate_retry"` \| `"no_retry"` + +Defined in: [src/abstract/Runner.ts:48](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L48) + +If the single query's error needs to be retried (e.g. it's a deadlock +error), returns the number of milliseconds to wait before retrying. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `error` | `unknown` | + +#### Returns + +`number` \| `"immediate_retry"` \| `"no_retry"` + +*** + +### shouldDebatchOnError() + +> `abstract` **shouldDebatchOnError**(`error`): `boolean` + +Defined in: [src/abstract/Runner.ts:65](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L65) + +If this method returns true for an error object, the batch is split back +into sub-queries, they are executed individually, and then the response of +each query is delivered to each caller individually. Used mostly for e.g. +batch-deadlock errors or for FK constraint errors when it makes sense to +retry other members of the batch and not fail it entirely hurting other +innocent queries. + +We can do this, because we know that if some transaction is aborted, it's +always safe to retry it. (If we're not sure about the transaction, e.g. the +Client doesn't support transactions at all, then the method should return +false.) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `error` | `unknown` | + +#### Returns + +`boolean` + +*** + +### key() + +> **key**(`_input`): `string` + +Defined in: [src/abstract/Runner.ts:77](https://github.com/clickup/ent-framework/blob/master/src/abstract/Runner.ts#L77) + +Returns a batch-dedupping key for the input. By default, no dedupping is +performed (i.e. all inputs are processed individually and not collapsed +into one input; e.g. this is needed for inserts). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `_input` | `TInput` | + +#### Returns + +`string` diff --git a/docs/classes/Schema.md b/docs/classes/Schema.md new file mode 100644 index 0000000..8c6c538 --- /dev/null +++ b/docs/classes/Schema.md @@ -0,0 +1,281 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Schema + +# Class: `abstract` Schema\ + +Defined in: [src/abstract/Schema.ts:37](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L37) + +Schema is like a "table" in some database (sharded, but it's beyond the scope +of Schema). It is also a factory of Query: it knows how to build runnable +Query objects. This 2nd role is database engine specific (e.g. there might be +PgSchema, RedisSchema etc.): such composition simplifies the code and lowers +the number of abstractions. + +The set of supported Queries is opinionated and is crafted carefully to +support the minimal possible list of primitives, but at the same time, be not +too limited in the queries the DB engine can execute. + +## Extended by + +- [`PgSchema`](PgSchema.md) + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | - | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | + +## Constructors + +### new Schema() + +> **new Schema**\<`TTable`, `TUniqueKey`\>(`name`, `table`, `uniqueKey`): [`Schema`](Schema.md)\<`TTable`, `TUniqueKey`\> + +Defined in: [src/abstract/Schema.ts:116](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L116) + +#### Parameters + +| Parameter | Type | Description | +| ------ | ------ | ------ | +| `name` | `string` | For relational databases, it's likely a table name. | +| `table` | `TTable` | Structure of the table. | +| `uniqueKey` | `TUniqueKey` | Fields which the native unique key consists of (if any). | + +#### Returns + +[`Schema`](Schema.md)\<`TTable`, `TUniqueKey`\> + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `constructor` | [`SchemaClass`](../interfaces/SchemaClass.md) | Used in e.g. inverses. This casts this.constructor to SchemaClass with all static methods and `new` semantic (TS doesn't do it by default; for TS, x.constructor is Function). | +| `name` | `string` | For relational databases, it's likely a table name. | +| `table` | `TTable` | Structure of the table. | +| `uniqueKey` | `TUniqueKey` | Fields which the native unique key consists of (if any). | + +## Methods + +### idGen() + +> `abstract` **idGen**(): [`Query`](../interfaces/Query.md)\<`string`\> + +Defined in: [src/abstract/Schema.ts:54](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L54) + +Generates a new ID for the row. Used when e.g. there is a beforeInsert +trigger on the Ent which needs to know the ID beforehand. + +#### Returns + +[`Query`](../interfaces/Query.md)\<`string`\> + +*** + +### insert() + +> `abstract` **insert**(`input`): [`Query`](../interfaces/Query.md)\<`null` \| `string`\> + +Defined in: [src/abstract/Schema.ts:60](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L60) + +Creates a new row. Returns null if the row violates some unique key +constraint, otherwise returns the row ID. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`null` \| `string`\> + +*** + +### upsert() + +> `abstract` **upsert**(`input`): [`Query`](../interfaces/Query.md)\<`string`\> + +Defined in: [src/abstract/Schema.ts:65](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L65) + +Upserts a row. Always returns the row ID. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`string`\> + +*** + +### update() + +> `abstract` **update**(`id`, `input`): [`Query`](../interfaces/Query.md)\<`boolean`\> + +Defined in: [src/abstract/Schema.ts:70](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L70) + +Updates one single row by its ID. Returns true if it actually existed. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | +| `input` | [`UpdateInput`](../type-aliases/UpdateInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`boolean`\> + +*** + +### delete() + +> `abstract` **delete**(`id`): [`Query`](../interfaces/Query.md)\<`boolean`\> + +Defined in: [src/abstract/Schema.ts:75](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L75) + +Deletes a row by id. Returns true if it actually existed. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`boolean`\> + +*** + +### load() + +> `abstract` **load**(`id`): [`Query`](../interfaces/Query.md)\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +Defined in: [src/abstract/Schema.ts:81](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L81) + +"Load" family of methods means that we load exactly one row. This one +returns a row by its ID or null if it's not found. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +*** + +### loadBy() + +> `abstract` **loadBy**(`input`): [`Query`](../interfaces/Query.md)\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +Defined in: [src/abstract/Schema.ts:87](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L87) + +Loads one single row by its unique key ("by" denotes that it's based on an +unique key, not on an ID). Returns null if it's not found. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TUniqueKey`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`null` \| [`Row`](../type-aliases/Row.md)\<`TTable`\>\> + +*** + +### selectBy() + +> `abstract` **selectBy**(`input`): [`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +Defined in: [src/abstract/Schema.ts:96](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L96) + +"Select" family of methods means that we load multiple rows ("by" denotes +that it's based on an unique key, not on an arbitrary query). This one +returns all rows whose unique key prefix matches the input. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`SelectByInput`](../type-aliases/SelectByInput.md)\<`TTable`, `TUniqueKey`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +*** + +### select() + +> `abstract` **select**(`input`): [`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +Defined in: [src/abstract/Schema.ts:103](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L103) + +Returns all rows matching an arbitrary query. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`SelectInput`](../type-aliases/SelectInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>[]\> + +*** + +### count() + +> `abstract` **count**(`input`): [`Query`](../interfaces/Query.md)\<`number`\> + +Defined in: [src/abstract/Schema.ts:108](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L108) + +Returns the number of rows matching an arbitrary query. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`CountInput`](../type-aliases/CountInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`number`\> + +*** + +### exists() + +> `abstract` **exists**(`input`): [`Query`](../interfaces/Query.md)\<`boolean`\> + +Defined in: [src/abstract/Schema.ts:114](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L114) + +An optimized version of count() for the cases where we only need to know +whether at least one row exists, and don't need a precise count. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`ExistsInput`](../type-aliases/ExistsInput.md)\<`TTable`\> | + +#### Returns + +[`Query`](../interfaces/Query.md)\<`boolean`\> diff --git a/docs/classes/Shard.md b/docs/classes/Shard.md new file mode 100644 index 0000000..9725642 --- /dev/null +++ b/docs/classes/Shard.md @@ -0,0 +1,111 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Shard + +# Class: Shard\ + +Defined in: [src/abstract/Shard.ts:21](https://github.com/clickup/ent-framework/blob/master/src/abstract/Shard.ts#L21) + +Shard lives within an Island with one master and N replicas. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TClient` *extends* [`Client`](Client.md) | + +## Constructors + +### new Shard() + +> **new Shard**\<`TClient`\>(`no`, `runOnShard`): [`Shard`](Shard.md)\<`TClient`\> + +Defined in: [src/abstract/Shard.ts:29](https://github.com/clickup/ent-framework/blob/master/src/abstract/Shard.ts#L29) + +#### Parameters + +| Parameter | Type | Description | +| ------ | ------ | ------ | +| `no` | `number` | Shard number. | +| `runOnShard` | \<`TRes`\>(`shardNo`, `body`, `onAttemptError`?) => `Promise`\<`TRes`\> | A middleware to wrap queries with. It's responsible for locating the right Island and retrying the call to body() (i.e. failed queries) in case e.g. a shard is moved to another Island. | + +#### Returns + +[`Shard`](Shard.md)\<`TClient`\> + +## Properties + +| Property | Type | Default value | Description | +| ------ | ------ | ------ | ------ | +| `lastKnownIslandNo` | `null` \| `number` | `null` | The last known Island number where this Shard was discovered. It may be out of date after the Shard is moved, and also it may be null in case there was no discovery happened yet. | +| `no` | `number` | `undefined` | Shard number. | +| `runOnShard` | \<`TRes`\>(`shardNo`: `number`, `body`: (`island`, `attempt`) => `Promise`\<`TRes`\>, `onAttemptError`?: (`error`, `attempt`) => `void`) => `Promise`\<`TRes`\> | `undefined` | A middleware to wrap queries with. It's responsible for locating the right Island and retrying the call to body() (i.e. failed queries) in case e.g. a shard is moved to another Island. | + +## Methods + +### client() + +> **client**(`timeline`): `Promise`\<`TClient`\> + +Defined in: [src/abstract/Shard.ts:46](https://github.com/clickup/ent-framework/blob/master/src/abstract/Shard.ts#L46) + +Chooses the right Client to be used for this Shard. We don't memoize, +because the Shard may relocate to another Island during re-discovery. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `timeline` | [`Timeline`](Timeline.md) \| *typeof* [`MASTER`](../variables/MASTER.md) \| *typeof* [`STALE_REPLICA`](../variables/STALE_REPLICA.md) | + +#### Returns + +`Promise`\<`TClient`\> + +*** + +### run() + +> **run**\<`TOutput`\>(`query`, `annotation`, `timeline`, `freshness`, `onAttemptError`?): `Promise`\<`TOutput`\> + +Defined in: [src/abstract/Shard.ts:59](https://github.com/clickup/ent-framework/blob/master/src/abstract/Shard.ts#L59) + +Runs a query after choosing the right Client (destination connection, +Shard, annotation etc.) + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TOutput` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `query` | [`Query`](../interfaces/Query.md)\<`TOutput`\> | +| `annotation` | [`QueryAnnotation`](../interfaces/QueryAnnotation.md) | +| `timeline` | [`Timeline`](Timeline.md) | +| `freshness` | `null` \| *typeof* [`MASTER`](../variables/MASTER.md) \| *typeof* [`STALE_REPLICA`](../variables/STALE_REPLICA.md) | +| `onAttemptError`? | (`error`, `attempt`) => `void` | + +#### Returns + +`Promise`\<`TOutput`\> + +*** + +### assertDiscoverable() + +> **assertDiscoverable**(): `Promise`\<`void`\> + +Defined in: [src/abstract/Shard.ts:101](https://github.com/clickup/ent-framework/blob/master/src/abstract/Shard.ts#L101) + +Throws if this Shard does not exist, or its Island is down, or something +else is wrong with it. + +#### Returns + +`Promise`\<`void`\> diff --git a/docs/classes/ShardError.md b/docs/classes/ShardError.md new file mode 100644 index 0000000..8954250 --- /dev/null +++ b/docs/classes/ShardError.md @@ -0,0 +1,43 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ShardError + +# Class: ShardError + +Defined in: [src/abstract/ShardError.ts:5](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardError.ts#L5) + +This non-retriable error is thrown when the system cannot detect the target +shard to work with (e.g. a null ID or a missing field or something else). + +## Extends + +- `Error` + +## Extended by + +- [`ShardIsNotDiscoverableError`](ShardIsNotDiscoverableError.md) + +## Constructors + +### new ShardError() + +> **new ShardError**(`message`, `where`?): [`ShardError`](ShardError.md) + +Defined in: [src/abstract/ShardError.ts:6](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardError.ts#L6) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `message` | `string` | +| `where`? | `string` | + +#### Returns + +[`ShardError`](ShardError.md) + +#### Overrides + +`Error.constructor` diff --git a/docs/classes/ShardIsNotDiscoverableError.md b/docs/classes/ShardIsNotDiscoverableError.md new file mode 100644 index 0000000..4bae72e --- /dev/null +++ b/docs/classes/ShardIsNotDiscoverableError.md @@ -0,0 +1,41 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ShardIsNotDiscoverableError + +# Class: ShardIsNotDiscoverableError + +Defined in: [src/abstract/ShardIsNotDiscoverableError.ts:10](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardIsNotDiscoverableError.ts#L10) + +This non-retriable error is thrown when shardsDiscoverCache.cached() returns +no shard with the requested number. + +## Extends + +- [`ShardError`](ShardError.md) + +## Constructors + +### new ShardIsNotDiscoverableError() + +> **new ShardIsNotDiscoverableError**(`shardNo`, `errors`, `islands`, `elapsed`): [`ShardIsNotDiscoverableError`](ShardIsNotDiscoverableError.md) + +Defined in: [src/abstract/ShardIsNotDiscoverableError.ts:11](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardIsNotDiscoverableError.ts#L11) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `shardNo` | `number` | +| `errors` | [`SwallowedErrorLoggerProps`](../interfaces/SwallowedErrorLoggerProps.md)[] | +| `islands` | [`Island`](Island.md)\<[`Client`](Client.md)\>[] | +| `elapsed` | `number` | + +#### Returns + +[`ShardIsNotDiscoverableError`](ShardIsNotDiscoverableError.md) + +#### Overrides + +[`ShardError`](ShardError.md).[`constructor`](ShardError.md#constructors) diff --git a/docs/classes/ShardLocator.md b/docs/classes/ShardLocator.md new file mode 100644 index 0000000..d4e4f5d --- /dev/null +++ b/docs/classes/ShardLocator.md @@ -0,0 +1,147 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ShardLocator + +# Class: ShardLocator\ + +Defined in: [src/ent/ShardLocator.ts:21](https://github.com/clickup/ent-framework/blob/master/src/ent/ShardLocator.ts#L21) + +Knows how to locate Shard(s) based on various inputs. In some contexts, we +expect exactly one Shard returned, and in other contexts, multiple Shards are +okay. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TClient` *extends* [`Client`](Client.md) | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TField` *extends* `string` | + +## Constructors + +### new ShardLocator() + +> **new ShardLocator**\<`TClient`, `TTable`, `TField`\>(`__namedParameters`): [`ShardLocator`](ShardLocator.md)\<`TClient`, `TTable`, `TField`\> + +Defined in: [src/ent/ShardLocator.ts:34](https://github.com/clickup/ent-framework/blob/master/src/ent/ShardLocator.ts#L34) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `__namedParameters` | \{ `cluster`: [`Cluster`](Cluster.md)\<`TClient`, `any`\>; `entName`: `string`; `shardAffinity`: [`ShardAffinity`](../type-aliases/ShardAffinity.md)\<`TField`\>; `uniqueKey`: `undefined` \| readonly `string`[]; `inverses`: readonly [`Inverse`](Inverse.md)\<`TClient`, `TTable`\>[]; \} | +| `__namedParameters.cluster` | [`Cluster`](Cluster.md)\<`TClient`, `any`\> | +| `__namedParameters.entName` | `string` | +| `__namedParameters.shardAffinity` | [`ShardAffinity`](../type-aliases/ShardAffinity.md)\<`TField`\> | +| `__namedParameters.uniqueKey` | `undefined` \| readonly `string`[] | +| `__namedParameters.inverses` | readonly [`Inverse`](Inverse.md)\<`TClient`, `TTable`\>[] | + +#### Returns + +[`ShardLocator`](ShardLocator.md)\<`TClient`, `TTable`, `TField`\> + +## Methods + +### singleShardForInsert() + +> **singleShardForInsert**(`input`, `op`): `Promise`\<[`Shard`](Shard.md)\<`TClient`\>\> + +Defined in: [src/ent/ShardLocator.ts:76](https://github.com/clickup/ent-framework/blob/master/src/ent/ShardLocator.ts#L76) + +Called in a context when we must know exactly 1 Shard to work with (e.g. +INSERT, UPSERT etc.). If op === "insert" (fallback to random Shard), then +returns a random Shard in case when it can't infer the Shard number from +the input (used in e.g. INSERT operations); otherwise throws ShardError +(happens in e.g. UPSERT). + +The "randomness" of the "random Shard" is deterministic by the Ent's unique +key (if it's defined), so Ents with the same unique key will map to the +same "random" Shard (considering the total number of discovered Shards is +unchanged). Notice that this logic applies at INSERT time: since we often +times add Shards to the Cluster, we can't rely on it consistently at SELECT +time (but relying at INSERT time is more or less fine: it protects against +most of "unique key violation" problems, although still doesn't prevent all +of them for a fraction of the second when the number of Shards has just +been changed). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | `Record`\<`string`, `unknown`\> | +| `op` | `"insert"` \| `"upsert"` | + +#### Returns + +`Promise`\<[`Shard`](Shard.md)\<`TClient`\>\> + +*** + +### multiShardsFromInput() + +> **multiShardsFromInput**(`vc`, `input`, `op`): `Promise`\<[`Shard`](Shard.md)\<`TClient`\>[]\> + +Defined in: [src/ent/ShardLocator.ts:110](https://github.com/clickup/ent-framework/blob/master/src/ent/ShardLocator.ts#L110) + +Called in a context when multiple Shards may be involved, e.g. when +selecting Ents referred by some Inverses. May also return the empty list of +Shards when, although there are fields with Inverses in input (i.e. the +filtering is correct), there are no Inverse rows existing in the database. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `input` | `Record`\<`string`, `unknown`\> | +| `op` | `string` | + +#### Returns + +`Promise`\<[`Shard`](Shard.md)\<`TClient`\>[]\> + +*** + +### singleShardFromID() + +> **singleShardFromID**(`field`, `id`, `op`): `Promise`\<`null` \| [`Shard`](Shard.md)\<`TClient`\>\> + +Defined in: [src/ent/ShardLocator.ts:182](https://github.com/clickup/ent-framework/blob/master/src/ent/ShardLocator.ts#L182) + +A wrapper for Cluster#shard() which injects Ent name to the exception (in +case of e.g. "Cannot locate Shard" exception). This is just a convenience +for debugging. + +If this method returns null, that means the caller should give up trying to +load the Ent with this ID, because it won't find it anyways (e.g. when we +try to load a sharded Ent using an ID from the global Shard). This is +identical to the case of an Ent not existing in the database. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `field` | `string` | +| `id` | `undefined` \| `null` \| `string` | +| `op` | `string` | + +#### Returns + +`Promise`\<`null` \| [`Shard`](Shard.md)\<`TClient`\>\> + +*** + +### allShards() + +> **allShards**(): `Promise`\[]\> + +Defined in: [src/ent/ShardLocator.ts:239](https://github.com/clickup/ent-framework/blob/master/src/ent/ShardLocator.ts#L239) + +All shards for this particular Ent depending on its affinity. + +#### Returns + +`Promise`\[]\> diff --git a/docs/classes/ShardNamer.md b/docs/classes/ShardNamer.md new file mode 100644 index 0000000..35377c9 --- /dev/null +++ b/docs/classes/ShardNamer.md @@ -0,0 +1,109 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ShardNamer + +# Class: `abstract` ShardNamer + +Defined in: [src/abstract/ShardNamer.ts:19](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L19) + +Client-specific logic on how to synchronously convert an ID into Shard number +(only for the use cases when ID is prefixed with a Shard number), how to +build Shard names, and how to extract Shard number from a Shard name. + +## Extended by + +- [`PgShardNamer`](PgShardNamer.md) + +## Constructors + +### new ShardNamer() + +> **new ShardNamer**(`options`): [`ShardNamer`](ShardNamer.md) + +Defined in: [src/abstract/ShardNamer.ts:34](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L34) + +Initializes an instance of ShardNamer. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`ShardNamerOptions`](../interfaces/ShardNamerOptions.md) | + +#### Returns + +[`ShardNamer`](ShardNamer.md) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `shardNoPadLen` | `number` | Number of decimal digits in an ID allocated for Shard number. Calculated dynamically based on `ShardNamerOptions#nameFormat` (e.g. for "sh%04d", it will be 4 since it expands to "sh0012"). | +| `options` | [`ShardNamerOptions`](../interfaces/ShardNamerOptions.md) | - | + +## Methods + +### shardNoByID() + +> `abstract` **shardNoByID**(`id`): `number` + +Defined in: [src/abstract/ShardNamer.ts:24](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L24) + +Synchronously extracts Shard number from an ID prefix, for the use cases +where IDs have this information. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `id` | `string` | + +#### Returns + +`number` + +*** + +### shardNoByName() + +> **shardNoByName**(`name`): `null` \| `number` + +Defined in: [src/abstract/ShardNamer.ts:46](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L46) + +Converts a Shard name to Shard number. Returns null if it's not a correct +Shard name. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `name` | `string` | + +#### Returns + +`null` \| `number` + +*** + +### shardNameByNo() + +> **shardNameByNo**(`no`): `string` + +Defined in: [src/abstract/ShardNamer.ts:58](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L58) + +Builds the Shard name (e.g. for PG, "schema name") by Shard number using +`ShardNamerOptions#nameFormat`. + +E.g. nameFormat="sh%04d" generates names like "sh0042". + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `no` | `number` | + +#### Returns + +`string` diff --git a/docs/classes/Timeline.md b/docs/classes/Timeline.md new file mode 100644 index 0000000..820ff02 --- /dev/null +++ b/docs/classes/Timeline.md @@ -0,0 +1,164 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Timeline + +# Class: Timeline + +Defined in: [src/abstract/Timeline.ts:36](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L36) + +Tracks replication lag timeline position at master per "user" and Ent. +- serialization format: "pos:expiresAt" +- wipes expired records (expiration is calculated at assignment moment) + +How replication lag (timeline) tracking works: for each +microshard+Ent+"user", we know the “last write-ahead log write position” +which that user (typically, VC#principal) made recently. This info can be +propagated through e.g. user's session and push notifications/subscriptions +channels automatically (“serialized timeline” and “timelines merging”). So +the next time the same user tries to read the data from the same Ent on the +same microshard, Ent Framework makes a choice, whether the replica is “good +enough” for this already; if not, it falls back to master read. I.e. the data +is not granular to individual Ent ID, it’s granular to the +user+Ent+microshard, and thus it is decoupled from IDs. + +## Constructors + +### new Timeline() + +> **new Timeline**(`state`): [`Timeline`](Timeline.md) + +Defined in: [src/abstract/Timeline.ts:39](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L39) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `state` | \{ `pos`: `bigint`; `expiresAt`: `number`; \} \| `"unknown"` | `"unknown"` | + +#### Returns + +[`Timeline`](Timeline.md) + +## Methods + +### deserialize() + +> `static` **deserialize**(`data`, `prevTimeline`): [`Timeline`](Timeline.md) + +Defined in: [src/abstract/Timeline.ts:45](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L45) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `data` | `undefined` \| `string` | +| `prevTimeline` | `null` \| [`Timeline`](Timeline.md) | + +#### Returns + +[`Timeline`](Timeline.md) + +*** + +### cloneMap() + +> `static` **cloneMap**(`timelines`): `Map`\<`string`, [`Timeline`](Timeline.md)\> + +Defined in: [src/abstract/Timeline.ts:67](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L67) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `timelines` | `ReadonlyMap`\<`string`, [`Timeline`](Timeline.md)\> | + +#### Returns + +`Map`\<`string`, [`Timeline`](Timeline.md)\> + +*** + +### serialize() + +> **serialize**(): `undefined` \| `string` + +Defined in: [src/abstract/Timeline.ts:80](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L80) + +#### Returns + +`undefined` \| `string` + +*** + +### setPos() + +> **setPos**(`pos`, `maxLagMs`): `void` + +Defined in: [src/abstract/Timeline.ts:87](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L87) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `pos` | `bigint` | +| `maxLagMs` | `number` | + +#### Returns + +`void` + +*** + +### isCaughtUp() + +> **isCaughtUp**(`replicaPos`): [`TimelineCaughtUpReason`](../type-aliases/TimelineCaughtUpReason.md) + +Defined in: [src/abstract/Timeline.ts:100](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L100) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `replicaPos` | `bigint` | + +#### Returns + +[`TimelineCaughtUpReason`](../type-aliases/TimelineCaughtUpReason.md) + +*** + +### isSaved() + +> **isSaved**(): `boolean` + +Defined in: [src/abstract/Timeline.ts:110](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L110) + +#### Returns + +`boolean` + +*** + +### setIsSaved() + +> **setIsSaved**(): `void` + +Defined in: [src/abstract/Timeline.ts:118](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L118) + +#### Returns + +`void` + +*** + +### reset() + +> **reset**(): `void` + +Defined in: [src/abstract/Timeline.ts:122](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L122) + +#### Returns + +`void` diff --git a/docs/classes/TimelineManager.md b/docs/classes/TimelineManager.md new file mode 100644 index 0000000..be71d5f --- /dev/null +++ b/docs/classes/TimelineManager.md @@ -0,0 +1,78 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / TimelineManager + +# Class: TimelineManager + +Defined in: [src/abstract/TimelineManager.ts:10](https://github.com/clickup/ent-framework/blob/master/src/abstract/TimelineManager.ts#L10) + +A side effect based container which holds the current master or replica +timeline position. For master, the expectation is that the pos will be +updated after each query only, so no need to use refreshMs. For replica, it's +also updated after each query PLUS the class will call triggerRefresh() hook +not more often than every refreshMs interval. + +## Constructors + +### new TimelineManager() + +> **new TimelineManager**(`maxLagMs`, `refreshMs`, `triggerRefresh`): [`TimelineManager`](TimelineManager.md) + +Defined in: [src/abstract/TimelineManager.ts:15](https://github.com/clickup/ent-framework/blob/master/src/abstract/TimelineManager.ts#L15) + +#### Parameters + +| Parameter | Type | Description | +| ------ | ------ | ------ | +| `maxLagMs` | `MaybeCallable`\<`number`\> | Time interval after which a replica is declared as "caught up" even if it's not caught up. This is to not read from master forever when something has happened with the replica. | +| `refreshMs` | `MaybeCallable`\<`number`\> | Up to how often we call triggerRefresh(). | +| `triggerRefresh` | () => `Promise`\<`unknown`\> | This method is called from time to time to refresh the data which is later returned by currentPos(). Makes sense for replica connections which execute queries rarely: for them, the framework triggers the update when the fresh data is needed. | + +#### Returns + +[`TimelineManager`](TimelineManager.md) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `maxLagMs` | `MaybeCallable`\<`number`\> | Time interval after which a replica is declared as "caught up" even if it's not caught up. This is to not read from master forever when something has happened with the replica. | + +## Methods + +### currentPos() + +> **currentPos**(): `Promise`\<`bigint`\> + +Defined in: [src/abstract/TimelineManager.ts:33](https://github.com/clickup/ent-framework/blob/master/src/abstract/TimelineManager.ts#L33) + +Returns the current Client's replication timeline position (e.g. WAL +position). + +#### Returns + +`Promise`\<`bigint`\> + +*** + +### setCurrentPos() + +> **setCurrentPos**(`pos`, `force`?): `void` + +Defined in: [src/abstract/TimelineManager.ts:56](https://github.com/clickup/ent-framework/blob/master/src/abstract/TimelineManager.ts#L56) + +Sets the actual timeline pos. Must be called by the Client after each +interaction with the database. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `pos` | `bigint` | +| `force`? | `boolean` | + +#### Returns + +`void` diff --git a/docs/classes/TimelineStorage.md b/docs/classes/TimelineStorage.md new file mode 100644 index 0000000..07d9fc1 --- /dev/null +++ b/docs/classes/TimelineStorage.md @@ -0,0 +1,84 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / TimelineStorage + +# Class: `abstract` TimelineStorage + +Defined in: [src/ent/TimelineStorage.ts:14](https://github.com/clickup/ent-framework/blob/master/src/ent/TimelineStorage.ts#L14) + +An abstract class that defines the interface for loading and storing +timelines per VC principals. + +## Extended by + +- [`PgTimelineStorage`](PgTimelineStorage.md) + +## Constructors + +### new TimelineStorage() + +> **new TimelineStorage**(`options`): [`TimelineStorage`](TimelineStorage.md) + +Defined in: [src/ent/TimelineStorage.ts:43](https://github.com/clickup/ent-framework/blob/master/src/ent/TimelineStorage.ts#L43) + +Initializes an instance of TimelineStorage. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`TimelineStorageOptions`](../interfaces/TimelineStorageOptions.md) | + +#### Returns + +[`TimelineStorage`](TimelineStorage.md) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`TimelineStorageOptions`](../interfaces/TimelineStorageOptions.md)\>\> | Default values for the constructor options. | +| `options` | `Required`\<[`TimelineStorageOptions`](../interfaces/TimelineStorageOptions.md)\> | Client configuration options. | + +## Methods + +### load() + +> `abstract` **load**(`principal`): `Promise`\<`string`[]\> + +Defined in: [src/ent/TimelineStorage.ts:33](https://github.com/clickup/ent-framework/blob/master/src/ent/TimelineStorage.ts#L33) + +Loads the timelines from the storage for a given principal. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `principal` | `string` | + +#### Returns + +`Promise`\<`string`[]\> + +*** + +### save() + +> `abstract` **save**(`principal`, `dataStr`): `Promise`\<`void`\> + +Defined in: [src/ent/TimelineStorage.ts:38](https://github.com/clickup/ent-framework/blob/master/src/ent/TimelineStorage.ts#L38) + +Saves the timelines in the storage for a given principal. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `principal` | `string` | +| `dataStr` | `string` | + +#### Returns + +`Promise`\<`void`\> diff --git a/docs/classes/ToolPing.md b/docs/classes/ToolPing.md new file mode 100644 index 0000000..5cfb7ad --- /dev/null +++ b/docs/classes/ToolPing.md @@ -0,0 +1,55 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ToolPing + +# Class: ToolPing + +Defined in: [src/tools/ToolPing.ts:34](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolPing.ts#L34) + +A tool which plays the role of Linux `ping` command, but for master() or +replica() Client of a Shard. Allows to verify that there is no downtime +happening when a PG node goes down or experiences a failover/switchover. + +## Constructors + +### new ToolPing() + +> **new ToolPing**(`options`): [`ToolPing`](ToolPing.md) + +Defined in: [src/tools/ToolPing.ts:49](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolPing.ts#L49) + +Initializes the instance. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`ToolPingOptions`](../interfaces/ToolPingOptions.md) | + +#### Returns + +[`ToolPing`](ToolPing.md) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`ToolPingOptions`](../interfaces/ToolPingOptions.md)\>\> | Default values for the constructor options. | +| `options` | `Required`\<[`ToolPingOptions`](../interfaces/ToolPingOptions.md)\> | Options of this tool. | + +## Methods + +### \[asyncIterator\]() + +> **\[asyncIterator\]**(): `AsyncGenerator`\<`string`, `any`, `any`\> + +Defined in: [src/tools/ToolPing.ts:57](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolPing.ts#L57) + +Runs an endless loop that pings a master() or replica() Client of the +passed Island. Yields the colored output line by line. + +#### Returns + +`AsyncGenerator`\<`string`, `any`, `any`\> diff --git a/docs/classes/ToolScoreboard.md b/docs/classes/ToolScoreboard.md new file mode 100644 index 0000000..80d3361 --- /dev/null +++ b/docs/classes/ToolScoreboard.md @@ -0,0 +1,73 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ToolScoreboard + +# Class: ToolScoreboard + +Defined in: [src/tools/ToolScoreboard.ts:75](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolScoreboard.ts#L75) + +A tool which plays the role of Linux `top` command, but for the Cluster. +Tracks the state of the Cluster and Clients health. + +## Constructors + +### new ToolScoreboard() + +> **new ToolScoreboard**(`options`): [`ToolScoreboard`](ToolScoreboard.md) + +Defined in: [src/tools/ToolScoreboard.ts:127](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolScoreboard.ts#L127) + +Initializes the instance. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`ToolScoreboardOptions`](../interfaces/ToolScoreboardOptions.md) | + +#### Returns + +[`ToolScoreboard`](ToolScoreboard.md) + +## Properties + +| Property | Type | Default value | Description | +| ------ | ------ | ------ | ------ | +| `DEFAULT_OPTIONS` | `Required`\<`PickPartial`\<[`ToolScoreboardOptions`](../interfaces/ToolScoreboardOptions.md)\>\> | `undefined` | Default values for the constructor options. | +| `options` | `Required`\<[`ToolScoreboardOptions`](../interfaces/ToolScoreboardOptions.md)\> | `undefined` | Options of this tool. | +| `islands` | `Map`\<`number`, \{ `shards`: `number`; `clients`: `Map`\<`ClientIdent`, [`Client`](Client.md)\>; \}\> | `undefined` | Registry of all Islands with Clients. | +| `queries` | `DefaultMap`\<`number`, `DefaultMap`\<`ClientIdent`, `ToolScoreboardQuery`[]\>\> | `undefined` | Log of queries sent (ping, discovery, tick). | +| `poolStats` | `DefaultMap`\<`number`, `Map`\<`ClientIdent`, \{ `totalConns`: `number`; `idleConns`: `number`; `queuedReqs`: `number`; \}\>\> | `undefined` | Pool stats of Clients. | +| `swallowedErrors` | `ToolScoreboardSwallowedError`[] | `[]` | Registry of the recent swallowed errors (pings-independent). | +| `queryErrors` | `ToolScoreboardQueryError`[] | `[]` | Errors extracted from the queries log. | + +## Methods + +### \[asyncIterator\]() + +> **\[asyncIterator\]**(): `AsyncGenerator`\<[`ToolScoreboard`](ToolScoreboard.md), `any`, `any`\> + +Defined in: [src/tools/ToolScoreboard.ts:135](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolScoreboard.ts#L135) + +Runs an endless loop that updates the Scoreboard with the current state of +the Cluster and yields back on every refreshMs tick. + +#### Returns + +`AsyncGenerator`\<[`ToolScoreboard`](ToolScoreboard.md), `any`, `any`\> + +*** + +### render() + +> **render**(): `string` + +Defined in: [src/tools/ToolScoreboard.ts:304](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolScoreboard.ts#L304) + +Renders the current state of the Scoreboard as a string. + +#### Returns + +`string` diff --git a/docs/classes/Triggers.md b/docs/classes/Triggers.md new file mode 100644 index 0000000..a073621 --- /dev/null +++ b/docs/classes/Triggers.md @@ -0,0 +1,125 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Triggers + +# Class: Triggers\ + +Defined in: [src/ent/Triggers.ts:169](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L169) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new Triggers() + +> **new Triggers**\<`TTable`\>(`beforeInsert`, `beforeUpdate`, `beforeDelete`, `beforeMutation`, `afterInsert`, `afterUpdate`, `afterDelete`, `afterMutation`): [`Triggers`](Triggers.md)\<`TTable`\> + +Defined in: [src/ent/Triggers.ts:170](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L170) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `beforeInsert` | [`InsertTrigger`](../type-aliases/InsertTrigger.md)\<`TTable`\>[] | +| `beforeUpdate` | \[`null` \| [`DepsBuilder`](../type-aliases/DepsBuilder.md)\<`TTable`\>, [`BeforeUpdateTrigger`](../type-aliases/BeforeUpdateTrigger.md)\<`TTable`\>\][] | +| `beforeDelete` | [`DeleteTrigger`](../type-aliases/DeleteTrigger.md)\<`TTable`\>[] | +| `beforeMutation` | \[`null` \| [`DepsBuilder`](../type-aliases/DepsBuilder.md)\<`TTable`\>, [`BeforeMutationTrigger`](../type-aliases/BeforeMutationTrigger.md)\<`TTable`\>\][] | +| `afterInsert` | [`InsertTrigger`](../type-aliases/InsertTrigger.md)\<`TTable`\>[] | +| `afterUpdate` | \[`null` \| [`DepsBuilder`](../type-aliases/DepsBuilder.md)\<`TTable`\>, [`AfterUpdateTrigger`](../type-aliases/AfterUpdateTrigger.md)\<`TTable`\>\][] | +| `afterDelete` | [`DeleteTrigger`](../type-aliases/DeleteTrigger.md)\<`TTable`\>[] | +| `afterMutation` | \[`null` \| [`DepsBuilder`](../type-aliases/DepsBuilder.md)\<`TTable`\>, [`AfterMutationTrigger`](../type-aliases/AfterMutationTrigger.md)\<`TTable`\>\][] | + +#### Returns + +[`Triggers`](Triggers.md)\<`TTable`\> + +## Methods + +### hasInsertTriggers() + +> **hasInsertTriggers**(): `boolean` + +Defined in: [src/ent/Triggers.ts:189](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L189) + +#### Returns + +`boolean` + +*** + +### hasUpdateTriggers() + +> **hasUpdateTriggers**(): `boolean` + +Defined in: [src/ent/Triggers.ts:198](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L198) + +#### Returns + +`boolean` + +*** + +### wrapInsert() + +> **wrapInsert**(`func`, `vc`, `input`): `Promise`\<`null` \| `string`\> + +Defined in: [src/ent/Triggers.ts:207](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L207) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `func` | (`input`) => `Promise`\<`null` \| `string`\> | +| `vc` | [`VC`](VC.md) | +| `input` | \{ \[K in string \| number \| symbol\]: Value\ \} & \{ \[K in string \| number \| symbol\]?: Value\ \} & [`RowWithID`](../type-aliases/RowWithID.md) | + +#### Returns + +`Promise`\<`null` \| `string`\> + +*** + +### wrapUpdate() + +> **wrapUpdate**(`func`, `vc`, `oldRow`, `input`): `Promise`\<`boolean`\> + +Defined in: [src/ent/Triggers.ts:251](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L251) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `func` | (`input`) => `Promise`\<`boolean`\> | +| `vc` | [`VC`](VC.md) | +| `oldRow` | \{ \[P in string \| symbol\]: Readonly\ \} & Record\\>\[P\] \} | +| `input` | [`UpdateInput`](../type-aliases/UpdateInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`boolean`\> + +*** + +### wrapDelete() + +> **wrapDelete**(`func`, `vc`, `oldRow`): `Promise`\<`boolean`\> + +Defined in: [src/ent/Triggers.ts:312](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L312) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `func` | () => `Promise`\<`boolean`\> | +| `vc` | [`VC`](VC.md) | +| `oldRow` | \{ \[P in string \| symbol\]: Readonly\ \} & Record\\>\[P\] \} | + +#### Returns + +`Promise`\<`boolean`\> diff --git a/docs/classes/True.md b/docs/classes/True.md new file mode 100644 index 0000000..0b81934 --- /dev/null +++ b/docs/classes/True.md @@ -0,0 +1,53 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / True + +# Class: True + +Defined in: [src/ent/predicates/True.ts:7](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/True.ts#L7) + +Always passes; used for e.g. globally accessed objects. + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<`never`\> + +## Constructors + +### new True() + +> **new True**(): [`True`](True.md) + +#### Returns + +[`True`](True.md) + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | + +## Methods + +### check() + +> **check**(`_vc`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/True.ts:10](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/True.ts#L10) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `_vc` | [`VC`](VC.md) | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/VC.md b/docs/classes/VC.md new file mode 100644 index 0000000..98df359 --- /dev/null +++ b/docs/classes/VC.md @@ -0,0 +1,598 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / VC + +# Class: VC + +Defined in: [src/ent/VC.ts:48](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L48) + +VC - Viewer Context. + +VC is set per HTTP request (or per worker job) in each Ent and represents the +person who is about to run some database operation. It can represent a user, +or a guest, or a bot observing that Ent. + +Depending on the Ent's Configuration object and privacy rules, it may allow +the user to load/insert/update/etc. or to traverse to related objects. + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `principal` | `string` | A principal (typically user ID) represented by this VC. | +| `freshness` | `null` \| *typeof* [`MASTER`](../variables/MASTER.md) \| *typeof* [`STALE_REPLICA`](../variables/STALE_REPLICA.md) | Allows to set VC to always use either a master or a replica DB. E.g. if freshness=MASTER, then all the timeline data is ignored, and all the requests are sent to master. | +| `heartbeater` | `object` | The heartbeat callback is called before each primitive operation. It plays the similar role as AbortController: when called, it may throw sometimes (signalled externally). Delay callback can also be passed since it's pretty common use case to wait for some time and be aborted on a heartbeat exception. | +| `heartbeater.heartbeat` | () => `Promise`\<`void`\> | - | +| `heartbeater.delay` | (`ms`: `number`) => `Promise`\<`void`\> | - | + +## Methods + +### createGuestPleaseDoNotUseCreationPointsMustBeLimited() + +> `static` **createGuestPleaseDoNotUseCreationPointsMustBeLimited**(`__namedParameters`): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:71](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L71) + +Please please don't call this method except one or two core places. The +idea is that we create an "origin" VC once and then derive all other VCs +from it (possibly upgrading or downgrading permissions, controlling +master/replica read policy etc.). It's also good to trace the entire chain +of calls and reasons, why some object was accessed. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `__namedParameters` | \{ `trace`: `string`; `cachesExpirationMs`: `number`; \} | +| `__namedParameters.trace`? | `string` | +| `__namedParameters.cachesExpirationMs`? | `number` | + +#### Returns + +[`VC`](VC.md) + +*** + +### \[custom\]() + +> **\[custom\]**(): `string` + +Defined in: [src/ent/VC.ts:93](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L93) + +This is to show VCs in console.log() and inspect() nicely. + +#### Returns + +`string` + +*** + +### cache() + +#### Call Signature + +> **cache**\<`TInstance`\>(`Class`): `TInstance` + +Defined in: [src/ent/VC.ts:101](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L101) + +Some IDs are cached in VC (e.g. is this ID readable? is it writable? is +this VC an admin VC?). Also, people may define their own VC-local caches. + +##### Type Parameters + +| Type Parameter | +| ------ | +| `TInstance` | + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Class` | (`vc`) => `TInstance` | + +##### Returns + +`TInstance` + +#### Call Signature + +> **cache**\<`TInstance`\>(`tag`, `creator`): `TInstance` + +Defined in: [src/ent/VC.ts:107](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L107) + +Same as the above overload, but allows to use a custom creating function. +This is useful when e.g. cached values are async-created. + +##### Type Parameters + +| Type Parameter | +| ------ | +| `TInstance` | + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| `tag` | `symbol` | +| `creator` | (`vc`) => `TInstance` | + +##### Returns + +`TInstance` + +*** + +### loader() + +> **loader**\<`TLoadArgs`, `TReturn`\>(`HandlerClass`): [`Loader`](Loader.md)\<`TLoadArgs`, `TReturn`\> + +Defined in: [src/ent/VC.ts:132](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L132) + +Returns a cached instance of Loader whose actual code is defined in +HandlerClass. In case there is no such Loader yet, creates it. + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TLoadArgs` *extends* `unknown`[] | +| `TReturn` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `HandlerClass` | (`vc`) => [`Handler`](../interfaces/Handler.md)\<`TLoadArgs`, `TReturn`\> | +| `HandlerClass.$loader`? | `symbol` | + +#### Returns + +[`Loader`](Loader.md)\<`TLoadArgs`, `TReturn`\> + +*** + +### timeline() + +> **timeline**(`shard`, `schemaName`): [`Timeline`](Timeline.md) + +Defined in: [src/ent/VC.ts:148](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L148) + +Returns Shard+schemaName timeline which tracks replica staleness for the +particular schema name (most likely, table). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `shard` | [`Shard`](Shard.md)\<[`Client`](Client.md)\> | +| `schemaName` | `string` | + +#### Returns + +[`Timeline`](Timeline.md) + +*** + +### serializeTimelines() + +> **serializeTimelines**(): `undefined` \| `string` + +Defined in: [src/ent/VC.ts:164](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L164) + +Serializes Shard timelines (master WAL positions) to a string format. The +method always returns a value which is compatible to +deserializeTimelines() input. + +#### Returns + +`undefined` \| `string` + +*** + +### deserializeTimelines() + +> **deserializeTimelines**(...`dataStrs`): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:196](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L196) + +Restores all replication timelines in the VC based on the serialized info +provided. Returns the new VC derived from the current one, but with empty +caches. + +This method has a side effect of changing the timelines of the current VC +(and actually all parent VCs), because it reflects the changes in the +global DB state as seen by the current VC's principal. It restores +previously serialized timelines to the existing VC and all its parent VCs +which share the same principal. (The latter happens, because +`this.timelines` map is passed by reference to all derived VCs starting +from the one which sets principal; see `new VC(...)` clauses all around and +toLowerInternal() logic.) The timelines are merged according to WAL +positions (larger WAL positions win). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| ...`dataStrs` | readonly (`undefined` \| `string`)[] | + +#### Returns + +[`VC`](VC.md) + +*** + +### saveTimelines() + +> **saveTimelines**(`storage`): `Promise`\<`undefined` \| `string`\> + +Defined in: [src/ent/VC.ts:224](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L224) + +Saves the timelines to the storage. The function minimizes the number of +writes to the storage: + +- Calling saveTimelines() the 2nd time without making changes to the + timelines is a no-op. +- Calling saveTimelines() after loadTimelines() without making changes to + the timelines is a no-op. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `storage` | [`TimelineStorage`](TimelineStorage.md) | + +#### Returns + +`Promise`\<`undefined` \| `string`\> + +*** + +### loadTimelines() + +> **loadTimelines**(`storage`): `Promise`\<[`VC`](VC.md)\> + +Defined in: [src/ent/VC.ts:256](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L256) + +Loads the timelines from the storage and deserializes them into the current +VC. Returns the new VC derived from the current one, but with empty caches. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `storage` | [`TimelineStorage`](TimelineStorage.md) | + +#### Returns + +`Promise`\<[`VC`](VC.md)\> + +*** + +### withEmptyCache() + +> **withEmptyCache**(): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:275](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L275) + +Returns a new VC derived from the current one, but with empty cache. + +#### Returns + +[`VC`](VC.md) + +*** + +### withTransitiveMasterFreshness() + +> **withTransitiveMasterFreshness**(): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:293](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L293) + +Returns a new VC derived from the current one, but with master freshness. +Master freshness is inherited by ent.vc after an Ent is loaded. + +#### Returns + +[`VC`](VC.md) + +*** + +### withOneTimeStaleReplica() + +> **withOneTimeStaleReplica**(): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:320](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L320) + +Returns a new VC derived from the current one, but which forces an Ent to +be loaded always from replica. Freshness is NOT inherited by Ents (not +transitive): e.g. if an Ent is loaded with STALE_REPLICA freshness, its +ent.vc will have the DEFAULT freshness. + +Also, if an Ent is inserted with a VC of STALE_REPLICA freshness, its VC +won't remember it, so next immediate reads will go to a replica and not to +the master. + +#### Returns + +[`VC`](VC.md) + +*** + +### withDefaultFreshness() + +> **withDefaultFreshness**(): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:343](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L343) + +Creates a new VC with default freshness (i.e. not sticky to master or +replica, auto-detected on request). Generally, it's not a good idea to use +this derivation since we lose some bit of internal knowledge from the past +history of the VC, but for e.g. tests or benchmarks, it's fine. + +#### Returns + +[`VC`](VC.md) + +*** + +### withFlavor() + +#### Call Signature + +> **withFlavor**(`prepend`, ...`flavors`): `this` + +Defined in: [src/ent/VC.ts:364](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L364) + +Returns a new VC derived from the current one adding some more flavors to +it. If no flavors were added, returns the same VC (`this`). + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| `prepend` | `"prepend"` | +| ...`flavors` | (`undefined` \| [`VCFlavor`](VCFlavor.md))[] | + +##### Returns + +`this` + +#### Call Signature + +> **withFlavor**(...`flavors`): `this` + +Defined in: [src/ent/VC.ts:365](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L365) + +Returns a new VC derived from the current one adding some more flavors to +it. If no flavors were added, returns the same VC (`this`). + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| ...`flavors` | (`undefined` \| [`VCFlavor`](VCFlavor.md))[] | + +##### Returns + +`this` + +*** + +### withoutFlavor() + +> **withoutFlavor**(...`flavorClasses`): `this` + +Defined in: [src/ent/VC.ts:403](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L403) + +Returns a new VC derived from the current one removing the specified flavors. +If no flavors were removed, returns the same VC (`this`). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| ...`flavorClasses` | (...`args`) => [`VCFlavor`](VCFlavor.md)[] | + +#### Returns + +`this` + +*** + +### withNewTrace() + +> **withNewTrace**(`trace`): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:436](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L436) + +Derives the VC with new trace ID. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `trace` | `undefined` \| `string` | + +#### Returns + +[`VC`](VC.md) + +*** + +### withHeartbeater() + +> **withHeartbeater**(`heartbeater`): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:452](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L452) + +Derives the VC with the provided heartbeater injected. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `heartbeater` | \{ `heartbeat`: () => `Promise`\<`void`\>; `delay`: (`ms`) => `Promise`\<`void`\>; \} | +| `heartbeater.heartbeat` | () => `Promise`\<`void`\> | +| `heartbeater.delay` | (`ms`) => `Promise`\<`void`\> | + +#### Returns + +[`VC`](VC.md) + +*** + +### toOmniDangerous() + +> **toOmniDangerous**(): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:472](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L472) + +Creates a new VC upgraded to omni permissions. This VC will not +be placed to some Ent's ent.vc property; instead, it will be +automatically downgraded to either the owning VC of this Ent or +to a guest VC (see Ent.ts). + +#### Returns + +[`VC`](VC.md) + +*** + +### toGuest() + +> **toGuest**(): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:489](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L489) + +Creates a new VC downgraded to guest permissions. + +#### Returns + +[`VC`](VC.md) + +*** + +### isOmni() + +> **isOmni**(): `boolean` + +Defined in: [src/ent/VC.ts:504](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L504) + +Checks if it's an omni VC. + +#### Returns + +`boolean` + +*** + +### isGuest() + +> **isGuest**(): `boolean` + +Defined in: [src/ent/VC.ts:511](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L511) + +Checks if it's a guest VC. + +#### Returns + +`boolean` + +*** + +### isLoggedIn() + +> **isLoggedIn**(): `boolean` + +Defined in: [src/ent/VC.ts:518](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L518) + +Checks if it's a regular user (i.e. owning) VC. + +#### Returns + +`boolean` + +*** + +### flavor() + +> **flavor**\<`TFlavor`\>(`flavor`): `null` \| `TFlavor` + +Defined in: [src/ent/VC.ts:525](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L525) + +Returns VC's flavor of the particular type. + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TFlavor` *extends* [`VCFlavor`](VCFlavor.md) | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `flavor` | (...`args`) => `TFlavor` | + +#### Returns + +`null` \| `TFlavor` + +*** + +### toString() + +> **toString**(`withInstanceNumber`): `string` + +Defined in: [src/ent/VC.ts:534](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L534) + +Used for debugging purposes. + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `withInstanceNumber` | `boolean` | `false` | + +#### Returns + +`string` + +*** + +### toAnnotation() + +> **toAnnotation**(): [`QueryAnnotation`](../interfaces/QueryAnnotation.md) + +Defined in: [src/ent/VC.ts:553](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L553) + +Returns a debug annotation of this VC. + +#### Returns + +[`QueryAnnotation`](../interfaces/QueryAnnotation.md) + +*** + +### toLowerInternal() + +> **toLowerInternal**(`principal`): [`VC`](VC.md) + +Defined in: [src/ent/VC.ts:584](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L584) + +Used internally by Ent framework to lower permissions of an injected VC. +For guest, principal === null. +- freshness is always reset to default one it VC is demoted +- isRoot is changed to false once a root VC is switched to a per-user VC + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `principal` | `null` \| `string` | + +#### Returns + +[`VC`](VC.md) diff --git a/docs/classes/VCCaches.md b/docs/classes/VCCaches.md new file mode 100644 index 0000000..6c79bfd --- /dev/null +++ b/docs/classes/VCCaches.md @@ -0,0 +1,89 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / VCCaches + +# Class: VCCaches\ + +Defined in: [src/ent/VCCaches.ts:6](https://github.com/clickup/ent-framework/blob/master/src/ent/VCCaches.ts#L6) + +Holds an auto-expiring map of VC caches. + +## Extends + +- `Map`\<`TKey`, `TValue`\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TKey` | +| `TValue` | + +## Constructors + +### new VCCaches() + +> **new VCCaches**\<`TKey`, `TValue`\>(`expirationMs`): [`VCCaches`](VCCaches.md)\<`TKey`, `TValue`\> + +Defined in: [src/ent/VCCaches.ts:7](https://github.com/clickup/ent-framework/blob/master/src/ent/VCCaches.ts#L7) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `expirationMs` | `number` | + +#### Returns + +[`VCCaches`](VCCaches.md)\<`TKey`, `TValue`\> + +#### Overrides + +`Map.constructor` + +## Methods + +### get() + +> **get**(`key`): `undefined` \| `TValue` + +Defined in: [src/ent/VCCaches.ts:15](https://github.com/clickup/ent-framework/blob/master/src/ent/VCCaches.ts#L15) + +Calls the Map's get() and defers cache clearing to the next WeakTicker +tick (i.e. schedules clearing on inactivity). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `key` | `TKey` | + +#### Returns + +`undefined` \| `TValue` + +#### Overrides + +`Map.get` + +*** + +### onTick() + +> **onTick**(`tickNoSinceScheduling`): `"keep"` \| `"unschedule"` + +Defined in: [src/ent/VCCaches.ts:26](https://github.com/clickup/ent-framework/blob/master/src/ent/VCCaches.ts#L26) + +Called periodically after VC#cache() was called at least once. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `tickNoSinceScheduling` | `number` | + +#### Returns + +`"keep"` \| `"unschedule"` diff --git a/docs/classes/VCFlavor.md b/docs/classes/VCFlavor.md new file mode 100644 index 0000000..5f9c2ee --- /dev/null +++ b/docs/classes/VCFlavor.md @@ -0,0 +1,52 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / VCFlavor + +# Class: `abstract` VCFlavor + +Defined in: [src/ent/VCFlavor.ts:9](https://github.com/clickup/ent-framework/blob/master/src/ent/VCFlavor.ts#L9) + +VCFlavor is some piece of info which is transitively attached to a VC and is +preserved when VC derivation (upgrade/downgrade) is happening. This piece of +info may be just an object of a separate class with no data (acts like a +boolean flag), or it can be an object with payload. + +For each flavor type, only a single VCFlavor object may exist. + +## Extended by + +- [`VCWithStacks`](VCWithStacks.md) +- [`VCWithQueryCache`](VCWithQueryCache.md) + +## Constructors + +### new VCFlavor() + +> **new VCFlavor**(): [`VCFlavor`](VCFlavor.md) + +#### Returns + +[`VCFlavor`](VCFlavor.md) + +## Properties + +| Property | Type | +| ------ | ------ | +| `_tag` | `"VCFlavorClass"` | +| `_tag` | `"VCFlavorInstance"` | + +## Methods + +### toDebugString() + +> **toDebugString**(): `string` + +Defined in: [src/ent/VCFlavor.ts:16](https://github.com/clickup/ent-framework/blob/master/src/ent/VCFlavor.ts#L16) + +Appended to the end of VC.toString() result. + +#### Returns + +`string` diff --git a/docs/classes/VCHasFlavor.md b/docs/classes/VCHasFlavor.md new file mode 100644 index 0000000..1515048 --- /dev/null +++ b/docs/classes/VCHasFlavor.md @@ -0,0 +1,61 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / VCHasFlavor + +# Class: VCHasFlavor + +Defined in: [src/ent/predicates/VCHasFlavor.ts:8](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/VCHasFlavor.ts#L8) + +Checks that the VC has some flavor. + +## Implements + +- [`Predicate`](../interfaces/Predicate.md)\<`never`\> + +## Constructors + +### new VCHasFlavor() + +> **new VCHasFlavor**(`Flavor`): [`VCHasFlavor`](VCHasFlavor.md) + +Defined in: [src/ent/predicates/VCHasFlavor.ts:11](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/VCHasFlavor.ts#L11) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `Flavor` | (...`args`) => [`VCFlavor`](VCFlavor.md) | + +#### Returns + +[`VCHasFlavor`](VCHasFlavor.md) + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | + +## Methods + +### check() + +> **check**(`vc`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/VCHasFlavor.ts:15](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/VCHasFlavor.ts#L15) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | + +#### Returns + +`Promise`\<`boolean`\> + +#### Implementation of + +[`Predicate`](../interfaces/Predicate.md).[`check`](../interfaces/Predicate.md#check) diff --git a/docs/classes/VCTrace.md b/docs/classes/VCTrace.md new file mode 100644 index 0000000..28eb91b --- /dev/null +++ b/docs/classes/VCTrace.md @@ -0,0 +1,53 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / VCTrace + +# Class: VCTrace + +Defined in: [src/ent/VCTrace.ts:14](https://github.com/clickup/ent-framework/blob/master/src/ent/VCTrace.ts#L14) + +A "trace" objects which allows to group database related stuff while logging +it. Traces are inherited during VC derivation, similar to flavors, but +they're a part of VC core interface to allow faster access. + +## Constructors + +### new VCTrace() + +> **new VCTrace**(`trace`?): [`VCTrace`](VCTrace.md) + +Defined in: [src/ent/VCTrace.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/VCTrace.ts#L17) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `trace`? | `string` | + +#### Returns + +[`VCTrace`](VCTrace.md) + +## Properties + +| Property | Type | +| ------ | ------ | +| `trace` | `string` | + +## Methods + +### tryExtractCreationDate() + +> **tryExtractCreationDate**(): `null` \| `Date` + +Defined in: [src/ent/VCTrace.ts:26](https://github.com/clickup/ent-framework/blob/master/src/ent/VCTrace.ts#L26) + +In case the trace was created by this tool, tries to extract the date of +its creation. As a sanity check, verifies that this date is not too far +away from the present time. + +#### Returns + +`null` \| `Date` diff --git a/docs/classes/VCWithQueryCache.md b/docs/classes/VCWithQueryCache.md new file mode 100644 index 0000000..e487560 --- /dev/null +++ b/docs/classes/VCWithQueryCache.md @@ -0,0 +1,65 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / VCWithQueryCache + +# Class: VCWithQueryCache + +Defined in: [src/ent/VCFlavor.ts:30](https://github.com/clickup/ent-framework/blob/master/src/ent/VCFlavor.ts#L30) + +If set, Ent cache is enabled for operations in this VC. + +## Extends + +- [`VCFlavor`](VCFlavor.md) + +## Constructors + +### new VCWithQueryCache() + +> **new VCWithQueryCache**(`options`): [`VCWithQueryCache`](VCWithQueryCache.md) + +Defined in: [src/ent/VCFlavor.ts:31](https://github.com/clickup/ent-framework/blob/master/src/ent/VCFlavor.ts#L31) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | \{ `maxQueries`: `number`; \} | +| `options.maxQueries` | `number` | + +#### Returns + +[`VCWithQueryCache`](VCWithQueryCache.md) + +#### Overrides + +[`VCFlavor`](VCFlavor.md).[`constructor`](VCFlavor.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `_tag` | `"VCFlavorClass"` | +| `_tag` | `"VCFlavorInstance"` | +| `options` | `object` | +| `options.maxQueries` | `number` | + +## Methods + +### toDebugString() + +> **toDebugString**(): `string` + +Defined in: [src/ent/VCFlavor.ts:16](https://github.com/clickup/ent-framework/blob/master/src/ent/VCFlavor.ts#L16) + +Appended to the end of VC.toString() result. + +#### Returns + +`string` + +#### Inherited from + +[`VCFlavor`](VCFlavor.md).[`toDebugString`](VCFlavor.md#todebugstring) diff --git a/docs/classes/VCWithStacks.md b/docs/classes/VCWithStacks.md new file mode 100644 index 0000000..0a583b9 --- /dev/null +++ b/docs/classes/VCWithStacks.md @@ -0,0 +1,55 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / VCWithStacks + +# Class: VCWithStacks + +Defined in: [src/ent/VCFlavor.ts:25](https://github.com/clickup/ent-framework/blob/master/src/ent/VCFlavor.ts#L25) + +If turned on, the debug logs will contain caller stack traces for each Ent +query. This is expensive, use in dev mode only! + +## Extends + +- [`VCFlavor`](VCFlavor.md) + +## Constructors + +### new VCWithStacks() + +> **new VCWithStacks**(): [`VCWithStacks`](VCWithStacks.md) + +#### Returns + +[`VCWithStacks`](VCWithStacks.md) + +#### Inherited from + +[`VCFlavor`](VCFlavor.md).[`constructor`](VCFlavor.md#constructors) + +## Properties + +| Property | Type | +| ------ | ------ | +| `_tag` | `"VCFlavorClass"` | +| `_tag` | `"VCFlavorInstance"` | + +## Methods + +### toDebugString() + +> **toDebugString**(): `string` + +Defined in: [src/ent/VCFlavor.ts:16](https://github.com/clickup/ent-framework/blob/master/src/ent/VCFlavor.ts#L16) + +Appended to the end of VC.toString() result. + +#### Returns + +`string` + +#### Inherited from + +[`VCFlavor`](VCFlavor.md).[`toDebugString`](VCFlavor.md#todebugstring) diff --git a/docs/classes/Validation.md b/docs/classes/Validation.md new file mode 100644 index 0000000..0acae73 --- /dev/null +++ b/docs/classes/Validation.md @@ -0,0 +1,124 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Validation + +# Class: Validation\ + +Defined in: [src/ent/Validation.ts:69](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L69) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Constructors + +### new Validation() + +> **new Validation**\<`TTable`\>(`entName`, `rules`): [`Validation`](Validation.md)\<`TTable`\> + +Defined in: [src/ent/Validation.ts:78](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L78) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `entName` | `string` | +| `rules` | [`ValidationRules`](../type-aliases/ValidationRules.md)\<`TTable`\> | + +#### Returns + +[`Validation`](Validation.md)\<`TTable`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `tenantPrincipalField?` | [`InsertFieldsRequired`](../type-aliases/InsertFieldsRequired.md)\<`TTable`\> & `string` | +| `inferPrincipal` | (`vc`: [`VC`](VC.md), `row`: [`Row`](../type-aliases/Row.md)\<`TTable`\>) => `Promise`\<[`VC`](VC.md)\> | +| `load` | [`LoadRule`](../type-aliases/LoadRule.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>\>[] | +| `insert` | [`WriteRules`](../type-aliases/WriteRules.md)\<[`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>\> | +| `update` | [`WriteRules`](../type-aliases/WriteRules.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>\> | +| `delete` | [`WriteRules`](../type-aliases/WriteRules.md)\<[`Row`](../type-aliases/Row.md)\<`TTable`\>\> | +| `validate` | [`Require`](Require.md)\<[`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>\>[] | + +## Methods + +### validateLoad() + +> **validateLoad**(`vc`, `row`): `Promise`\<`void`\> + +Defined in: [src/ent/Validation.ts:91](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L91) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | [`Row`](../type-aliases/Row.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`void`\> + +*** + +### validateInsert() + +> **validateInsert**(`vc`, `input`): `Promise`\<`void`\> + +Defined in: [src/ent/Validation.ts:102](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L102) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`void`\> + +*** + +### validateUpdate() + +> **validateUpdate**(`vc`, `old`, `input`, `privacyOnly`): `Promise`\<`void`\> + +Defined in: [src/ent/Validation.ts:114](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L114) + +#### Parameters + +| Parameter | Type | Default value | +| ------ | ------ | ------ | +| `vc` | [`VC`](VC.md) | `undefined` | +| `old` | [`Row`](../type-aliases/Row.md)\<`TTable`\> | `undefined` | +| `input` | [`UpdateInput`](../type-aliases/UpdateInput.md)\<`TTable`\> | `undefined` | +| `privacyOnly` | `boolean` | `false` | + +#### Returns + +`Promise`\<`void`\> + +*** + +### validateDelete() + +> **validateDelete**(`vc`, `row`): `Promise`\<`void`\> + +Defined in: [src/ent/Validation.ts:141](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L141) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](VC.md) | +| `row` | [`Row`](../type-aliases/Row.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`void`\> diff --git a/docs/functions/Base64BufferType.md b/docs/functions/Base64BufferType.md new file mode 100644 index 0000000..073e72e --- /dev/null +++ b/docs/functions/Base64BufferType.md @@ -0,0 +1,63 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Base64BufferType + +# Function: Base64BufferType() + +> **Base64BufferType**(): `object` + +Defined in: [src/types.ts:392](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L392) + +A value stored in the DB as a base64 encoded binary buffer. Actually, DB +engines (like PostgreSQL) support native binary data fields (and store binary +data efficiently), but sometimes (especially for small things, like +public/private keys), it's easier to store the binary data as base64 encoded +strings rather than dealing with the native binary data type. + +## Returns + +`object` + +### dbValueToJs() + +> **dbValueToJs**: (`dbValue`) => `Buffer` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `dbValue` | `string` | + +#### Returns + +`Buffer` + +### stringify() + +> **stringify**: (`jsValue`) => `string` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `jsValue` | `Buffer` | + +#### Returns + +`string` + +### parse() + +> **parse**: (`str`) => `Buffer` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `str` | `string` | + +#### Returns + +`Buffer` diff --git a/docs/functions/BaseEnt.md b/docs/functions/BaseEnt.md new file mode 100644 index 0000000..2fac758 --- /dev/null +++ b/docs/functions/BaseEnt.md @@ -0,0 +1,47 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / BaseEnt + +# Function: BaseEnt() + +> **BaseEnt**\<`TTable`, `TUniqueKey`, `TClient`\>(`cluster`, `schema`): [`HelpersClass`](../interfaces/HelpersClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> + +Defined in: [src/ent/BaseEnt.ts:29](https://github.com/clickup/ent-framework/blob/master/src/ent/BaseEnt.ts#L29) + +This is a helper function to create new Ent classes. Run once per each +Ent+Cluster on app boot. See examples in __tests__/TestObjects.ts and +EntTest.ts. + +Since all Ent objects are immutable (following the modern practices), +1. Ent is not a DataMapper pattern; +2. Ent is not an ActiveRecord; +3. At last, Ent is not quite a DAO pattern too. + +We assume that Ents are very simple (we don't need triggers or multi-ent +touching mutations), because we anyway have a GraphQL layer on top of it. + +Finally, a naming decision has been made: we translate database field names +directly to Ent field names, no camelCase. This has proven its simplicity +benefits in the past: the less translation layers we have, the easier it is +to debug and develop. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | +| `TClient` *extends* [`Client`](../classes/Client.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `cluster` | [`Cluster`](../classes/Cluster.md)\<`TClient`, `any`\> | +| `schema` | [`Schema`](../classes/Schema.md)\<`TTable`, `TUniqueKey`\> | + +## Returns + +[`HelpersClass`](../interfaces/HelpersClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> diff --git a/docs/functions/BigIntArrayType.md b/docs/functions/BigIntArrayType.md new file mode 100644 index 0000000..2ec9845 --- /dev/null +++ b/docs/functions/BigIntArrayType.md @@ -0,0 +1,71 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / BigIntArrayType + +# Function: BigIntArrayType() + +> **BigIntArrayType**\<`T`\>(): `object` + +Defined in: [src/pg/types/BigIntArrayType.ts:19](https://github.com/clickup/ent-framework/blob/master/src/pg/types/BigIntArrayType.ts#L19) + +An array of IDs. Notice that: +1. Node-postgres natively supports this type on read path, but on write path, + we have to stringify by ourselves. +2. GIN index doesn't support NULL, because PG's "&&" operator (intersection + check) doesn't work with NULLs. But we still allow NULLs in + BigIntArrayType, since to query such values, the user could use a separate + partial index. + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `T` *extends* `null` \| `string` | `null` \| `string` | + +## Returns + +`object` + +### dbValueToJs() + +> **dbValueToJs**: (`dbValue`) => `T`[] + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `dbValue` | `T`[] | + +#### Returns + +`T`[] + +### stringify() + +> **stringify**: (`jsValue`) => `string` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `jsValue` | `T`[] | + +#### Returns + +`string` + +### parse() + +> **parse**: (`str`) => `T`[] + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `str` | `string` | + +#### Returns + +`T`[] diff --git a/docs/functions/ByteaBufferType.md b/docs/functions/ByteaBufferType.md new file mode 100644 index 0000000..c82cbd2 --- /dev/null +++ b/docs/functions/ByteaBufferType.md @@ -0,0 +1,59 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ByteaBufferType + +# Function: ByteaBufferType() + +> **ByteaBufferType**(): `object` + +Defined in: [src/pg/types/ByteaBufferType.ts:6](https://github.com/clickup/ent-framework/blob/master/src/pg/types/ByteaBufferType.ts#L6) + +A value stored in the DB as a bytea buffer. + +## Returns + +`object` + +### dbValueToJs() + +> **dbValueToJs**: (`dbValue`) => `Buffer` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `dbValue` | `Buffer` | + +#### Returns + +`Buffer` + +### stringify() + +> **stringify**: (`jsValue`) => `string` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `jsValue` | `Buffer` | + +#### Returns + +`string` + +### parse() + +> **parse**: (`str`) => `Buffer` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `str` | `string` | + +#### Returns + +`Buffer` diff --git a/docs/functions/CacheMixin.md b/docs/functions/CacheMixin.md new file mode 100644 index 0000000..771509c --- /dev/null +++ b/docs/functions/CacheMixin.md @@ -0,0 +1,31 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / CacheMixin + +# Function: CacheMixin() + +> **CacheMixin**\<`TTable`, `TUniqueKey`, `TClient`\>(`Base`): [`PrimitiveClass`](../type-aliases/PrimitiveClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> + +Defined in: [src/ent/mixins/CacheMixin.ts:31](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/CacheMixin.ts#L31) + +Modifies the passed class adding VC-stored cache layer to it. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | +| `TClient` *extends* [`Client`](../classes/Client.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `Base` | [`PrimitiveClass`](../type-aliases/PrimitiveClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> | + +## Returns + +[`PrimitiveClass`](../type-aliases/PrimitiveClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> diff --git a/docs/functions/ConfigMixin.md b/docs/functions/ConfigMixin.md new file mode 100644 index 0000000..8dfe22a --- /dev/null +++ b/docs/functions/ConfigMixin.md @@ -0,0 +1,34 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ConfigMixin + +# Function: ConfigMixin() + +> **ConfigMixin**\<`TTable`, `TUniqueKey`, `TClient`\>(`Base`, `cluster`, `schema`): [`ConfigClass`](../interfaces/ConfigClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> + +Defined in: [src/ent/mixins/ConfigMixin.ts:86](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/ConfigMixin.ts#L86) + +Modifies the passed class adding support for Ent configuration (such as: +Cluster, table schema, privacy rules, triggers etc.). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | +| `TClient` *extends* [`Client`](../classes/Client.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `Base` | () => `object` | +| `cluster` | [`Cluster`](../classes/Cluster.md)\<`TClient`, `any`\> | +| `schema` | [`Schema`](../classes/Schema.md)\<`TTable`, `TUniqueKey`\> | + +## Returns + +[`ConfigClass`](../interfaces/ConfigClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> diff --git a/docs/functions/HelpersMixin.md b/docs/functions/HelpersMixin.md new file mode 100644 index 0000000..9f1a1ac --- /dev/null +++ b/docs/functions/HelpersMixin.md @@ -0,0 +1,33 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / HelpersMixin + +# Function: HelpersMixin() + +> **HelpersMixin**\<`TTable`, `TUniqueKey`, `TClient`\>(`Base`): [`HelpersClass`](../interfaces/HelpersClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> + +Defined in: [src/ent/mixins/HelpersMixin.ts:139](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/HelpersMixin.ts#L139) + +Modifies the passed class adding convenience methods (like loadX() which +throws when an Ent can't be loaded instead of returning null as it's done in +the primitive operations). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | +| `TClient` *extends* [`Client`](../classes/Client.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `Base` | [`PrimitiveClass`](../type-aliases/PrimitiveClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> | + +## Returns + +[`HelpersClass`](../interfaces/HelpersClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> diff --git a/docs/functions/JSONType.md b/docs/functions/JSONType.md new file mode 100644 index 0000000..d71d3a6 --- /dev/null +++ b/docs/functions/JSONType.md @@ -0,0 +1,65 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / JSONType + +# Function: JSONType() + +> **JSONType**\<`TCurrent`\>(): `object` + +Defined in: [src/types.ts:418](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L418) + +An arbitrary JSON field type. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TCurrent` *extends* [`JSONValue`](../type-aliases/JSONValue.md) | + +## Returns + +`object` + +### dbValueToJs() + +> **dbValueToJs**: (`dbValue`) => `TCurrent` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `dbValue` | `TCurrent` | + +#### Returns + +`TCurrent` + +### stringify() + +> **stringify**: (`jsValue`) => `string` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `jsValue` | `TCurrent` | + +#### Returns + +`string` + +### parse() + +> **parse**: (`str`) => `TCurrent` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `str` | `string` | + +#### Returns + +`TCurrent` diff --git a/docs/functions/PrimitiveMixin.md b/docs/functions/PrimitiveMixin.md new file mode 100644 index 0000000..4ceeac2 --- /dev/null +++ b/docs/functions/PrimitiveMixin.md @@ -0,0 +1,32 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PrimitiveMixin + +# Function: PrimitiveMixin() + +> **PrimitiveMixin**\<`TTable`, `TUniqueKey`, `TClient`\>(`Base`): [`PrimitiveClass`](../type-aliases/PrimitiveClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> + +Defined in: [src/ent/mixins/PrimitiveMixin.ts:197](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/PrimitiveMixin.ts#L197) + +Modifies the passed class adding support for the minimal number of basic Ent +operations. Internally, uses Schema abstractions to run them. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | +| `TClient` *extends* [`Client`](../classes/Client.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `Base` | [`ConfigClass`](../interfaces/ConfigClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> | + +## Returns + +[`PrimitiveClass`](../type-aliases/PrimitiveClass.md)\<`TTable`, `TUniqueKey`, `TClient`\> diff --git a/docs/functions/StringArrayType.md b/docs/functions/StringArrayType.md new file mode 100644 index 0000000..ac6f1c7 --- /dev/null +++ b/docs/functions/StringArrayType.md @@ -0,0 +1,66 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / StringArrayType + +# Function: StringArrayType() + +> **StringArrayType**\<`T`\>(): `object` + +Defined in: [src/pg/types/StringArrayType.ts:14](https://github.com/clickup/ent-framework/blob/master/src/pg/types/StringArrayType.ts#L14) + +An array of Strings. Note: node-postgres natively supports this type on read +path, but on write path, we have to stringify by ourselves. + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `T` *extends* `null` \| `string` | `null` \| `string` | + +## Returns + +`object` + +### dbValueToJs() + +> **dbValueToJs**: (`dbValue`) => `T`[] + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `dbValue` | `T`[] | + +#### Returns + +`T`[] + +### stringify() + +> **stringify**: (`jsValue`) => `string` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `jsValue` | `T`[] | + +#### Returns + +`string` + +### parse() + +> **parse**: (`str`) => `T`[] + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `str` | `string` | + +#### Returns + +`T`[] diff --git a/docs/functions/buildShape.md b/docs/functions/buildShape.md new file mode 100644 index 0000000..aa4347f --- /dev/null +++ b/docs/functions/buildShape.md @@ -0,0 +1,24 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / buildShape + +# Function: buildShape() + +> **buildShape**(`sql`): `string` + +Defined in: [src/pg/helpers/buildShape.ts:52](https://github.com/clickup/ent-framework/blob/master/src/pg/helpers/buildShape.ts#L52) + +Extracts a "shape" from some commonly built SQL queries. This function may be +used from the outside for logging/debugging, so it's here, not in __tests__. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `sql` | `string` | + +## Returns + +`string` diff --git a/docs/functions/escapeIdent.md b/docs/functions/escapeIdent.md new file mode 100644 index 0000000..6c7cd77 --- /dev/null +++ b/docs/functions/escapeIdent.md @@ -0,0 +1,23 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / escapeIdent + +# Function: escapeIdent() + +> **escapeIdent**(`ident`): `string` + +Defined in: [src/pg/helpers/escapeIdent.ts:4](https://github.com/clickup/ent-framework/blob/master/src/pg/helpers/escapeIdent.ts#L4) + +Optionally encloses a PG identifier (like table name) in "". + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `ident` | `string` | + +## Returns + +`string` diff --git a/docs/functions/escapeLiteral.md b/docs/functions/escapeLiteral.md new file mode 100644 index 0000000..bd1f1b5 --- /dev/null +++ b/docs/functions/escapeLiteral.md @@ -0,0 +1,29 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / escapeLiteral + +# Function: escapeLiteral() + +> **escapeLiteral**(`literal`): `string` + +Defined in: [src/pg/helpers/escapeLiteral.ts:15](https://github.com/clickup/ent-framework/blob/master/src/pg/helpers/escapeLiteral.ts#L15) + +Builds a part of SQL query using ?-placeholders to prevent SQL Injection. +Everywhere where we want to accept a piece of SQL, we should instead accept a +Literal tuple. + +The function converts a Literal tuple [fmt, ...args] into a string, escaping +the args and interpolating them into the format SQL where "?" is a +placeholder for the replacing value. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `literal` | [`Literal`](../type-aliases/Literal.md) | + +## Returns + +`string` diff --git a/docs/functions/evaluate.md b/docs/functions/evaluate.md new file mode 100644 index 0000000..3143f2f --- /dev/null +++ b/docs/functions/evaluate.md @@ -0,0 +1,72 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / evaluate + +# Function: evaluate() + +> **evaluate**\<`TInput`\>(`vc`, `input`, `rules`, `fashion`): `Promise`\<\{ `allow`: `boolean`; `results`: [`RuleResult`](../interfaces/RuleResult.md)[]; `cause`: `string`; \}\> + +Defined in: [src/ent/rules/evaluate.ts:49](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/evaluate.ts#L49) + +This is a hearth of permissions checking, a machine which evaluates the rules +chain from top to bottom (one after another) and makes the decision based on +the following logic: +- ALLOW immediately allows the chain, the rest of the rules are not checked. + It's an eager allowance. +- DENY immediately denies the chain, the rest of the rules are not checked. + It's an eager denial. +- TOLERATE delegates the decision to the next rules; if it's the last + decision in the chain, then allows the chain. I.e. it's like an allowance, + but only if everyone else is tolerant. +- SKIP also delegates the decision to the next rules, but if it's the last + rule in the chain (i.e. nothing to skip to anymore), denies the chain. I.e. + it's "I don't vote here, please ask others". +- An empty chain is always denied. + +Having TOLERATE decision may sound superfluous, but unfortunately it's not. +The TOLERATE enables usage of the same machinery for both read-like checks +(where we typically want ANY of the rules to be okay with the row) and for +write-like checks (where we typically want ALL rules to be okay with the +row). Having the same logic for everything simplifies the code. + +If parallel argument is true, all the rules are run at once in concurrent +promises before the machine starts. This doesn't affect the final result, +just speeds up processing if we know that there is a high chance that most of +the rules will likely return TOLERATE and we'll anyway need to evaluate all +of them (e.g. most of the rules are Require, like in write operations). As +opposed, for read operation, there is a high chance for the first rule (which +is often AllowIf) to succeed, so we evaluate the rules sequentially, not in +parallel (to minimize the number of DB queries). + +Example of a chain (the order of rules always matters!): +- new Require(new OutgoingEdgePointsToVC("user_id")) +- new Require(new CanReadOutgoingEdge("post_id", EntPost)) + +Example of a chain: +- new AllowIf(new OutgoingEdgePointsToVC("user_id")) +- new AllowIf(new CanReadOutgoingEdge("post_id", EntPost)) + +Example of a chain: +- new DenyIf(new UserIsPendingApproval()) +- new AllowIf(new OutgoingEdgePointsToVC("user_id")) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` *extends* `object` | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `input` | `TInput` | +| `rules` | [`Rule`](../classes/Rule.md)\<`TInput`\>[] | +| `fashion` | `"parallel"` \| `"sequential"` | + +## Returns + +`Promise`\<\{ `allow`: `boolean`; `results`: [`RuleResult`](../interfaces/RuleResult.md)[]; `cause`: `string`; \}\> diff --git a/docs/functions/isBigintStr.md b/docs/functions/isBigintStr.md new file mode 100644 index 0000000..a00bb32 --- /dev/null +++ b/docs/functions/isBigintStr.md @@ -0,0 +1,24 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / isBigintStr + +# Function: isBigintStr() + +> **isBigintStr**(`str`): `boolean` + +Defined in: [src/helpers/isBigintStr.ts:8](https://github.com/clickup/ent-framework/blob/master/src/helpers/isBigintStr.ts#L8) + +It's hard to support PG bigint type in JS, so people use strings instead. +This function checks that a string can be passed to PG as a bigint. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `str` | `string` | + +## Returns + +`boolean` diff --git a/docs/functions/testSpecTypeIntegrity.md b/docs/functions/testSpecTypeIntegrity.md new file mode 100644 index 0000000..f6d1c99 --- /dev/null +++ b/docs/functions/testSpecTypeIntegrity.md @@ -0,0 +1,45 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / testSpecTypeIntegrity + +# Function: testSpecTypeIntegrity() + +> **testSpecTypeIntegrity**\<`TDBValue`, `TJsValue`\>(`SpecType`, `dbValue`): `object` + +Defined in: [src/helpers/testSpecTypeIntegrity.ts:9](https://github.com/clickup/ent-framework/blob/master/src/helpers/testSpecTypeIntegrity.ts#L9) + +A tool to verify integrity of custom field types. It is meant to be called +from Jest expect(). The helper runs dbValueToJs, stringify and parse methods +on the type and makes sure that parse() is the opposite of stringify(). The +returned object can then be compared against a Jest snapshot. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TDBValue` | +| `TJsValue` | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `SpecType` | \{ `dbValueToJs`: (`dbValue`) => `TJsValue`; `stringify`: (`jsValue`) => `string`; `parse`: (`str`) => `TJsValue`; \} | +| `SpecType.dbValueToJs` | (`dbValue`) => `TJsValue` | +| `SpecType.stringify` | (`jsValue`) => `string` | +| `SpecType.parse` | (`str`) => `TJsValue` | +| `dbValue` | `TDBValue` | + +## Returns + +`object` + +### jsValueDecoded + +> **jsValueDecoded**: `TJsValue` + +### stringifiedBack + +> **stringifiedBack**: `string` diff --git a/docs/globals.md b/docs/globals.md new file mode 100644 index 0000000..cf1c357 --- /dev/null +++ b/docs/globals.md @@ -0,0 +1,219 @@ +[**ent-framework**](README.md) + +*** + +# ent-framework + +## Classes + +- [Batcher](classes/Batcher.md) +- [Client](classes/Client.md) +- [ClientError](classes/ClientError.md) +- [Cluster](classes/Cluster.md) +- [Island](classes/Island.md) +- [Loader](classes/Loader.md) +- [LocalCache](classes/LocalCache.md) +- [QueryBase](classes/QueryBase.md) +- [QueryPing](classes/QueryPing.md) +- [Runner](classes/Runner.md) +- [Schema](classes/Schema.md) +- [Shard](classes/Shard.md) +- [ShardError](classes/ShardError.md) +- [ShardIsNotDiscoverableError](classes/ShardIsNotDiscoverableError.md) +- [ShardNamer](classes/ShardNamer.md) +- [Timeline](classes/Timeline.md) +- [TimelineManager](classes/TimelineManager.md) +- [Configuration](classes/Configuration.md) +- [IDsCache](classes/IDsCache.md) +- [Inverse](classes/Inverse.md) +- [QueryCache](classes/QueryCache.md) +- [ShardLocator](classes/ShardLocator.md) +- [TimelineStorage](classes/TimelineStorage.md) +- [Triggers](classes/Triggers.md) +- [VC](classes/VC.md) +- [VCCaches](classes/VCCaches.md) +- [VCFlavor](classes/VCFlavor.md) +- [VCWithStacks](classes/VCWithStacks.md) +- [VCWithQueryCache](classes/VCWithQueryCache.md) +- [VCTrace](classes/VCTrace.md) +- [Validation](classes/Validation.md) +- [EntAccessError](classes/EntAccessError.md) +- [EntNotFoundError](classes/EntNotFoundError.md) +- [EntNotInsertableError](classes/EntNotInsertableError.md) +- [EntNotReadableError](classes/EntNotReadableError.md) +- [EntNotUpdatableError](classes/EntNotUpdatableError.md) +- [EntUniqueKeyError](classes/EntUniqueKeyError.md) +- [EntValidationError](classes/EntValidationError.md) +- [CanDeleteOutgoingEdge](classes/CanDeleteOutgoingEdge.md) +- [CanReadOutgoingEdge](classes/CanReadOutgoingEdge.md) +- [CanUpdateOutgoingEdge](classes/CanUpdateOutgoingEdge.md) +- [FieldIs](classes/FieldIs.md) +- [IncomingEdgeFromVCExists](classes/IncomingEdgeFromVCExists.md) +- [Or](classes/Or.md) +- [OutgoingEdgePointsToVC](classes/OutgoingEdgePointsToVC.md) +- [FuncToPredicate](classes/FuncToPredicate.md) +- [IDsCacheReadable](classes/IDsCacheReadable.md) +- [IDsCacheUpdatable](classes/IDsCacheUpdatable.md) +- [IDsCacheDeletable](classes/IDsCacheDeletable.md) +- [IDsCacheCanReadIncomingEdge](classes/IDsCacheCanReadIncomingEdge.md) +- [RowIs](classes/RowIs.md) +- [True](classes/True.md) +- [VCHasFlavor](classes/VCHasFlavor.md) +- [AllowIf](classes/AllowIf.md) +- [DenyIf](classes/DenyIf.md) +- [Require](classes/Require.md) +- [Rule](classes/Rule.md) +- [PgClient](classes/PgClient.md) +- [PgError](classes/PgError.md) +- [PgQueryCount](classes/PgQueryCount.md) +- [PgQueryDelete](classes/PgQueryDelete.md) +- [PgQueryDeleteWhere](classes/PgQueryDeleteWhere.md) +- [PgQueryExists](classes/PgQueryExists.md) +- [PgQueryIDGen](classes/PgQueryIDGen.md) +- [PgQueryInsert](classes/PgQueryInsert.md) +- [PgQueryLoad](classes/PgQueryLoad.md) +- [PgQueryLoadBy](classes/PgQueryLoadBy.md) +- [PgQuerySelect](classes/PgQuerySelect.md) +- [PgQuerySelectBy](classes/PgQuerySelectBy.md) +- [PgQueryUpdate](classes/PgQueryUpdate.md) +- [PgQueryUpsert](classes/PgQueryUpsert.md) +- [PgRunner](classes/PgRunner.md) +- [PgSchema](classes/PgSchema.md) +- [PgShardNamer](classes/PgShardNamer.md) +- [PgTimelineStorage](classes/PgTimelineStorage.md) +- [ToolPing](classes/ToolPing.md) +- [ToolScoreboard](classes/ToolScoreboard.md) + +## Interfaces + +- [ClientOptions](interfaces/ClientOptions.md) +- [ClientConnectionIssue](interfaces/ClientConnectionIssue.md) +- [ClientPingInput](interfaces/ClientPingInput.md) +- [ClusterOptions](interfaces/ClusterOptions.md) +- [IslandOptions](interfaces/IslandOptions.md) +- [Handler](interfaces/Handler.md) +- [LocalCacheOptions](interfaces/LocalCacheOptions.md) +- [Loggers](interfaces/Loggers.md) +- [ClientQueryLoggerProps](interfaces/ClientQueryLoggerProps.md) +- [SwallowedErrorLoggerProps](interfaces/SwallowedErrorLoggerProps.md) +- [RunOnShardErrorLoggerProps](interfaces/RunOnShardErrorLoggerProps.md) +- [ClientEndLoggerProps](interfaces/ClientEndLoggerProps.md) +- [ClientConnectedLoggerProps](interfaces/ClientConnectedLoggerProps.md) +- [Query](interfaces/Query.md) +- [QueryAnnotation](interfaces/QueryAnnotation.md) +- [SchemaClass](interfaces/SchemaClass.md) +- [ShardNamerOptions](interfaces/ShardNamerOptions.md) +- [TimelineStorageOptions](interfaces/TimelineStorageOptions.md) +- [StandardSchemaV1FailureResult](interfaces/StandardSchemaV1FailureResult.md) +- [EntValidationErrorInfo](interfaces/EntValidationErrorInfo.md) +- [ConfigInstance](interfaces/ConfigInstance.md) +- [ConfigClass](interfaces/ConfigClass.md) +- [HelpersInstance](interfaces/HelpersInstance.md) +- [HelpersClass](interfaces/HelpersClass.md) +- [PrimitiveInstance](interfaces/PrimitiveInstance.md) +- [AbstractIs](interfaces/AbstractIs.md) +- [Predicate](interfaces/Predicate.md) +- [RuleResult](interfaces/RuleResult.md) +- [EntClass](interfaces/EntClass.md) +- [Ent](interfaces/Ent.md) +- [PgClientOptions](interfaces/PgClientOptions.md) +- [PgClientConn](interfaces/PgClientConn.md) +- [PgClientSubPoolConfig](interfaces/PgClientSubPoolConfig.md) +- [PgTimelineStorageOptions](interfaces/PgTimelineStorageOptions.md) +- [ToolPingOptions](interfaces/ToolPingOptions.md) +- [ToolScoreboardOptions](interfaces/ToolScoreboardOptions.md) + +## Type Aliases + +- [ClientRole](type-aliases/ClientRole.md) +- [ClientErrorPostAction](type-aliases/ClientErrorPostAction.md) +- [ClientErrorKind](type-aliases/ClientErrorKind.md) +- [ClusterIslands](type-aliases/ClusterIslands.md) +- [WhyClient](type-aliases/WhyClient.md) +- [TimelineCaughtUpReason](type-aliases/TimelineCaughtUpReason.md) +- [AnyClass](type-aliases/AnyClass.md) +- [ShardAffinity](type-aliases/ShardAffinity.md) +- [TriggerInsertInput](type-aliases/TriggerInsertInput.md) +- [TriggerUpdateInput](type-aliases/TriggerUpdateInput.md) +- [TriggerUpdateNewRow](type-aliases/TriggerUpdateNewRow.md) +- [TriggerUpdateOrDeleteOldRow](type-aliases/TriggerUpdateOrDeleteOldRow.md) +- [InsertTrigger](type-aliases/InsertTrigger.md) +- [BeforeUpdateTrigger](type-aliases/BeforeUpdateTrigger.md) +- [AfterUpdateTrigger](type-aliases/AfterUpdateTrigger.md) +- [DeleteTrigger](type-aliases/DeleteTrigger.md) +- [BeforeMutationTrigger](type-aliases/BeforeMutationTrigger.md) +- [AfterMutationTrigger](type-aliases/AfterMutationTrigger.md) +- [DepsBuilder](type-aliases/DepsBuilder.md) +- [LoadRule](type-aliases/LoadRule.md) +- [WriteRules](type-aliases/WriteRules.md) +- [ValidationRules](type-aliases/ValidationRules.md) +- [PrimitiveClass](type-aliases/PrimitiveClass.md) +- [ValidatorPlainResult](type-aliases/ValidatorPlainResult.md) +- [ValidatorZodSafeParseResult](type-aliases/ValidatorZodSafeParseResult.md) +- [ValidatorStandardSchemaResult](type-aliases/ValidatorStandardSchemaResult.md) +- [FieldIsValidatorPlain](type-aliases/FieldIsValidatorPlain.md) +- [FieldIsValidatorZodSafeParse](type-aliases/FieldIsValidatorZodSafeParse.md) +- [FieldIsValidatorStandardSchemaV1](type-aliases/FieldIsValidatorStandardSchemaV1.md) +- [RowIsValidatorPlain](type-aliases/RowIsValidatorPlain.md) +- [RowIsValidatorZodSafeParse](type-aliases/RowIsValidatorZodSafeParse.md) +- [RowIsValidatorStandardSchemaV1](type-aliases/RowIsValidatorStandardSchemaV1.md) +- [RuleDecision](type-aliases/RuleDecision.md) +- [UpdateOriginalInput](type-aliases/UpdateOriginalInput.md) +- [SelectInputCustom](type-aliases/SelectInputCustom.md) +- [Literal](type-aliases/Literal.md) +- [RowWithID](type-aliases/RowWithID.md) +- [SpecType](type-aliases/SpecType.md) +- [Spec](type-aliases/Spec.md) +- [Table](type-aliases/Table.md) +- [Field](type-aliases/Field.md) +- [FieldAliased](type-aliases/FieldAliased.md) +- [FieldOfPotentialUniqueKey](type-aliases/FieldOfPotentialUniqueKey.md) +- [FieldOfIDType](type-aliases/FieldOfIDType.md) +- [FieldOfIDTypeRequired](type-aliases/FieldOfIDTypeRequired.md) +- [ValueRequired](type-aliases/ValueRequired.md) +- [Value](type-aliases/Value.md) +- [Row](type-aliases/Row.md) +- [InsertFieldsRequired](type-aliases/InsertFieldsRequired.md) +- [InsertFieldsOptional](type-aliases/InsertFieldsOptional.md) +- [InsertInput](type-aliases/InsertInput.md) +- [UpdateField](type-aliases/UpdateField.md) +- [UpdateInput](type-aliases/UpdateInput.md) +- [UniqueKey](type-aliases/UniqueKey.md) +- [LoadByInput](type-aliases/LoadByInput.md) +- [SelectByInput](type-aliases/SelectByInput.md) +- [Where](type-aliases/Where.md) +- [Order](type-aliases/Order.md) +- [SelectInput](type-aliases/SelectInput.md) +- [CountInput](type-aliases/CountInput.md) +- [ExistsInput](type-aliases/ExistsInput.md) +- [DeleteWhereInput](type-aliases/DeleteWhereInput.md) +- [Hints](type-aliases/Hints.md) +- [JSONValue](type-aliases/JSONValue.md) + +## Variables + +- [MASTER](variables/MASTER.md) +- [STALE\_REPLICA](variables/STALE_REPLICA.md) +- [GLOBAL\_SHARD](variables/GLOBAL_SHARD.md) +- [GUEST\_ID](variables/GUEST_ID.md) +- [OMNI\_ID](variables/OMNI_ID.md) +- [ID](variables/ID.md) + +## Functions + +- [BaseEnt](functions/BaseEnt.md) +- [CacheMixin](functions/CacheMixin.md) +- [ConfigMixin](functions/ConfigMixin.md) +- [HelpersMixin](functions/HelpersMixin.md) +- [PrimitiveMixin](functions/PrimitiveMixin.md) +- [evaluate](functions/evaluate.md) +- [isBigintStr](functions/isBigintStr.md) +- [testSpecTypeIntegrity](functions/testSpecTypeIntegrity.md) +- [buildShape](functions/buildShape.md) +- [escapeIdent](functions/escapeIdent.md) +- [escapeLiteral](functions/escapeLiteral.md) +- [BigIntArrayType](functions/BigIntArrayType.md) +- [ByteaBufferType](functions/ByteaBufferType.md) +- [StringArrayType](functions/StringArrayType.md) +- [Base64BufferType](functions/Base64BufferType.md) +- [JSONType](functions/JSONType.md) diff --git a/docs/interfaces/AbstractIs.md b/docs/interfaces/AbstractIs.md new file mode 100644 index 0000000..256b4c8 --- /dev/null +++ b/docs/interfaces/AbstractIs.md @@ -0,0 +1,52 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / AbstractIs + +# Interface: AbstractIs\ + +Defined in: [src/ent/predicates/AbstractIs.ts:6](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/AbstractIs.ts#L6) + +A base interface for all user validation predicates. + +## Extends + +- [`Predicate`](Predicate.md)\<`TRow`\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TRow` | + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `field` | `null` \| `string` | The field this validation predicate is related to (null means that it applies to the entire Ent). | +| `message` | `null` \| `string` | In case the predicate returns false or doesn't provide error messages by throwing EntValidationError, this message will be used. When message is null, it means that we expect the validator to return detailed information about each field errored (e.g. ValidatorStandardSchemaResult). | +| `name` | `string` | - | + +## Methods + +### check() + +> **check**(`vc`, `input`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/Predicate.ts:19](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L19) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `input` | `TRow` | + +#### Returns + +`Promise`\<`boolean`\> + +#### Inherited from + +[`Predicate`](Predicate.md).[`check`](Predicate.md#check) diff --git a/docs/interfaces/ClientConnectedLoggerProps.md b/docs/interfaces/ClientConnectedLoggerProps.md new file mode 100644 index 0000000..f9806df --- /dev/null +++ b/docs/interfaces/ClientConnectedLoggerProps.md @@ -0,0 +1,21 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientConnectedLoggerProps + +# Interface: ClientConnectedLoggerProps\ + +Defined in: [src/abstract/Loggers.ts:78](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loggers.ts#L78) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TNode` | + +## Properties + +| Property | Type | +| ------ | ------ | +| `node` | `TNode` | diff --git a/docs/interfaces/ClientConnectionIssue.md b/docs/interfaces/ClientConnectionIssue.md new file mode 100644 index 0000000..b08f44a --- /dev/null +++ b/docs/interfaces/ClientConnectionIssue.md @@ -0,0 +1,21 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientConnectionIssue + +# Interface: ClientConnectionIssue + +Defined in: [src/abstract/Client.ts:45](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L45) + +An information about Client's connection related issue. + +## Properties + +| Property | Type | +| ------ | ------ | +| `timestamp` | `Date` | +| `cause` | `unknown` | +| `postAction` | [`ClientErrorPostAction`](../type-aliases/ClientErrorPostAction.md) | +| `kind` | [`ClientErrorKind`](../type-aliases/ClientErrorKind.md) | +| `comment` | `string` | diff --git a/docs/interfaces/ClientEndLoggerProps.md b/docs/interfaces/ClientEndLoggerProps.md new file mode 100644 index 0000000..d5713bf --- /dev/null +++ b/docs/interfaces/ClientEndLoggerProps.md @@ -0,0 +1,23 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientEndLoggerProps + +# Interface: ClientEndLoggerProps\ + +Defined in: [src/abstract/Loggers.ts:72](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loggers.ts#L72) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TNode` | + +## Properties + +| Property | Type | +| ------ | ------ | +| `client` | [`Client`](../classes/Client.md) | +| `key` | `string` | +| `node` | `TNode` | diff --git a/docs/interfaces/ClientOptions.md b/docs/interfaces/ClientOptions.md new file mode 100644 index 0000000..ca65dcb --- /dev/null +++ b/docs/interfaces/ClientOptions.md @@ -0,0 +1,24 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientOptions + +# Interface: ClientOptions + +Defined in: [src/abstract/Client.ts:18](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L18) + +Options for Client constructor. + +## Extended by + +- [`PgClientOptions`](PgClientOptions.md) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `name` | `string` | Name of the Client; used for logging. | +| `shardNamer?` | `null` \| [`ShardNamer`](../classes/ShardNamer.md) | Info on how to build/parse Shard names. If not set, then Cluster injects its own ShardNamer here right after creating a Client instance. | +| `loggers?` | `null` \| [`Loggers`](Loggers.md)\<`any`\> | Loggers to be called at different stages. Client code calls into them. Also, Cluster injects its own loggers here, in addition to the provided ones (if any). | +| `batchDelayMs?` | `MaybeCallable`\<`number`\> | If passed, there will be an artificial queries accumulation delay while batching the requests. Default is 0 (turned off). Passed to Batcher#batchDelayMs. | diff --git a/docs/interfaces/ClientPingInput.md b/docs/interfaces/ClientPingInput.md new file mode 100644 index 0000000..67b3579 --- /dev/null +++ b/docs/interfaces/ClientPingInput.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientPingInput + +# Interface: ClientPingInput + +Defined in: [src/abstract/Client.ts:56](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L56) + +Input for Client#ping(). + +## Properties + +| Property | Type | +| ------ | ------ | +| `execTimeMs` | `number` | +| `isWrite` | `boolean` | +| `annotation` | [`QueryAnnotation`](QueryAnnotation.md) | diff --git a/docs/interfaces/ClientQueryLoggerProps.md b/docs/interfaces/ClientQueryLoggerProps.md new file mode 100644 index 0000000..166840b --- /dev/null +++ b/docs/interfaces/ClientQueryLoggerProps.md @@ -0,0 +1,35 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientQueryLoggerProps + +# Interface: ClientQueryLoggerProps + +Defined in: [src/abstract/Loggers.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loggers.ts#L27) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `annotations` | [`QueryAnnotation`](QueryAnnotation.md)[] | - | +| `op` | `string` | - | +| `shard` | `string` | - | +| `table` | `string` | - | +| `batchFactor` | `number` | - | +| `msg` | `string` | - | +| `output` | `unknown` | - | +| `elapsed` | `object` | - | +| `elapsed.total` | `number` | - | +| `elapsed.acquire` | `number` | - | +| `connStats` | `object` | - | +| `connStats.id` | `string` | The stats related to the used connection. | +| `connStats.queriesSent` | `number` | The number of queries sent to the connection. | +| `poolStats` | `object` | - | +| `poolStats.totalConns` | `number` | Total number of connections in the pool. | +| `poolStats.idleConns` | `number` | Connections not busy running a query. | +| `poolStats.queuedReqs` | `number` | Once all idle connections are over, requests are queued waiting for a new available connection. This is the number of such queued requests. | +| `error` | `undefined` \| `string` | - | +| `role` | [`ClientRole`](../type-aliases/ClientRole.md) | - | +| `backend` | `string` | - | +| `address` | `string` | - | diff --git a/docs/interfaces/ClusterOptions.md b/docs/interfaces/ClusterOptions.md new file mode 100644 index 0000000..e4d7662 --- /dev/null +++ b/docs/interfaces/ClusterOptions.md @@ -0,0 +1,35 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClusterOptions + +# Interface: ClusterOptions\ + +Defined in: [src/abstract/Cluster.ts:45](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L45) + +Options for Cluster constructor. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TClient` *extends* [`Client`](../classes/Client.md) | +| `TNode` | + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `islands` | `MaybeAsyncCallable`\<[`ClusterIslands`](../type-aliases/ClusterIslands.md)\<`TNode`\>\> | Islands configuration of the Cluster. | +| `createClient` | (`node`: `TNode`) => `TClient` | Given a node of some Island, instantiates a Client for this node. Called when a new node appears in the Cluster statically or dynamically. | +| `loggers` | [`Loggers`](Loggers.md)\<`any`\> | Loggers to be injected into all Clients returned by createClient(). | +| `localCache?` | `null` \| [`LocalCache`](../classes/LocalCache.md)\<`never`\> | An instance of LocalCache which may be used for auxiliary purposes when discovering Shards/Clients. | +| `reloadIslandsIntervalMs?` | `MaybeCallable`\<`number`\> | How often to recheck for changes in `options.islands`. If it is SYNC, then by default - often, like every 500 ms (since it's assumed that `options.islands` calculation is cheap). If it is ASYNC, then by default - not so often, every `shardsDiscoverIntervalMs` (we assume that getting the list of Island nodes may be expensive, e.g. fetching from AWS API or so). If the Islands list here changes, then we trigger Shards rediscovery and Clients recreation ASAP. | +| `shardNamer?` | `null` \| [`ShardNamer`](../classes/ShardNamer.md) | Info on how to build/parse Shard names. | +| `shardsDiscoverIntervalMs?` | `MaybeCallable`\<`number`\> | How often to run Shards rediscovery in normal circumstances. | +| `shardsDiscoverIntervalJitter?` | `MaybeCallable`\<`number`\> | Jitter for shardsDiscoverIntervalMs and reloadIslandsIntervalMs. | +| `runOnShardErrorRetryCount?` | `MaybeCallable`\<`number`\> | Used in the following situations: 1. If we think that we know the Island of a particular Shard, but an attempt to access it fails. This means that maybe the Shard is migrating to another Island. So, we wait a bit and retry that many times. We should not do it too many times though, because all DB requests will be blocked waiting for the resolution. 2. If we sent a WRITE request to a Client, but it appeared that this Client is a replica, and the master moved to some other Client. In this case, we wait a bit and ping all Clients of the Island to refresh, who is master and who is replica. | +| `runOnShardErrorRediscoverClusterDelayMs?` | `MaybeCallable`\<`number`\> | How much time to wait before we retry rediscovering the entire Cluster after a Shard-to-Island resolution error. The time here should be just enough to wait for switching the Shard from one Island to another (typically quick). | +| `runOnShardErrorRediscoverIslandDelayMs?` | `MaybeCallable`\<`number`\> | How much time to wait before sending discover requests to all Clients of the Island trying to find the new master (or to reconnect). The time here may reach several seconds, since some DBs shut down the old master and promote some replica to it not simultaneously. | +| `clientEndDelayMs?` | `MaybeCallable`\<`number`\> | Delay the Client ending when this Client got removed from the Cluster. This allows to lower the number of client_is_ended errors in case a Client instance gets captured somewhere in the application code. | diff --git a/docs/interfaces/ConfigClass.md b/docs/interfaces/ConfigClass.md new file mode 100644 index 0000000..0e3aa52 --- /dev/null +++ b/docs/interfaces/ConfigClass.md @@ -0,0 +1,63 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ConfigClass + +# Interface: ConfigClass\ + +Defined in: [src/ent/mixins/ConfigMixin.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/ConfigMixin.ts#L17) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | +| `TClient` *extends* [`Client`](../classes/Client.md) | + +## Constructors + +### new ConfigClass() + +> **new ConfigClass**(): [`ConfigInstance`](ConfigInstance.md) + +Defined in: [src/ent/mixins/ConfigMixin.ts:79](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/ConfigMixin.ts#L79) + +TS requires us to have a public constructor to infer instance types in +various places. We make this constructor throw if it's called. + +#### Returns + +[`ConfigInstance`](ConfigInstance.md) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `Configuration` | (`cfg`: [`Configuration`](../classes/Configuration.md)\<`TTable`\>) => [`Configuration`](../classes/Configuration.md)\<`TTable`\> | A helper class to work-around TS weakness in return value type inference: https://github.com/Microsoft/TypeScript/issues/31273. It could've been just a function, but having a class is a little more natural. | +| `CLUSTER` | [`Cluster`](../classes/Cluster.md)\<`TClient`, `any`\> | A Cluster where this Ent lives. | +| `SCHEMA` | [`Schema`](../classes/Schema.md)\<`TTable`, `TUniqueKey`\> | A schema which represents this Ent. | +| `SHARD_AFFINITY` | [`ShardAffinity`](../type-aliases/ShardAffinity.md)\<[`FieldOfIDType`](../type-aliases/FieldOfIDType.md)\<`TTable`\>\> | Defines how to find the right Shard during Ent insertion. | +| `SHARD_LOCATOR` | [`ShardLocator`](../classes/ShardLocator.md)\<`TClient`, `TTable`, [`FieldOfIDType`](../type-aliases/FieldOfIDType.md)\<`TTable`\>\> | Shard locator for this Ent, responsible for resolving IDs into Shard objects. | +| `VALIDATION` | [`Validation`](../classes/Validation.md)\<`TTable`\> | Privacy rules for this Ent class. | +| `TRIGGERS` | [`Triggers`](../classes/Triggers.md)\<`TTable`\> | Triggers for this Ent class. | +| `INVERSES` | [`Inverse`](../classes/Inverse.md)\<`TClient`, `TTable`\>[] | Inverse assoc managers for fields. | + +## Methods + +### configure() + +> **configure**(): [`Configuration`](../classes/Configuration.md)\<`TTable`\> + +Defined in: [src/ent/mixins/ConfigMixin.ts:29](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/ConfigMixin.ts#L29) + +Some Ent parameters need to be configured lazily, on the 1st access, +because there could be cyclic references between Ent classes (e.g. in their +privacy rules). So configure() is called on some later stage, at the moment +of actual Ent operations (like loading, creation etc.). There is no static +abstract methods in TS yet, so making it non-abstract. + +#### Returns + +[`Configuration`](../classes/Configuration.md)\<`TTable`\> diff --git a/docs/interfaces/ConfigInstance.md b/docs/interfaces/ConfigInstance.md new file mode 100644 index 0000000..c5d7239 --- /dev/null +++ b/docs/interfaces/ConfigInstance.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ConfigInstance + +# Interface: ConfigInstance + +Defined in: [src/ent/mixins/ConfigMixin.ts:15](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/ConfigMixin.ts#L15) + +## Extended by + +- [`PrimitiveInstance`](PrimitiveInstance.md) diff --git a/docs/interfaces/Ent.md b/docs/interfaces/Ent.md new file mode 100644 index 0000000..6e9e9c7 --- /dev/null +++ b/docs/interfaces/Ent.md @@ -0,0 +1,54 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Ent + +# Interface: Ent\ + +Defined in: [src/ent/types.ts:69](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L69) + +A very shallow interface of one Ent. + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | `object` | + +## Properties + +| Property | Type | +| ------ | ------ | +| `id` | `string` | +| `vc` | [`VC`](../classes/VC.md) | + +## Methods + +### deleteOriginal() + +> **deleteOriginal**(): `Promise`\<`boolean`\> + +Defined in: [src/ent/types.ts:72](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L72) + +#### Returns + +`Promise`\<`boolean`\> + +*** + +### updateOriginal() + +> **updateOriginal**(`input`): `Promise`\<`boolean`\> + +Defined in: [src/ent/types.ts:73](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L73) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`UpdateOriginalInput`](../type-aliases/UpdateOriginalInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`boolean`\> diff --git a/docs/interfaces/EntClass.md b/docs/interfaces/EntClass.md new file mode 100644 index 0000000..9c50f07 --- /dev/null +++ b/docs/interfaces/EntClass.md @@ -0,0 +1,253 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntClass + +# Interface: EntClass\ + +Defined in: [src/ent/types.ts:28](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L28) + +A very shallow interface of Ent class (as a collection of static methods). +Used in some places where we need the very minimum from the Ent. + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | `DesperateAny` | + +## Constructors + +### new EntClass() + +> **new EntClass**(): [`Ent`](Ent.md)\<`TTable`\> + +Defined in: [src/ent/types.ts:34](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L34) + +#### Returns + +[`Ent`](Ent.md)\<`TTable`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `SCHEMA` | [`Schema`](../classes/Schema.md)\<`TTable`, [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\>\> | +| `VALIDATION` | [`Validation`](../classes/Validation.md)\<`TTable`\> | +| `SHARD_LOCATOR` | [`ShardLocator`](../classes/ShardLocator.md)\<[`Client`](../classes/Client.md), `TTable`, `string`\> | +| `name` | `string` | + +## Methods + +### loadX() + +> **loadX**(`vc`, `id`): `Promise`\<[`Ent`](Ent.md)\<`TTable`\>\> + +Defined in: [src/ent/types.ts:35](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L35) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `id` | `string` | + +#### Returns + +`Promise`\<[`Ent`](Ent.md)\<`TTable`\>\> + +*** + +### loadNullable() + +> **loadNullable**(`vc`, `id`): `Promise`\<`null` \| [`Ent`](Ent.md)\<`TTable`\>\> + +Defined in: [src/ent/types.ts:36](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L36) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `id` | `string` | + +#### Returns + +`Promise`\<`null` \| [`Ent`](Ent.md)\<`TTable`\>\> + +*** + +### loadIfReadableNullable() + +> **loadIfReadableNullable**(`vc`, `id`): `Promise`\<`null` \| [`Ent`](Ent.md)\<`TTable`\>\> + +Defined in: [src/ent/types.ts:37](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L37) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `id` | `string` | + +#### Returns + +`Promise`\<`null` \| [`Ent`](Ent.md)\<`TTable`\>\> + +*** + +### count() + +> **count**(`vc`, `where`): `Promise`\<`number`\> + +Defined in: [src/ent/types.ts:38](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L38) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `where` | [`CountInput`](../type-aliases/CountInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`number`\> + +*** + +### exists() + +> **exists**(`vc`, `where`): `Promise`\<`boolean`\> + +Defined in: [src/ent/types.ts:39](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L39) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `where` | [`ExistsInput`](../type-aliases/ExistsInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`boolean`\> + +*** + +### select() + +> **select**(`vc`, `where`, `limit`, `order`?): `Promise`\<[`Ent`](Ent.md)\<`TTable`\>[]\> + +Defined in: [src/ent/types.ts:40](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L40) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `where` | [`Where`](../type-aliases/Where.md)\<`TTable`\> | +| `limit` | `number` | +| `order`? | [`Order`](../type-aliases/Order.md)\<`TTable`\> | + +#### Returns + +`Promise`\<[`Ent`](Ent.md)\<`TTable`\>[]\> + +*** + +### selectChunked() + +> **selectChunked**(`vc`, `where`, `chunkSize`, `limit`, `custom`?): `AsyncIterableIterator`\<[`Ent`](Ent.md)\<`TTable`\>[], `any`, `any`\> + +Defined in: [src/ent/types.ts:46](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L46) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `where` | [`Where`](../type-aliases/Where.md)\<`TTable`\> | +| `chunkSize` | `number` | +| `limit` | `number` | +| `custom`? | \{\} | + +#### Returns + +`AsyncIterableIterator`\<[`Ent`](Ent.md)\<`TTable`\>[], `any`, `any`\> + +*** + +### loadByX() + +> **loadByX**(`vc`, `keys`): `Promise`\<[`Ent`](Ent.md)\<`TTable`\>\> + +Defined in: [src/ent/types.ts:53](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L53) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `keys` | `{ [K in string]: Value }` | + +#### Returns + +`Promise`\<[`Ent`](Ent.md)\<`TTable`\>\> + +*** + +### loadByNullable() + +> **loadByNullable**(`vc`, `input`): `Promise`\<`null` \| [`Ent`](Ent.md)\<`TTable`\>\> + +Defined in: [src/ent/types.ts:57](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L57) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `input` | `{ [K in string]: Value }` | + +#### Returns + +`Promise`\<`null` \| [`Ent`](Ent.md)\<`TTable`\>\> + +*** + +### insert() + +> **insert**(`vc`, `input`): `Promise`\<`string`\> + +Defined in: [src/ent/types.ts:61](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L61) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`string`\> + +*** + +### upsert() + +> **upsert**(`vc`, `input`): `Promise`\<`string`\> + +Defined in: [src/ent/types.ts:62](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L62) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `input` | [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`string`\> diff --git a/docs/interfaces/EntValidationErrorInfo.md b/docs/interfaces/EntValidationErrorInfo.md new file mode 100644 index 0000000..838fdf0 --- /dev/null +++ b/docs/interfaces/EntValidationErrorInfo.md @@ -0,0 +1,18 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / EntValidationErrorInfo + +# Interface: EntValidationErrorInfo + +Defined in: [src/ent/errors/EntValidationError.ts:46](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntValidationError.ts#L46) + +Auxiliary information which every validation predicate should emit. + +## Properties + +| Property | Type | +| ------ | ------ | +| `field` | `null` \| `string` | +| `message` | `string` | diff --git a/docs/interfaces/Handler.md b/docs/interfaces/Handler.md new file mode 100644 index 0000000..355da84 --- /dev/null +++ b/docs/interfaces/Handler.md @@ -0,0 +1,25 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Handler + +# Interface: Handler\ + +Defined in: [src/abstract/Loader.ts:4](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loader.ts#L4) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TLoadArgs` *extends* `unknown`[] | +| `TReturn` | + +## Properties + +| Property | Type | +| ------ | ------ | +| `onCollect` | (...`args`: `TLoadArgs`) => `void` \| `"flush"` \| `"wait"` | +| `onWait?` | () => `Promise`\<`void`\> | +| `onFlush` | (`collected`: `number`) => `Promise`\<`void`\> | +| `onReturn` | (...`args`: `TLoadArgs`) => `TReturn` | diff --git a/docs/interfaces/HelpersClass.md b/docs/interfaces/HelpersClass.md new file mode 100644 index 0000000..ec7c9d7 --- /dev/null +++ b/docs/interfaces/HelpersClass.md @@ -0,0 +1,90 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / HelpersClass + +# Interface: HelpersClass\ + +Defined in: [src/ent/mixins/HelpersMixin.ts:68](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/HelpersMixin.ts#L68) + +## Extends + +- `OmitNew`\<[`PrimitiveClass`](../type-aliases/PrimitiveClass.md)\<`TTable`, `TUniqueKey`, `TClient`\>\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](../type-aliases/UniqueKey.md)\<`TTable`\> | +| `TClient` *extends* [`Client`](../classes/Client.md) | + +## Constructors + +### new HelpersClass() + +> **new HelpersClass**(): [`HelpersInstance`](HelpersInstance.md)\<`TTable`\> & [`RowWithID`](../type-aliases/RowWithID.md) & `{ [K in string]: Value }` + +Defined in: [src/ent/mixins/HelpersMixin.ts:131](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/HelpersMixin.ts#L131) + +TS requires us to have a public constructor to infer instance types in +various places. We make this constructor throw if it's called. + +#### Returns + +[`HelpersInstance`](HelpersInstance.md)\<`TTable`\> & [`RowWithID`](../type-aliases/RowWithID.md) & `{ [K in string]: Value }` + +#### Inherited from + +`OmitNew>.constructor` + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `Configuration` | (`cfg`: [`Configuration`](../classes/Configuration.md)\<`TTable`\>) => [`Configuration`](../classes/Configuration.md)\<`TTable`\> | A helper class to work-around TS weakness in return value type inference: https://github.com/Microsoft/TypeScript/issues/31273. It could've been just a function, but having a class is a little more natural. | +| `CLUSTER` | [`Cluster`](../classes/Cluster.md)\<`TClient`, `any`\> | A Cluster where this Ent lives. | +| `SCHEMA` | [`Schema`](../classes/Schema.md)\<`TTable`, `TUniqueKey`\> | A schema which represents this Ent. | +| `SHARD_AFFINITY` | [`ShardAffinity`](../type-aliases/ShardAffinity.md)\<[`FieldOfIDType`](../type-aliases/FieldOfIDType.md)\<`TTable`\>\> | Defines how to find the right Shard during Ent insertion. | +| `SHARD_LOCATOR` | [`ShardLocator`](../classes/ShardLocator.md)\<`TClient`, `TTable`, [`FieldOfIDType`](../type-aliases/FieldOfIDType.md)\<`TTable`\>\> | Shard locator for this Ent, responsible for resolving IDs into Shard objects. | +| `VALIDATION` | [`Validation`](../classes/Validation.md)\<`TTable`\> | Privacy rules for this Ent class. | +| `TRIGGERS` | [`Triggers`](../classes/Triggers.md)\<`TTable`\> | Triggers for this Ent class. | +| `INVERSES` | [`Inverse`](../classes/Inverse.md)\<`TClient`, `TTable`\>[] | Inverse assoc managers for fields. | +| `insert` | (`vc`: [`VC`](../classes/VC.md), `input`: [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>) => `Promise`\<`string`\> | Same as insertIfNotExists(), but throws if the Ent violates unique key constraints. | +| `insertReturning` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `input`: [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>) => `Promise`\<`TEnt`\> | Same as insert(), but returns the created Ent. | +| `upsertReturning` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `input`: [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>) => `Promise`\<`TEnt`\> | Same, but returns the created/updated Ent. | +| `loadIfReadableNullable` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `id`: `string`) => `Promise`\<`null` \| `TEnt`\> | Same as loadNullable(), but if no permissions to read, returns null and doesn't throw. It's more a convenience function rather than a concept. | +| `loadX` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `id`: `string`) => `Promise`\<`TEnt`\> | Loads an Ent by its ID. Throws if no such Ent is found. This method is used VERY often. | +| `loadByX` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `input`: [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TUniqueKey`\>) => `Promise`\<`TEnt`\> | Loads an Ent by its ID. Throws if no such Ent is found. This method is used VERY often. | +| `insertIfNotExists` | (`vc`: [`VC`](../classes/VC.md), `input`: [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>) => `Promise`\<`null` \| `string`\> | Runs INSERT mutation for the Ent. - The Shard is inferred from the input fields using SHARD_AFFINITY. - Returns ID of the newly inserted row. - Returns null if the Ent violates unique key constraints. - If the Ent has some triggers set up, this will be translated into two schema operations: idGen() and insert(), and before-triggers will run in between having the ID known in advance. | +| `upsert` | (`vc`: [`VC`](../classes/VC.md), `input`: [`InsertInput`](../type-aliases/InsertInput.md)\<`TTable`\>) => `Promise`\<`string`\> | Inserts an Ent or updates an existing one if unique key matches. - Don't use upsert() too often, because upsert may still delete IDs even if the object was updated, not inserted (there is no good ways to solve this in some DB engines like relational DBs so far). - Upsert can't work if some triggers are defined for the Ent, because we don't know Ent ID in advance (whether the upsert succeeds or skips on duplication). | +| `loadNullable` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `id`: `string`) => `Promise`\<`null` \| `TEnt`\> | Loads an Ent by its ID. Returns null if no such Ent exists. Try to use loadX() instead as much as you can. | +| `loadByNullable` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `input`: [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TUniqueKey`\>) => `Promise`\<`null` \| `TEnt`\> | Loads an Ent by its unique key. Returns null if no such Ent exists. Notice that the key must be REALLY unique, otherwise the database may return multiple items, and the API will break. Don't try to use this method with non-unique keys! | +| `selectBy` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `input`: [`LoadByInput`](../type-aliases/LoadByInput.md)\<`TTable`, `TuplePrefixes`\<`TUniqueKey`\>\>) => `Promise`\<`TEnt`[]\> | Selects the list of Ents by their unique key prefix. The query can span multiple Shards if their locations can be inferred from inverses related to the fields mentioned in the query. Ordering of the results is not guaranteed. | +| `select` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `where`: [`Where`](../type-aliases/Where.md)\<`TTable`\>, `limit`: `number`, `order`?: [`Order`](../type-aliases/Order.md)\<`TTable`\>, `custom`?: `object`) => `Promise`\<`TEnt`[]\> | Selects the list of Ents by some predicate. - The query can span multiple Shards if their locations can be inferred from inverses related to the fields mentioned in the query. - In multi-Shard case, ordering of results is not guaranteed. - In multi-Shard case, it may return more results than requested by limit (basically, limit is applied to each Shard individually). The caller has then freedom to reorder & slice the results as they wish. | +| `selectChunked` | \<`TEnt`\>(`this`: () => `TEnt`, `vc`: [`VC`](../classes/VC.md), `where`: [`Where`](../type-aliases/Where.md)\<`TTable`\>, `chunkSize`: `number`, `limit`: `number`, `custom`?: `object`) => `AsyncIterableIterator`\<`TEnt`[], `any`, `any`\> | Same as select(), but returns data in chunks. - Uses multiple select() queries under the hood. - The query can span multiple Shards if their locations can be inferred from inverses related to the fields mentioned in the query. - Ents in each chunk always belong to the same Shard and are ordered by ID (there is no support for custom ordering). Make sure you have the right index in the database. | +| `count` | (`vc`: [`VC`](../classes/VC.md), `where`: [`CountInput`](../type-aliases/CountInput.md)\<`TTable`\>) => `Promise`\<`number`\> | Returns count of Ents matching a predicate. The query can span multiple Shards if their locations can be inferred from inverses related to the fields mentioned in the query. | +| `exists` | (`vc`: [`VC`](../classes/VC.md), `where`: [`ExistsInput`](../type-aliases/ExistsInput.md)\<`TTable`\>) => `Promise`\<`boolean`\> | A more optimal approach than count() when we basically just need to know whether we have "0 or not 0" rows. | + +## Methods + +### configure() + +> **configure**(): [`Configuration`](../classes/Configuration.md)\<`TTable`\> + +Defined in: [src/ent/mixins/ConfigMixin.ts:29](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/ConfigMixin.ts#L29) + +Some Ent parameters need to be configured lazily, on the 1st access, +because there could be cyclic references between Ent classes (e.g. in their +privacy rules). So configure() is called on some later stage, at the moment +of actual Ent operations (like loading, creation etc.). There is no static +abstract methods in TS yet, so making it non-abstract. + +#### Returns + +[`Configuration`](../classes/Configuration.md)\<`TTable`\> + +#### Inherited from + +`OmitNew.configure` diff --git a/docs/interfaces/HelpersInstance.md b/docs/interfaces/HelpersInstance.md new file mode 100644 index 0000000..6413213 --- /dev/null +++ b/docs/interfaces/HelpersInstance.md @@ -0,0 +1,195 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / HelpersInstance + +# Interface: HelpersInstance\ + +Defined in: [src/ent/mixins/HelpersMixin.ts:20](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/HelpersMixin.ts#L20) + +## Extends + +- [`PrimitiveInstance`](PrimitiveInstance.md)\<`TTable`\> + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | VC of this Ent. | +| `id` | `string` | For simplicity, every Ent has an ID field name hardcoded to "id". | + +## Methods + +### updateChanged() + +> **updateChanged**(`input`): `Promise`\<`null` \| `false` \| [`UpdateField`](../type-aliases/UpdateField.md)\<`TTable`\>[]\> + +Defined in: [src/ent/mixins/HelpersMixin.ts:36](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/HelpersMixin.ts#L36) + +Same as updateOriginal(), but updates only the fields which are different +in input and in the current object. +- This method can works with CAS; see $cas property of the passed object. + If CAS fails, returns false, the same way as updateOriginal() does. +- If there was no such Ent in the DB, returns false, the same way as + updateOriginal() does. +- If no changed fields are detected, returns null as an indication (it's + still falsy, but is different from the parent updateOriginal's `false`). +- Otherwise, when an update happened, returns the list of fields which were + different and triggered that change (a truthy value). The order of fields + in the list matches the order of fields in the Ent schema definition. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`UpdateOriginalInput`](../type-aliases/UpdateOriginalInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`null` \| `false` \| [`UpdateField`](../type-aliases/UpdateField.md)\<`TTable`\>[]\> + +*** + +### updateChangedReturningX() + +> **updateChangedReturningX**\<`TEnt`\>(`this`, `input`): `Promise`\<`TEnt`\> + +Defined in: [src/ent/mixins/HelpersMixin.ts:44](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/HelpersMixin.ts#L44) + +Same as updateChanged(), but returns the updated Ent (or the original one +if no fields were updated). + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TEnt` *extends* [`HelpersInstance`](HelpersInstance.md)\<`TTable`\> | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `this` | `TEnt` | +| `input` | [`UpdateInput`](../type-aliases/UpdateInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`TEnt`\> + +*** + +### updateReturningNullable() + +> **updateReturningNullable**\<`TEnt`\>(`this`, `input`): `Promise`\<`null` \| `TEnt`\> + +Defined in: [src/ent/mixins/HelpersMixin.ts:53](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/HelpersMixin.ts#L53) + +Same as updateOriginal(), but returns the updated Ent (or null of there +was no such Ent in the database). + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TEnt` *extends* [`HelpersInstance`](HelpersInstance.md)\<`TTable`\> | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `this` | `TEnt` | +| `input` | [`UpdateInput`](../type-aliases/UpdateInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`null` \| `TEnt`\> + +*** + +### updateReturningX() + +> **updateReturningX**\<`TEnt`\>(`this`, `input`): `Promise`\<`TEnt`\> + +Defined in: [src/ent/mixins/HelpersMixin.ts:62](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/HelpersMixin.ts#L62) + +Same as updateOriginal(), but throws if the object wasn't updated or +doesn't exist after the update. + +#### Type Parameters + +| Type Parameter | +| ------ | +| `TEnt` *extends* [`HelpersInstance`](HelpersInstance.md)\<`TTable`\> | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `this` | `TEnt` | +| `input` | [`UpdateInput`](../type-aliases/UpdateInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`TEnt`\> + +*** + +### updateOriginal() + +> **updateOriginal**(`input`): `Promise`\<`boolean`\> + +Defined in: [src/ent/mixins/PrimitiveMixin.ts:59](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/PrimitiveMixin.ts#L59) + +Updates the object in the DB, but doesn't update the Ent itself (since it's +immutable). +- This method can work with CAS; see $cas property of the passed object. +- If a special value "skip-if-someone-else-changed-updating-ent-props" is + passed to $cas, then the list of props for CAS is brought from the input, + and the values of these props are brought from the Ent itself (i.e. from + `this`). +- If a special value, a list of field names, is passed to $cas, then it + works like described above, but the list of prop names is brought from + that list of field names. +- Returns false if there is no such object in the DB, or if CAS check + didn't succeed. +- Returns true if the object was found and updated. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`UpdateOriginalInput`](../type-aliases/UpdateOriginalInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`boolean`\> + +#### Inherited from + +[`PrimitiveInstance`](PrimitiveInstance.md).[`updateOriginal`](PrimitiveInstance.md#updateoriginal) + +*** + +### deleteOriginal() + +> **deleteOriginal**(): `Promise`\<`boolean`\> + +Defined in: [src/ent/mixins/PrimitiveMixin.ts:65](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/PrimitiveMixin.ts#L65) + +Deletes the object in the DB. Returns true if the object was found. Keeps +the current object untouched (since it's immutable). + +#### Returns + +`Promise`\<`boolean`\> + +#### Inherited from + +[`PrimitiveInstance`](PrimitiveInstance.md).[`deleteOriginal`](PrimitiveInstance.md#deleteoriginal) diff --git a/docs/interfaces/IslandOptions.md b/docs/interfaces/IslandOptions.md new file mode 100644 index 0000000..85ea5f8 --- /dev/null +++ b/docs/interfaces/IslandOptions.md @@ -0,0 +1,27 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / IslandOptions + +# Interface: IslandOptions\ + +Defined in: [src/abstract/Island.ts:26](https://github.com/clickup/ent-framework/blob/master/src/abstract/Island.ts#L26) + +Options for Island constructor. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TClient` *extends* [`Client`](../classes/Client.md) | + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `no` | `number` | Island number. | +| `clients` | readonly `TClient`[] | Clients of that Island (the order is arbitrary). | +| `createShard` | (`no`: `number`) => [`Shard`](../classes/Shard.md)\<`TClient`\> | Should return a Memoize'd Shards object by its number. | +| `localCache?` | `null` \| [`LocalCache`](../classes/LocalCache.md)\<\{ `address`: `string`; `role`: [`ClientRole`](../type-aliases/ClientRole.md); \}\> | An auxillary LocalCache used to fallback-infer master/replica role in case some Client is unavailable right now. | +| `shardNosConcurrentRetryDelayMs?` | `number` | If nonzero, runs the second shardNos() call attempt on a Client if the 1st call on that Client gets stuck for longer than the provided number of ms. This option is used to detect the unhealthy DB connection quicker, and thus, exit from rediscover() faster (the Shards map can likely be loaded from a replica still, so the down DB is not the end of the world). The idea is that the 1st shardNos() could get stuck due to the load balancer trying to wait until the DB goes back up again (e.g. for PgBouncer, that is query_wait_timeout situation; "pause_client" is printed to PgBouncer debug logs, and then the Client gets frozen for up to query_wait_timeout; other engines may have similar behavior). But for the NEW connections/queries, after a small delay, the load balancer may realize that the DB is really down (the load balancer typically can get "connection refused" while connecting to the DB server really quickly), and the 2nd shardNos() call will reject almost immediately ("fast fail" workflow), way before the 1st call rejects (e.g. for PgBouncer and query_wait_timeout=15s, the 1st call may get stuck for up to 15 seconds!). So, we will not wait that long to figure out that the DB is down, and will detect that situation quicker. Typically, the connection attempt from load balancer to an unhealthy DB server ends up quickly with "connection refused" TCP error (e.g. when the load balancer and the DB server run on the same host), so the value in this option can be small. But not always. Sometimes, the new connection from load balancer to the DB server gets stuck in "connecting..." state (e.g. this happens when the load balancer runs in a Docker container, and the DB container gets killed; the connection attempt will eventually fail, but in 1+ minutes and with "no route to host" error). In this case, the value in the option must be greater than e.g. server_connect_timeout (example for PgBouncer; basically, server_connect_timeout is PgBouncer's tool to detect "stuck" connection attempts (the connections which don't get "connection refused" quickly). | diff --git a/docs/interfaces/LocalCacheOptions.md b/docs/interfaces/LocalCacheOptions.md new file mode 100644 index 0000000..c363d26 --- /dev/null +++ b/docs/interfaces/LocalCacheOptions.md @@ -0,0 +1,22 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / LocalCacheOptions + +# Interface: LocalCacheOptions + +Defined in: [src/abstract/LocalCache.ts:8](https://github.com/clickup/ent-framework/blob/master/src/abstract/LocalCache.ts#L8) + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `dir` | `string` | Directory to store the cache in (auto-created). | +| `loggers` | `Pick`\<[`Loggers`](Loggers.md)\<`any`\>, `"swallowedErrorLogger"`\> | Loggers for e.g. swallowed errors. | +| `expirationMs?` | `number` | Max time (approximately) for an unread key to exist. | +| `ext?` | `string` | Extension of cache files (without dot). | +| `cleanupJitter?` | `number` | Jitter for cleanup runs. | +| `cleanupFirstRunDelayMs?` | `number` | How much time to wait till the very 1st cleanup run. The idea is that Node process may be short-lived, so the next cleanup run configured via cleanupRoundsPerExpiration may never happen, and we also need to cleanup in the very beginning of the object lifetime. | +| `cleanupRoundsPerExpiration?` | `number` | How many times per expirationMs interval should we run the cleanup. | +| `mtimeUpdatesOnReadPerExpiration?` | `number` | How often to update mtime on read operations. E.g. if this value is 10, then mtime will be updated not more than ~10 times within the expiration period (optimizing filesystem writes). | diff --git a/docs/interfaces/Loggers.md b/docs/interfaces/Loggers.md new file mode 100644 index 0000000..fa04c8b --- /dev/null +++ b/docs/interfaces/Loggers.md @@ -0,0 +1,30 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Loggers + +# Interface: Loggers\ + +Defined in: [src/abstract/Loggers.ts:11](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loggers.ts#L11) + +Loggers are called at different stages of the query lifecycle. We do not use +EventEmitter for several reasons: +1. It is not friendly to mocking in Jest. +2. The built-in EventEmitter is not strongly typed. + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TNode` | `DesperateAny` | + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `clientQueryLogger?` | (`props`: [`ClientQueryLoggerProps`](ClientQueryLoggerProps.md)) => `void` | Logs actual queries to the database (after batching). | +| `swallowedErrorLogger` | (`props`: [`SwallowedErrorLoggerProps`](SwallowedErrorLoggerProps.md)) => `void` | Logs errors which did not throw through (typically recoverable). | +| `runOnShardErrorLogger?` | (`props`: [`RunOnShardErrorLoggerProps`](RunOnShardErrorLoggerProps.md)) => `void` | Called when Island-from-Shard location fails (e.g. no such Shard), or when a query on a particular Shard fails due to any reason (like transport error). Mostly used in unit tests, since it's called for every retry. | +| `clientEndLogger?` | (`props`: [`ClientEndLoggerProps`](ClientEndLoggerProps.md)\<`TNode`\>) => `void` | Called when a Client gets ended due to dynamic Islands reconfiguration. Allows to debug flaky Island reconfiguration. | +| `clientConnectedLogger?` | (`props`: [`ClientConnectedLoggerProps`](ClientConnectedLoggerProps.md)\<`TNode`\>) => `void` | Called when a Client connects to the database. | diff --git a/docs/interfaces/PgClientConn.md b/docs/interfaces/PgClientConn.md new file mode 100644 index 0000000..4f24c23 --- /dev/null +++ b/docs/interfaces/PgClientConn.md @@ -0,0 +1,30 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgClientConn + +# Interface: PgClientConn\ + +Defined in: [src/pg/PgClient.ts:96](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L96) + +An opened low-level PostgreSQL connection. + +## Extends + +- `PoolClient` + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TPool` *extends* `pg.Pool` | `pg.Pool` | + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `pool` | `TPool` | Pool instance that created this connection. | +| `id` | `number` | An additional property to the vanilla client: auto-incrementing ID of the connection for logging purposes. | +| `queriesSent` | `number` | An additional property to the vanilla client: number of queries sent within this connection. | +| `closeAt` | `null` \| `number` | An additional property to the vanilla client: when do we want to hard-close that connection. | diff --git a/docs/interfaces/PgClientOptions.md b/docs/interfaces/PgClientOptions.md new file mode 100644 index 0000000..e75b0a2 --- /dev/null +++ b/docs/interfaces/PgClientOptions.md @@ -0,0 +1,43 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgClientOptions + +# Interface: PgClientOptions\ + +Defined in: [src/pg/PgClient.ts:40](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L40) + +Options for PgClient constructor. + +## Extends + +- [`ClientOptions`](ClientOptions.md) + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TPool` *extends* `pg.Pool` | `pg.Pool` | + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `name` | `string` | Name of the Client; used for logging. | +| `shardNamer?` | `null` \| [`ShardNamer`](../classes/ShardNamer.md) | Info on how to build/parse Shard names. If not set, then Cluster injects its own ShardNamer here right after creating a Client instance. | +| `loggers?` | `null` \| [`Loggers`](Loggers.md)\<`any`\> | Loggers to be called at different stages. Client code calls into them. Also, Cluster injects its own loggers here, in addition to the provided ones (if any). | +| `batchDelayMs?` | `MaybeCallable`\<`number`\> | If passed, there will be an artificial queries accumulation delay while batching the requests. Default is 0 (turned off). Passed to Batcher#batchDelayMs. | +| `config` | `PoolConfig` & `object` | Node-Postgres config. We can't make it MaybeCallable unfortunately, because it's used to initialize Node-Postgres Pool. | +| `createPool?` | (`config`: `PoolConfig`) => `TPool` | Should create an instance of Pool class compatible with node-postgres Pool. By default, node-postgres Pool is used. | +| `maxConnLifetimeMs?` | `MaybeCallable`\<`number`\> | Close the connection after the query if it was opened long time ago. | +| `maxConnLifetimeJitter?` | `MaybeCallable`\<`number`\> | Jitter for maxConnLifetimeMs. | +| `prewarmIntervalStep?` | `MaybeCallable`\<`number`\> | Add not more than this number of connections in each prewarm interval. New connections are expensive to establish (especially when SSL is enabled). | +| `prewarmIntervalMs?` | `MaybeCallable`\<`number`\> | How often to send bursts of prewarm queries to all Clients to keep the minimal number of open connections. The default value is half of the default node-postgres'es idleTimeoutMillis=10s. Together with 1..1.5x jitter (default prewarmIntervalJitter=0.5), it is still slightly below idleTimeoutMillis, and thus, doesn't let Ent Framework close the connections prematurely. | +| `prewarmIntervalJitter?` | `MaybeCallable`\<`number`\> | Jitter for prewarmIntervalMs. | +| `prewarmQuery?` | `MaybeCallable`\<`string`\> | What prewarm query to send. | +| `prewarmSubPools?` | `boolean` | If true, also sends prewarm queries and keeps the min number of connections in all sub-pools. See pool() method for details. | +| `hints?` | `null` \| `MaybeCallable`\<[`Hints`](../type-aliases/Hints.md)\> | PG "SET key=value" hints to run before each query. Often times we use it to pass statement_timeout option since e.g. PGBouncer doesn't support per-connection statement timeout in transaction pooling mode: it throws "unsupported startup parameter" error. I.e. we may want to emit "SET statement_timeout TO ..." before each query in multi-query mode. | +| `maxReplicationLagMs?` | `MaybeCallable`\<`number`\> | After how many milliseconds we give up waiting for the replica to catch up with the master. When role="replica", then this option is the only way to "unlatch" the reads from the master node after a write. | +| `role?` | [`ClientRole`](../type-aliases/ClientRole.md) | Sometimes, the role of this Client is known statically, e.g. when pointing to AWS Aurora writer and reader endpoints. If "master" or "replica" are provided, then no attempt is made to use functions like pg_current_wal_insert_lsn() etc. (they are barely supported in e.g. AWS Aurora). Instead, for "replica" role, it is treated as "always lagging up until maxReplicationLagMs after the last write". If role="unknown", then auto-detection and automatic lag tracking is performed using pg_current_wal_insert_lsn() and other built-in PostgreSQL functions. | +| `replicaTimelinePosRefreshMs?` | `MaybeCallable`\<`number`\> | Up to how often we call TimelineManager#triggerRefresh(). | diff --git a/docs/interfaces/PgClientSubPoolConfig.md b/docs/interfaces/PgClientSubPoolConfig.md new file mode 100644 index 0000000..9b2a1be --- /dev/null +++ b/docs/interfaces/PgClientSubPoolConfig.md @@ -0,0 +1,23 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgClientSubPoolConfig + +# Interface: PgClientSubPoolConfig + +Defined in: [src/pg/PgClient.ts:116](https://github.com/clickup/ent-framework/blob/master/src/pg/PgClient.ts#L116) + +A named low-level Pool config used to create sub-pools. Sub-pool derives +configuration from the default PgClientOptions#config, but allow overrides. +See PgClient#pool() method for details. + +## Extends + +- `Partial`\<`pg.PoolConfig`\> + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | diff --git a/docs/interfaces/PgTimelineStorageOptions.md b/docs/interfaces/PgTimelineStorageOptions.md new file mode 100644 index 0000000..608e648 --- /dev/null +++ b/docs/interfaces/PgTimelineStorageOptions.md @@ -0,0 +1,22 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PgTimelineStorageOptions + +# Interface: PgTimelineStorageOptions + +Defined in: [src/pg/PgTimelineStorage.ts:14](https://github.com/clickup/ent-framework/blob/master/src/pg/PgTimelineStorage.ts#L14) + +## Extends + +- [`TimelineStorageOptions`](TimelineStorageOptions.md) + +## Properties + +| Property | Type | +| ------ | ------ | +| `merge?` | (`dataStrs`: `string`[]) => `string` | +| `maxChunksPerPrincipal?` | `MaybeCallable`\<`number`\> | +| `cluster` | [`Cluster`](../classes/Cluster.md)\<[`PgClient`](../classes/PgClient.md)\<`Pool`\>, `any`\> | +| `table?` | `string` | diff --git a/docs/interfaces/Predicate.md b/docs/interfaces/Predicate.md new file mode 100644 index 0000000..d056843 --- /dev/null +++ b/docs/interfaces/Predicate.md @@ -0,0 +1,56 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Predicate + +# Interface: Predicate\ + +Defined in: [src/ent/predicates/Predicate.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L17) + +A predicate evaluates against some input (typically a row) and returns true +or false (or throws which is considering the similar way as returning false). + +I.e. Predicate is a "yes/no" logic. If it resolves to "no", then the +framework may disallow some Ent operation and include predicate object's +properties (like name or any other info) to the exception. + +Also, some predicates try to use caches in vc to make the decision faster +based on the previously computed results. E.g. CanReadOutgoingEdge predicate +knows that it already returned true for some ID once, it returns true again +immediately. This saves us lots of database operations. + +## Extended by + +- [`AbstractIs`](AbstractIs.md) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` | + +## Properties + +| Property | Type | +| ------ | ------ | +| `name` | `string` | + +## Methods + +### check() + +> **check**(`vc`, `input`): `Promise`\<`boolean`\> + +Defined in: [src/ent/predicates/Predicate.ts:19](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/Predicate.ts#L19) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `input` | `TInput` | + +#### Returns + +`Promise`\<`boolean`\> diff --git a/docs/interfaces/PrimitiveInstance.md b/docs/interfaces/PrimitiveInstance.md new file mode 100644 index 0000000..3422559 --- /dev/null +++ b/docs/interfaces/PrimitiveInstance.md @@ -0,0 +1,77 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PrimitiveInstance + +# Interface: PrimitiveInstance\ + +Defined in: [src/ent/mixins/PrimitiveMixin.ts:31](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/PrimitiveMixin.ts#L31) + +## Extends + +- [`ConfigInstance`](ConfigInstance.md) + +## Extended by + +- [`HelpersInstance`](HelpersInstance.md) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](../type-aliases/Table.md) | + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | VC of this Ent. | +| `id` | `string` | For simplicity, every Ent has an ID field name hardcoded to "id". | + +## Methods + +### updateOriginal() + +> **updateOriginal**(`input`): `Promise`\<`boolean`\> + +Defined in: [src/ent/mixins/PrimitiveMixin.ts:59](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/PrimitiveMixin.ts#L59) + +Updates the object in the DB, but doesn't update the Ent itself (since it's +immutable). +- This method can work with CAS; see $cas property of the passed object. +- If a special value "skip-if-someone-else-changed-updating-ent-props" is + passed to $cas, then the list of props for CAS is brought from the input, + and the values of these props are brought from the Ent itself (i.e. from + `this`). +- If a special value, a list of field names, is passed to $cas, then it + works like described above, but the list of prop names is brought from + that list of field names. +- Returns false if there is no such object in the DB, or if CAS check + didn't succeed. +- Returns true if the object was found and updated. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | [`UpdateOriginalInput`](../type-aliases/UpdateOriginalInput.md)\<`TTable`\> | + +#### Returns + +`Promise`\<`boolean`\> + +*** + +### deleteOriginal() + +> **deleteOriginal**(): `Promise`\<`boolean`\> + +Defined in: [src/ent/mixins/PrimitiveMixin.ts:65](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/PrimitiveMixin.ts#L65) + +Deletes the object in the DB. Returns true if the object was found. Keeps +the current object untouched (since it's immutable). + +#### Returns + +`Promise`\<`boolean`\> diff --git a/docs/interfaces/Query.md b/docs/interfaces/Query.md new file mode 100644 index 0000000..5577845 --- /dev/null +++ b/docs/interfaces/Query.md @@ -0,0 +1,43 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Query + +# Interface: Query\ + +Defined in: [src/abstract/Query.ts:8](https://github.com/clickup/ent-framework/blob/master/src/abstract/Query.ts#L8) + +A very lean interface for a Query. In practice each query is so different +that this interface is the only common part of them all. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TOutput` | + +## Properties + +| Property | Type | +| ------ | ------ | +| `IS_WRITE` | `boolean` | + +## Methods + +### run() + +> **run**(`client`, `annotation`): `Promise`\<`TOutput`\> + +Defined in: [src/abstract/Query.ts:10](https://github.com/clickup/ent-framework/blob/master/src/abstract/Query.ts#L10) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`Client`](../classes/Client.md) | +| `annotation` | [`QueryAnnotation`](QueryAnnotation.md) | + +#### Returns + +`Promise`\<`TOutput`\> diff --git a/docs/interfaces/QueryAnnotation.md b/docs/interfaces/QueryAnnotation.md new file mode 100644 index 0000000..b479d27 --- /dev/null +++ b/docs/interfaces/QueryAnnotation.md @@ -0,0 +1,24 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / QueryAnnotation + +# Interface: QueryAnnotation + +Defined in: [src/abstract/QueryAnnotation.ts:27](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryAnnotation.ts#L27) + +A debug annotation from each individual place which initiated the query. When +multiple queries are grouped into one large query by Ent Framework (even +cross-async-trace and cross-VC), the resulting large query is accompanied +with all those annotations. + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `trace` | `string` | Trace ID of the series of the queries. | +| `vc` | `string` | Something which identifies the acting user; it's named `vc` after Ent's VC for simplicity, but at this layer of abstractions, there are no Ents. | +| `debugStack` | `string` | Sometimes a query may be annotated by the source stack trace. It's typically expensive, so it's likely "" in production. | +| `whyClient` | `undefined` \| [`WhyClient`](../type-aliases/WhyClient.md) | Answers, why exactly this Client was selected to send the query to. | +| `attempt` | `number` | In case it's a retry, the attempt number will be greater than 0. | diff --git a/docs/interfaces/RuleResult.md b/docs/interfaces/RuleResult.md new file mode 100644 index 0000000..8cdf3ad --- /dev/null +++ b/docs/interfaces/RuleResult.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / RuleResult + +# Interface: RuleResult + +Defined in: [src/ent/rules/Rule.ts:15](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Rule.ts#L15) + +A full debug info about some Rule decision (which Rule produced this +decision, what was thrown etc.). + +## Properties + +| Property | Type | +| ------ | ------ | +| `decision` | [`RuleDecision`](../type-aliases/RuleDecision.md) | +| `rule` | [`Rule`](../classes/Rule.md)\<`object`\> | +| `cause` | `null` \| [`EntAccessError`](../classes/EntAccessError.md) | diff --git a/docs/interfaces/RunOnShardErrorLoggerProps.md b/docs/interfaces/RunOnShardErrorLoggerProps.md new file mode 100644 index 0000000..fc0c4ac --- /dev/null +++ b/docs/interfaces/RunOnShardErrorLoggerProps.md @@ -0,0 +1,16 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / RunOnShardErrorLoggerProps + +# Interface: RunOnShardErrorLoggerProps + +Defined in: [src/abstract/Loggers.ts:67](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loggers.ts#L67) + +## Properties + +| Property | Type | +| ------ | ------ | +| `error` | `unknown` | +| `attempt` | `number` | diff --git a/docs/interfaces/SchemaClass.md b/docs/interfaces/SchemaClass.md new file mode 100644 index 0000000..18b402d --- /dev/null +++ b/docs/interfaces/SchemaClass.md @@ -0,0 +1,29 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / SchemaClass + +# Interface: SchemaClass + +Defined in: [src/abstract/Schema.ts:15](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L15) + +## Constructors + +### new SchemaClass() + +> **new SchemaClass**\<`TTable`, `TUniqueKey`\>(`name`, `table`, `uniqueKey`?): [`Schema`](../classes/Schema.md)\<`TTable`, `TUniqueKey`\> + +Defined in: [src/abstract/Schema.ts:16](https://github.com/clickup/ent-framework/blob/master/src/abstract/Schema.ts#L16) + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `name` | `string` | +| `table` | `TTable` | +| `uniqueKey`? | `TUniqueKey` | + +#### Returns + +[`Schema`](../classes/Schema.md)\<`TTable`, `TUniqueKey`\> diff --git a/docs/interfaces/ShardNamerOptions.md b/docs/interfaces/ShardNamerOptions.md new file mode 100644 index 0000000..bec7a3f --- /dev/null +++ b/docs/interfaces/ShardNamerOptions.md @@ -0,0 +1,18 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ShardNamerOptions + +# Interface: ShardNamerOptions + +Defined in: [src/abstract/ShardNamer.ts:6](https://github.com/clickup/ent-framework/blob/master/src/abstract/ShardNamer.ts#L6) + +Options for ShardNamer constructor. + +## Properties + +| Property | Type | Description | +| ------ | ------ | ------ | +| `nameFormat` | `string` | A format string to turn a Shard number to Shard name (e.g. "sh%04d"). | +| `discoverQuery` | `MaybeCallable`\<`string`\> | A DB engine query that should return the names of Shards served by this Client. | diff --git a/docs/interfaces/StandardSchemaV1FailureResult.md b/docs/interfaces/StandardSchemaV1FailureResult.md new file mode 100644 index 0000000..d7739f4 --- /dev/null +++ b/docs/interfaces/StandardSchemaV1FailureResult.md @@ -0,0 +1,18 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / StandardSchemaV1FailureResult + +# Interface: StandardSchemaV1FailureResult + +Defined in: [src/ent/errors/EntAccessError.ts:7](https://github.com/clickup/ent-framework/blob/master/src/ent/errors/EntAccessError.ts#L7) + +Standard Schema V1 compatible error result. Every EntAccessError can be +converted to it. See https://standardschema.dev. + +## Properties + +| Property | Type | +| ------ | ------ | +| `issues` | readonly `object`[] | diff --git a/docs/interfaces/SwallowedErrorLoggerProps.md b/docs/interfaces/SwallowedErrorLoggerProps.md new file mode 100644 index 0000000..841dbe1 --- /dev/null +++ b/docs/interfaces/SwallowedErrorLoggerProps.md @@ -0,0 +1,18 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / SwallowedErrorLoggerProps + +# Interface: SwallowedErrorLoggerProps + +Defined in: [src/abstract/Loggers.ts:60](https://github.com/clickup/ent-framework/blob/master/src/abstract/Loggers.ts#L60) + +## Properties + +| Property | Type | +| ------ | ------ | +| `where` | `string` | +| `error` | `unknown` | +| `elapsed` | `null` \| `number` | +| `importance` | `"low"` \| `"normal"` | diff --git a/docs/interfaces/TimelineStorageOptions.md b/docs/interfaces/TimelineStorageOptions.md new file mode 100644 index 0000000..72b678a --- /dev/null +++ b/docs/interfaces/TimelineStorageOptions.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / TimelineStorageOptions + +# Interface: TimelineStorageOptions + +Defined in: [src/ent/TimelineStorage.ts:5](https://github.com/clickup/ent-framework/blob/master/src/ent/TimelineStorage.ts#L5) + +## Extended by + +- [`PgTimelineStorageOptions`](PgTimelineStorageOptions.md) + +## Properties + +| Property | Type | +| ------ | ------ | +| `merge?` | (`dataStrs`: `string`[]) => `string` | +| `maxChunksPerPrincipal?` | `MaybeCallable`\<`number`\> | diff --git a/docs/interfaces/ToolPingOptions.md b/docs/interfaces/ToolPingOptions.md new file mode 100644 index 0000000..24838b1 --- /dev/null +++ b/docs/interfaces/ToolPingOptions.md @@ -0,0 +1,21 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ToolPingOptions + +# Interface: ToolPingOptions + +Defined in: [src/tools/ToolPing.ts:21](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolPing.ts#L21) + +Ping tool constructor options. + +## Properties + +| Property | Type | +| ------ | ------ | +| `cluster` | [`Cluster`](../classes/Cluster.md)\<[`Client`](../classes/Client.md), `any`\> | +| `shard?` | `number` | +| `pingExecTimeMs?` | `number` | +| `pingPollMs?` | `number` | +| `pingIsWrite?` | `boolean` | diff --git a/docs/interfaces/ToolScoreboardOptions.md b/docs/interfaces/ToolScoreboardOptions.md new file mode 100644 index 0000000..60fcbf7 --- /dev/null +++ b/docs/interfaces/ToolScoreboardOptions.md @@ -0,0 +1,24 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ToolScoreboardOptions + +# Interface: ToolScoreboardOptions + +Defined in: [src/tools/ToolScoreboard.ts:35](https://github.com/clickup/ent-framework/blob/master/src/tools/ToolScoreboard.ts#L35) + +Scoreboard tool constructor options. + +## Properties + +| Property | Type | +| ------ | ------ | +| `cluster` | [`Cluster`](../classes/Cluster.md)\<[`Client`](../classes/Client.md), `any`\> | +| `refreshMs?` | `number` | +| `pingExecTimeMs?` | `number` | +| `pingParallelism?` | `number` | +| `pingPollMs?` | `number` | +| `tickMs?` | `number` | +| `maxQueries?` | `number` | +| `maxErrors?` | `number` | diff --git a/docs/llms-full.md b/docs/llms-full.md new file mode 100644 index 0000000..d40f313 --- /dev/null +++ b/docs/llms-full.md @@ -0,0 +1,6572 @@ +# Ent Framework + +
+ +The TypeScript library for working with microsharded PostgreSQL databases. + +* [Getting Started and Tutorials](https://ent-framework.net) +* [API documentation](https://github.com/clickup/ent-framework/blob/main/docs/globals.md) +* [Source code](https://github.com/clickup/ent-framework) +* [Ent Framework's Discord](https://discord.gg/QXvN6VTCKS) + +#### Core Features + +1. **Graph-like representation of entities.** With Ent Framework, you represent each Ent (a domain object of your business logic) as a TypeScript class with immutable properties. An Ent class instance maps to one row of some table in a relational database (like PostgreSQL). It may look similar to ORM, but has many aspects that traditional ORMs don't have. +2. **Row-level security in a graph (privacy layer).** You manage data as a graph where each node is an Ent instance, and each edge is a field link (think of foreign keys) to other Ents. To be allowed to read (or update/delete) some Ent, you define a set of explicit rules like "user can read EntA if they can read EntB or EntC". And, consequently, in EntB you define its own set of rules, like "user can read EntB if they can read EntD". +3. **Query batching and coalescing.** Ent Framework holistically solves the "N+1 selects" problem commonly known in ORM world. You still write you code as if you work with individual Ents and individual IDs, and the framework magically takes care of sending batched requests (both read and write) to the underlying relational database. You do not work with lists and JOINs anymore. +4. **Microsharding and replication lag tracking support out of the box.** Splitting your database horizontally is like a breeze now: Ent Framework takes care of routing the requests to the proper microshards. When scaling reads, Ent Framework knows whether a replica node is "good enough" for that particular query. It automatically uses the proper replica when possible, falling back to master when not. +5. **Pluggable to your existing relational database.** If your project already uses some ORM or runs raw SQL queries, Ent Framework can be plugged in. +6. **Tens of other features.** Some examples: cross-microshards foreign keys, composite fields, triggers, build-in caching etc. + +#### Installation + +``` +npm add ent-framework +pnpm add ent-framework +yarn add ent-framework +``` + +
+ + +# Code Structure + +Below, we'll show some Ent Framework usage examples. We will progress from the simplest code snippets to more and more advanced topics, like: + +* custom ID schemas +* privacy rules +* triggers +* composite field types +* Viewer Context flavors +* master-replica and automatic replication lag tracking +* microsharding and migrations +* cross-shards foreign keys and inverse indexes +* etc. + +### Code Structure + +The examples in this tutorial will approximately follow [examples/next-example](https://github.com/dimikot/ent-framework/tree/main/examples/next-example) `src` folder structure: + +* ents/ + * cluster.sql + * cluster.ts + * EntComment.ts + * EntTopic.ts + * EntUser.ts + * getServerVC.ts +* app/ + * api/ + * auth/\[...nextauth] + * route.ts + * topics/ + * route.ts + + +# Connect to a Database + +To start simple, create a PostgreSQL database and several tables there. You can also use you existing database: + +```bash +$ psql postgresql://postgres:postgres@127.0.0.1/postgres -f ents/cluster.sql +``` + +{% code title="ents/cluster.sql" %} + +```sql +CREATE TABLE users( + id bigserial PRIMARY KEY, + email varchar(256) NOT NULL UNIQUE, + is_admin boolean NOT NULL DEFAULT FALSE +); + +CREATE TABLE topics( + id bigserial PRIMARY KEY, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + slug varchar(64) NOT NULL UNIQUE, + creator_id bigint NOT NULL, + subject text DEFAULT NULL +); + +CREATE TABLE comments( + id bigserial PRIMARY KEY, + created_at timestamptz NOT NULL, + topic_id bigint REFERENCES topics, + creator_id bigint NOT NULL, + message text NOT NULL +); + +CREATE TABLE organizations( + id bigserial PRIMARY KEY, + name text NOT NULL UNIQUE +); + +CREATE TABLE organization_users( + id bigserial PRIMARY KEY, + organization_id bigint REFERENCES organizations, + user_id bigint REFERENCES users, + UNIQUE (organization_id, user_id) +); +``` + +{% endcode %} + +To access that database, create an instance of Cluster: + +{% code title="ents/cluster.ts" fullWidth="false" %} + +```typescript +import { Cluster } from "ent-framework"; +import type { PgClientOptions } from "ent-framework/pg"; +import { PgClient } from "ent-framework/pg"; +import type { PoolConfig } from "pg"; + +export const cluster = new Cluster({ + islands: async () => [ // sync or async + { + no: 0, + nodes: [ + { + name: "island0-master", + config: { + connectionString: process.env.DATABASE_URL, // e.g. from .env + // This object is of the standard node-postgres type PoolConfig. + // Thus, you can use host, port, user, password, database and other + // properties instead of connectionString if you want. + min: 5, + max: 20, + } satisfies PoolConfig, + }, + ], + }, + ], + createClient: (node) => new PgClient(node), + loggers: { + clientQueryLogger: (props) => console.debug(props.msg), + swallowedErrorLogger: (props) => console.log(props), + }, +}); + +// Pre-open min number of DB connections. +cluster.prewarm(); +``` + +{% endcode %} + +Terminology: + +1. **Cluster** consists of **Islands**. Each Island is identified by an integer number (there can be many islands for horizontal scaling of the cluster). +2. Island consists of master + replica **nodes** (in the above example, we only define one master node and no replicas). +3. Island also hosts **Microshards** (in the example above, we will have no microshards, aka just one global shard). Microshards may travel from island to island during shards rebalancing process; the engine tracks this automatically ("shards discovery"). + +Notice that we define the layout of the cluster using a callback. Ent Framework will call it from time to time to refresh the view of the cluster, so in this callback, you can read the data from some centralized configuration database (new nodes may be added, or empty nodes may be removed with no downtime). This is called "dynamic real-time reconfiguration". + +[PgClient](https://github.com/clickup/ent-framework/blob/main/docs/classes/PgClient.md) class accepts several options, one of them is the standard [node-postgres PoolConfig](https://node-postgres.com/apis/pool) interface. For simplicity, when we define a cluster shape in `islands`, we just return a list of such configs, to be passed into `createClient()` lambda. + +As of `prewarm()` call, it's explained in Advanced section. + + +# Create Ent Classes + +Once you have a Cluster instance, you can create Ent classes to access the data. + +{% code title="ents/EntUser.ts" %} + +```typescript +import { PgSchema } from "ent-framework/pg"; +import { ID, BaseEnt, GLOBAL_SHARD, AllowIf, OutgoingEdgePointsToVC } from "ent-framework"; +import { cluster } from "./cluster"; + +const schema = new PgSchema( + "users", + { + id: { type: ID, autoInsert: "nextval('users_id_seq')" }, + email: { type: String }, + is_admin: { type: Boolean, autoInsert: "false" }, + }, + ["email"] +); + +export class EntUser extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.id, + privacyLoad: [new AllowIf(new OutgoingEdgePointsToVC("id"))], + privacyInsert: [], + }); + } +} +``` + +{% endcode %} + +If your app uses UUID type for IDs, replace `{ type: ID, autoInsert: "nextval('users_id_seq')" }` with something like: + +```typescript +id: { type: String, autoInsert: "gen_random_uuid()" } +``` + +(Notice that you need to use type `String` and not `ID` for UUID fields. Read more about ID formats and microsharding aspects in [Locating a Shard and ID Format](/scalability/locating-a-shard-id-format) article.) + +Each Ent may also have one optional "unique key" (possible composite) which is treated by the engine in a specific optimized way. In the above example, it's `email`. + +{% code title="ents/EntTopic.ts" %} + +```typescript +import { PgSchema } from "ent-framework/pg"; +import { + ID, + BaseEnt, + GLOBAL_SHARD, + AllowIf, + OutgoingEdgePointsToVC, + Require, +} from "ent-framework"; +import { cluster } from "./cluster"; + +const schema = new PgSchema( + "topics", + { + id: { type: ID, autoInsert: "nextval('topics_id_seq')" }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + slug: { type: String }, + creator_id: { type: ID }, + subject: { type: String, allowNull: true }, + }, + ["slug"] +); + +export class EntTopic extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.creator_id, + privacyLoad: [new AllowIf(new OutgoingEdgePointsToVC("creator_id"))], + privacyInsert: [new Require(new OutgoingEdgePointsToVC("creator_id"))], + }); + } +} +``` + +{% endcode %} + +By default, all fields are non-nullable (unless you provide `allowNull` option). + +Disregard privacy rules for now, it's a more complicated topic which will be covered later. For now, the code should be obvious enough. + +{% code title="ents/EntComment.ts" %} + +```typescript +import { PgSchema } from "ent-framework/pg"; +import { + ID, + BaseEnt, + AllowIf, + CanReadOutgoingEdge, + OutgoingEdgePointsToVC, + Require, +} from "ent-framework"; +import { cluster } from "./cluster"; +import { EntTopic } from "./EntTopic"; + +const schema = new PgSchema( + "comments", + { + id: { type: ID, autoInsert: "nextval('comments_id_seq')" }, + created_at: { type: Date, autoInsert: "now()" }, + topic_id: { type: ID }, + creator_id: { type: ID }, + message: { type: String }, + }, + [] +); + +export class EntComment extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.creator_id, + privacyLoad: [ + new AllowIf(new CanReadOutgoingEdge("topic_id", EntTopic)), + new AllowIf(new OutgoingEdgePointsToVC("creator_id")), + ], + privacyInsert: [new Require(new OutgoingEdgePointsToVC("creator_id"))], + }); + } +} +``` + +{% endcode %} + +Since we have no microshards yet, `shardAffinity` basically does nothing. We'll talk about microsharding in [Locating a Shard and ID Format](/scalability/locating-a-shard-id-format). + + +# VC: Viewer Context and Principal + +One of the most important Ent Framework traits is that it always knows, "who" is sending some read/write query to the database, and is able to check permissions. Typically, that "who" is a user who opens a web page, or on behalf of whom a background worker job is running, but it can be any other **Principal**. This mechanism is quite different from traditional database abstraction layers or ORMs, which typically lack awareness of the specific user on whose behalf the queries are executed. + +To send a query, you must always have an instance of [VC](https://github.com/clickup/ent-framework/blob/main/docs/classes/VC.md) class in hand (stands for **Viewer Context**). The most important property in a VC is `principal`, it's a string which identifies the party who's acting. Typically, we store some user ID in `vc.principal`. + +It is intentionally not easy to create a brand new VC instance. In fact, you should only do it once in your app (this VC is called "root VC"), and all other VCs created should **derive** from that VC using its methods. + +Below is a basic example for [Next.js](https://nextjs.org/) framework. (Of course you can use any other framework like Express or whatever. Next.js is here only for illustrative purposes, it has nothing to do with Ent Framework.) + +## Integrate with e.g. Google Auth + +For simplicity of the example, we'll plug in "Login with Google" feature to our Next app, and then will use the user's email as a primary method of addressing an EntUser. + +{% code title="app/api/auth/\[...nextauth]/route.ts" %} + +```typescript +import NextAuth from "next-auth"; +import GoogleProvider from "next-auth/providers/google"; + +const handler = NextAuth({ + providers: [ + GoogleProvider({ + clientId: process.env.GOOGLE_ID, + clientSecret: process.env.GOOGLE_SECRET, + }), + ], +}); + +export { handler as GET, handler as POST }; +``` + +{% endcode %} + +Now on any page, you may place a [Sign in button component](https://github.com/dimikot/ent-framework/blob/main/examples/next-example/src/components/SignInButton.tsx): + +{% code title="components/SignInButton.tsx" %} + +```typescript +import { signIn } from "next-auth/react"; +... + signIn("google")}>Sign in +``` + +{% endcode %} + +Next.js exposes `getServerSession()` function for server components, to allow you access the session data of the user, including their email: + +{% code title="app/page.tsx" %} + +```typescript +import { getServerSession } from "next-auth"; + +export default async function Home() { + const session = await getServerSession(); + return session ? ( +
Welcome, {session.user?.name}!
+ ) : ( +
Please sign in to continue.
+ ); +} +``` + +{% endcode %} + +You can also use `getServerSession()` from inside of your API route handlers. + +## Build a Request VC Accessor Function + +The same way as `getServerSession()` gives us access to the user's session, let's build a function that returns a VC instance for that user. Technically, this function should work exactly the same way as `getServerSession()`: it will even use `session.user.email` field from there. + +And in case the user is not authenticated yet, we still need a "guest VC" to be returned by this function. Such VC can still access some "public" Ents (depending on their privacy rules). + +The VC instance should be "memoized" per the HTTP request, so if the VC accessor function is called multiple time, it should return the same object. This is critical: otherwise, many Ent Framework features (like queries batching and caching) will just not work as they should. + +Different frameworks have different ways of attaching a property to the request object. In Next, the easiest way so far is to use `WeakMap` and `headers()` API function. (In Express, you would likely just assign a value to `req.vc` in some middleware.) + +{% code title="ents/getServerVC.ts" %} + +```typescript +import { VC } from "ent-framework"; +import { getServerSession } from "next-auth"; +import { headers } from "next/headers"; +import { EntUser } from "./EntUser"; + +const vcStore = new WeakMap(); + +export async function getServerVC(): Promise { + const [heads, session] = await Promise.all([headers(), getServerSession()]); + let vc = vcStore.get(heads); + if (!vc) { + vc = VC.createGuestPleaseDoNotUseCreationPointsMustBeLimited(); + if (session?.user?.email) { + const vcOmni = vc.toOmniDangerous(); + let user = await EntUser.loadByNullable(vcOmni, { + email: session.user.email, + }); + if (!user) { + // User did not exist: upsert the Ent. + await EntUser.insertIfNotExists(vcOmni, { + email: session.user.email, + is_admin: false, + }); + user = await EntUser.loadByX(vcOmni, { + email: session.user.email, + }); + } + // Thanks to EntUser's privacyInferPrincipal rule, user.vc is + // automatically assigned to a new derived VC with principal equals to + // user.id. + vc = user.vc; + } + vcStore.set(heads, vc); + } + return vc; +} +``` + +{% endcode %} + +We will discuss what `loadByX()` is in the next sections. In short, it **loads** an Ent **by** unique key and throws an e**X**ception (this is what "X" stands for) if it doesn't exist. + +Here comes the catch: `loadByX()` requires to pass a VC whose principal is the user loading the data. And to derive that VC, we need to call `EntUser#loadByX()`. In our case, it's obviously a "chicken and egg" problem, so we just derive a new VC in "god mode" with `vc.toOmniDangerous()` and allow Ent Framework to bypass privacy checks for the very 1st `EntUser` loaded. + +## Use getServerVC() in Your Server Components and APIs + +So now, everywhere you could use `getServerSession()`, you can use `getServerVC()` as well. + +For instance, in a server component: + +{% code title="app/page.tsx" %} + +```typescript +import { getServerVC } from "@/ents/getServerVC"; + +export default async function Home() { + const vc = await getServerVC(); // <--- + return !vc.isGuest() ? ( +
Your vc.principal={vc.principal}.
+ ) : ( +
Please sign in to continue.
+ ); +} +``` + +{% endcode %} + +Or in an API route handle: + +{% code title="app/api/topics/route.ts" %} + +```typescript +import { EntTopic } from "@/ents/EntTopic"; +import { getServerVC } from "@/ents/getServerVC"; +import { NextApiRequest } from "next"; +import { NextResponse } from "next/server"; + +export async function POST(req: NextApiRequest) { + const vc = await getServerVC(); // <--- + const topic = await EntTopic.insertReturning(vc, { + slug: `t${Date.now()}`, + creator_id: vc.principal, + subject: req.body.subject, + }); + return NextResponse.json({ id: topic.id }); +} +``` + +{% endcode %} + +In other frameworks, you would access the per-request VC differently. For instance, in Express, you would likely just read `req.vc` value that you earlier assigned in a middleware. + + +# Ent API: insert\*() + +Ent Framework exposes an opinionated API which allows to write and read data from the microsharded database. + +{% code title="app/api/topics/route.ts" %} + +```typescript +import { EntComment } from "@/ents/EntComment"; +import { EntTopic } from "@/ents/EntTopic"; +import { EntUser } from "@/ents/EntUser"; +import { getServerVC } from "@/ents/getServerVC"; +import { NextApiRequest } from "next"; +import { NextResponse } from "next/server"; + +export async function POST(req: NextApiRequest) { + const vc = await getServerVC(); + const user = await EntUser.loadX(vc, vc.principal); + const topic = await EntTopic.insertReturning(vc, { + slug: `t${Date.now()}`, + creator_id: user.id, + subject: String(req.body.subject || "My Topic"), + }); + const commentID = await EntComment.insert(topic.vc, { + topic_id: topic.id, + creator_id: user.id, + message: String(req.body.subject || "My Message"), + }); + return NextResponse.json({ + message: `Created topic ${topic.id} and comment ${commentID}`, + }); +} +``` + +{% endcode %} + +There are several versions of `insert*` static methods on each Ent class. + +## **insertIfNotExists(vc, { field: "...", ... }): string | null** + +inserts a new Ent and returns its ID or null if the Ent violates unique index constraints. This is a low-level method, all other methods use it internally. + +## **insert(vc, { field: "...", ... }): string** + +Inserts a new Ent and returns its ID. + +Throws `EntUniqueKeyError` if it violates unique index constraints. Always returns an ID of just-inserted Ent. + +## **insertReturning(vc, { field: "...", ... }): Ent** + +Same as `insert()`, but immediately loads the just-inserted Ent back from the database and returns it. The reasoning is that the database may have fields with default values or even PG triggers, so we always need 2 round-trips to get the actual data. + +{% hint style="info" %} +In fact, `insert*()` methods do way more things. They check privacy rules to make sure that a VC can actually insert the data. They call Ent triggers. They infer a proper microshard to write the data to. We'll discuss all those topics later. +{% endhint %} + +## VC Embedding + +When some Ent is loaded in a VC, its `ent.vc` is assigned to that VC. In the above example, we use `req.vc` and `topic.vc` interchangeably.\ +\ +**Embedding a VC into each Ent is a crucial aspect of Ent Framework.** It allows to remove **lots** of boilerplate from the code. Instead of passing an instance of some VC everywhere from function to function, we can just pass Ents, and we'll always have an up-to-date VC: + +```typescript +async function loadTopicOfComment(comment: EntComment) { + return EntTopic.loadX(comment.vc, comment.topic_id); +} + +async function loadTopicOfCommentUglyDontDoItPlease(vc: VC, commentID: string) { + return EntTopic.loadX(vc, commentID); +} +``` + +You almost never need to pass a VC from function to function: pass Ent instances instead. Having an explicit `vc` argument somewhere is a smell. + + +# Built-in Field Types + +Before we move to the next Ent API calls, let's talk about the Ent field types that are natively supported in Ent Framework: + +
Field DefinitionTypeScript TypePostgreSQL Type
{ type: String }stringvarchar, text, bigint, numeric, ...
{ type: ID }stringvarchar, text, bigint, ...
{ type: Number }numberint, bigint, doube, ...
{ type: Date }Datetimestamptz, timestamp
{ type: Boolean }booleanboolean
{ type: EnumType<"a" | "b">() }"a" | "b"varchar, text, ...
{ type: EnumType<42 | 101>() }42 | 101integer, ...
{ type: EnumType<MyEnum>() }MyEnumvarchar, text, integer, ...
{ type: YourCustomType }see
/pages/05UZ44mjyfabZnuevqwM		jsonc, bytea or anything else
+ +You can also define custom field types: [Custom Field Types](/getting-started/custom-field-types) + +Fields may be *nullable* and *optional*, with the corresponding support from TypeScript side. + +Nullability and optionality concepts are often times mixed up. In Ent Framework, they are independent on each other and are used for different use cases. + +## Nullability: allowNull=true + +By default, all fields can't store a `null` TypeScript value. To allow storing of a null, use the `allowNull` syntax: + +```typescript +const schema = new PgSchema( + "topics", + { + ... + // TypeScript type will be: string | null. + company_id: { type: ID, allowNull: true }, + // TypeScript type will be: string (non-nullable). + slug: { type: String }, + }, + ["slug"] +); +``` + +**Notice that if your field is nullable, it doesn't mean that it is optional.** Nullability and optionality are independent concepts in both Ent Framework and TypeScript. E.g. you can have a required nullable field which allows saving `null` in it, but you will still need to explicitly pass this `null` in your TypeScript code: + +```typescript +await EntTopic.insertReturning(vc, { slug: "abc" }); +// ^ TypeScript error: missing required property, company_id. + +await EntTopic.insertReturning(vc, { company_id: null, slug: "abc" }); +// ^ OK. +``` + +By default, each field in the schema is **required at insert time**. I.e. if you run an `insert*()` call, then TypeScript won't let you skip a required field. + +## Optionality: autoInsert="..." + +To make a field optional, you can use `autoInsert="sql expression"` modifier: it makes the field optional at insert time. Ent Framework will use the raw SQL expression provided if you don't mention an explicit field value on an insert (which is convenient when doing refactoring for instance). + +Several examples: + +```typescript +const schema = new PgSchema( + "topics", + { + // If not passed in insert*() call, uses nextval('topics_id_seq'). + id: { type: ID, autoInsert: "nextval('topics_id_seq')" }, + // If not passed in insert*() call, uses now(). + created_at: { type: Date, autoInsert: "now()" }, + // If not passed in insert*() call, uses NULL. + company_id: { type: ID, allowNull: true, autoInsert: "NULL" }, + // Required AND non-nullable at the same time. + slug: { type: String }, + }, + ["slug"] +); +``` + +Notice that now `company_id` field is both *optional* and *nullable*. I.e. you can run this code: + +```typescript +await EntTopic.insertReturning(vc, { slug: "abc" }); +// ^ OK: company_id is both optional and nullable. +``` + +An example of optional, but non-nullable field is `created_at`. I.e. you can omit this field when inserting (and thus, Ent Framework will use `now()` SQL expression for its value), but you can't pass a `null` TypeScript value there, and your `topic.created_at` will be of type `Date`, not `Date | null` or `Date | undefined`. + +### autoUpdate + +There is also one more way to mark the field as optional, use `autoUpdate` modifier. It is very similar to `autoInsert`, but additionally, if the value is omitted at an `update*()` call, then it will be automatically set to the result of the provided SQL expression. A classical use case for it is `updated_at` field: + +```typescript +const schema = new PgSchema( + "topics", + { + // Defaults to now() if not mentioned at insert time. + created_at: { type: Date, autoInsert: "now()" }, + // Auto-set to now() if not mentioned at update time. + updated_at: { type: Date, autoUpdate: "now()" }, + ... + }, + ["slug"] +); +``` + + +# Ent API: load\*() by ID + +There is a basic primitive used very frequently: having some Ent ID, load this Ent into memory. + +{% code title="app/api/comments/\[id]/route.ts" %} + +```typescript +import { EntComment } from "@/ents/EntComment"; +import { getServerVC } from "@/ents/getServerVC"; +import { NextApiRequest } from "next"; +import { NextResponse } from "next/server"; + +export async function GET( + _req: NextApiRequest, + { params }: { params: Promise<{ id: string }> } +) { + const vc = await getServerVC(); + const comment = await EntComment.loadX(vc, (await params).id); + return NextResponse.json({ message: comment.message }); +} +``` + +{% endcode %} + +There are several versions of `load*` static methods on each Ent class: + +## **Ent.loadX(vc, id): Ent** + +Loads an Ent by ID. + +Throws `EntNotFoundError` if there is no such Ent in the database, or `EntNotReadableError` if the VC has no permissions to read it. + +## **Ent.loadNullable(vc, id): Ent | null** + +loads an Ent by ID if it exists in the database, otherwise returns null. + +If an Ent with such ID exists, but the VC doesn't have permissions to access it, the call will throw `EntNotReadableError`. + +## **Ent.loadIfReadableNullable(vc, id)**: Ent | null + +This is a special method designed to return `null` in two cases: when an Ent with the specified ID does not exist, or when the user lacks the necessary permissions to read it. Basically, it never throws. + +Permissions are enforced by the `privacyLoad` rules of the Ent, which were briefly introduced earlier and will be covered in more detail later. + +{% hint style="info" %} +In most of the cases, prefer `loadX()` and rely on the outer try-catch blocks, as opposed to `loadNullable()` with manual null-checking. Let the framework do its job. And you likely almost never need to use `loadIfReadableNullable()`: it's a smell. +{% endhint %} + +There is intentionally no method which loads multiple Ents at once taking an array of IDs. Read further on, why. + + +# N+1 Selects Solution + +To reveal some magic, could you please make a small favor? + +**Stop thinking in terms of lists when loading.** Always think in terms of an individual row/object and an individual ID. Not in terms of an array of IDs: + +```typescript +async function loadCommentsBadDontDoThis(ids: string[]): Promise { + // Please don't. +} + +async function loadComment(id: string): Promise { + // Do this: one ID as an input, one row as an output. +} +``` + +It sounds contradictory. In the example above, if we always use `loadComment(id)`, how do we avoid sending too many queries to the database, especially when it comes to loading children records for each loaded parent? (This problem is well known as "N+1 Selects".) + +The answer is: **let the DB access engine take care of batching**. + +## Traditional List Based Approach + +Imagine we have some list of comment IDs shown on the screen. For each comment, we want to load its creator, the owning topic, and for each topic, load its creator too. Then, return it all as a JSON to the client. + +Of course we want to send as few SQL queries to the database as possible to minimize connections utilization and round-trip latency. We also do not want to use JOINs (imagine `loadUsers()`, `loadTopics()` and `loadComments()` live in independent modules and don't want to know about each other, plus the data lives in different microshards). + +First, let's see, what will happen if we think in terms of "load a list of things" abstraction. This is how people used to fight the "N+1 Selects" problem in the past. + +```typescript +import { map, uniq, keyBy } from "lodash"; + +async function loadUsers(ids: string): Promise { + return sql.query("SELECT * FROM users WHERE id = ANY($1)", ids); +} + +async function loadTopics(ids: string): Promise { + return sql.query("SELECT * FROM topics WHERE id = ANY($1)", ids); +} + +async function loadComments(ids: string[]): Promise { + return sql.query("SELECT * FROM comments WHERE id = ANY($1)", ids); +} + +// Loads data using just 3 SQL queries. +app.get("/comments", async (req, res) => { + const commentIDs = String(req.query.ids).split(","); + const comments = keyBy(await loadComments(commentIDs), "id"); + + const topicIDs = uniq(map(comments, (comment) => comment.topic_id)); + const topics = keyBy(await loadTopics(topicIDs), "id"); + + const userIDs = uniq([ + ...map(comments, (comment) => comment.creator_id), + ...map(topics, (topic) => topic.creator_id), + ]); + const users = keyBy(await loadUsers(userIDs), "id"); + + res.json( + map(comments, (comment) => ({ + comment, + commentCreator: users[comment.creator_id], + topic: topics[comment.topic_id], + topicCreator: users[topics[comment.topic_id].creator_id], + })) + ); +}); +``` + +Look at this spaghetti mess. The code appears very coupled. + +The root of the problem here is clear: we think in terms of the lists, and the code encourages us to "accumulate" lists manually. + +### Ent Framework Approach: Automatic Batching + +Now let's see what happens if we stop thinking in terms of lists and, instead, switch to "per individual object" paradigm. + +```typescript +// Still using just 3 SQL queries. But wait a second... +app.get("/comments", async (req, res) => { + const commentIDs = uniq(String(req.query.ids).split(",")); + res.json( + await Promise.all( + commentIDs.map(async (commentID) => { + const comment = await EntComment.loadX(req.vc, commentID); + const topic = await EntTopic.loadX(req.vc, comment.topic_id); + const [commentCreator, topicCreator] = await Promise.all([ + EntUser.loadX(req.vc, comment.creator_id), + EntUser.loadX(req.vc, topic.creator_id), + ]); + return { comment, commentCreator, topic, topicCreator }; + }) + ) + ); +}); + +``` + +All calls to `uniq()`, `keyBy()` and `map()` are gone. We now use only `loadX(vc, id)` which accepts an individual ID and returns an individual Ent. + +And still, it runs only 3 SQL queries under the hood: + +```sql +SELECT * FROM comments WHERE id IN(...); +SELECT * FROM topics WHERE id IN(...); +SELECT * FROM users WHERE id IN(...); +``` + +* **Batching:** Ent Framework recognizes that the `loadX()` calls happen in concurrent Promises and batches them together intelligently. +* **Coalescing:** in case multiple `loadX(vc, id)` try to load the same Ent by the same ID, Ent Framework coalesces those calls into one. +* **Caching:** if enabled, an Ent loaded in some VC remains in the VC's cache, so next time it's attempted to load again, the Ent is returned from the cache directly. Ents are immutable JS objects, so it simplifies things even further. + +{% hint style="info" %} +In fact, Ent Framework does similar batching not only for `loadX()`. It batches all other calls too, including inserts, updates, deletes and even more complicated expression-based multi-row selects. +{% endhint %} + +To learn more about batching, "parallel Promises", and how event loop works in Node, check out [Loaders and Custom Batching](/advanced/loaders-and-custom-batching) article. + +## Helper Loading Methods + +Each Ent is an immutable object, which means that you can't change its fields after loading from the DB. But you can add helper methods to simplify things like loading. + +Let's optimize the above example even further by adding `topic()` and `creator()` helper methods into Ent classes directly. + +```typescript +class EntComment extends ... { + async topic() { + return EntTopic.loadX(this.vc, this.topic_id); + } + + async creator() { + return EntUser.loadX(this.vc, this.creator_id); + } +} + +class EntTopic extends ... { + async creator() { + return EntUser.loadX(this.vc, this.creator_id); + } +} + +app.get("/comments", async (req, res) => { + const commentIDs = String(req.query.ids).split(","); + res.json( + await mapJoin(commentIDs, async (commentID) => { + const comment = await EntComment.loadX(req.vc, commentID); + const topic = await comment.topic(); + const [commentCreator, topicCreator] = await Promise.all([ + comment.creator(), + topic.creator(), + ]); + return { comment, commentCreator, topic, topicCreator }; + }) + ); +}); +``` + +{% hint style="info" %} +`mapJoin(arr, fn)` is a simple wrapper which calls `Promise.all(arr.map(fn))`. +{% endhint %} + +Now it's responsibility of each Ent to load the related data. + +This will, as previously, produce the same exact 3 DB queries: + +```sql +SELECT * FROM comments WHERE id IN(...); +SELECT * FROM topics WHERE id IN(...); +SELECT * FROM users WHERE id IN(...); +``` + +In traditional ORMs, such helper loading methods are added to the classes automatically. Ent Framework doesn't do it and requires you to write a bit of boilerplate. Why? For general purpose use cases, we may need not one, but 2 method for each field, like `creator()` and `creatorNullable()`, which is not elegant. This is because foreign keys do not work reliably enough across microshards, so in some cases, we should always be ready that some Ent is not in the database, even when its field is technically non-nullable. Luckily, in practice, it is not hard at all to add such methods manually, so we don't lose too much here. + +## Batching vs. JOINs + +In traditional SQL and in many ORMs, people use JOINs to minimize the number of queries they send to the database engine. Despite the JOINs having advantages, they are also problematic: + +1. One cannot do JOINs across microshards or machines. +2. JOINs encourage people to write highly coupled code, similar to the 1st example on this page. +3. JOINs generally can't run their subqueries in parallel. + +Ent Framework's automatic batching can be treated as an alternative to JOINs. It doesn't have any of the above problems, plus (and more importantly), the calls are batched across the entire async functions call stack, which means that you can split the code into independent abstraction layers easily. + +Stop thinking in terms of lists. Start thinking in terms of an individual Ent and its behavior. + +{% hint style="info" %} +Of course, in some cases, we still want to run JOINs. Ent Framework exposes low-level API to get access to the underlying DB, so you can craft and run arbitrary queries. It also provides you with a `Loader` abstraction and framework to build your own custom batching strategies. We'll discuss it all in details in the advanced section. +{% endhint %} + + +# Automatic Batching Examples + +In the previous chapter, we talked about Ent Framework calls batching. Let's provide some more examples. + +## Batching of load\*() Calls + +The following code will produce only one SQL query: + +```typescript +await Promise.all([ + EntTopic.loadX(vc, "123"), + EntTopic.loadX(vc, "456"), + EntTopic.loadX(vc, "789"), +]); +``` + +SQL query produced under the hood: + +```sql +SELECT * FROM topics WHERE id IN(...) +``` + +## Batching of insert\*() Calls + +Since `insertReturning()` first inserts the Ent into the database and then loads the inserted data back, the following code will produce 2 SQL queries. + +```typescript +await Promise.all([ + EntTopic.insertReturning(vc, { ... }), + EntTopic.insertReturning(vc, { ... }), + EntTopic.insertReturning(vc, { ... }), +]); +``` + +SQL queries produced: + +```sql +INSERT INTO topics (...) VALUES ... RETURNING id; +SELECT * FROM topics WHERE id IN(...); +``` + +Even if `insertReturning()` is called in nested functions, Ent Framework will still batch them properly and produce just 2 queries: + +```typescript +async function insertTopicsBatch(n: number) { + await mapJoin(range(n), async (i) => EntTopic.insertReturning(vc, { ... })); +} +... +await Promise.all([ + insertTopicsBatch(42), + insertTopicsBatch(101), +]); +``` + +## Batching of Update, Delete and all Other Calls + +All Ent Framework API calls are subject for batching the way --described above. + +## De-batching and Deadlocks + +As in most of MVCC databases, In PostgreSQL, reads never block writes, and writes never block reads. Still, if two clients update the same row in the database, one client has to wait for another one to finish. + +If the order of row updates is different in two clients, there is a change of [deadlocks](https://www.postgresql.org/docs/current/runtime-config-locks.html). E.g. imagine Alice updates row A and then row B in the same transaction, whilst Bob first updates B and then A. In this case, Alice will wait until Bob finishes updating row B, but at the same time, Bob will wait until Alice commits the transaction updating A. Thus, they would wait for each other infinitely, and PostgreSQL will cancel one of the transactions. (Notice that this situation never happens when both Alice and Bob update rows A and B in the same order.) + +Deadlocks may occur during the automatic queries batching. It is rare (especially since Ent Framework always orders the updating rows in a consistent way, by e.g. id), but may still happen. + +In case of a rare deadlock, when Ent Framework knows that it's safe to retry the write, it performs *de-batching*: splits the batched query into individual queries and runs them in parallel, independently. This solves the problem of deadlocks entirely, in an exchange of very rare slowdown of the mass insert, update or delete operations. + + +# Ent API: select() by Expression + +The previous chapters explained how to load an Ent by its ID: `load*()` API. Loading by ID is the most basic operation, and it is usually the most common one in the code as well. Now, let's talk about some more complicated ways of loading. + +TL;DR: + +```typescript +const comments = await EntComment.select( + vc, + { + topic_id: "123", + created_at: { $gte: new Date(Date.now() - 1000 * 3600 * 24) }, + }, + 100, // limit + [{ created_at: "ASC" }] // order by +); +``` + +## Dry Boring Theory + +Below, there will be a bit of theory, fasten your seatbelt. + +In graph terms, where each Ent is a **node**, an Ent's field that points to the ID of another Ent represents an **edge**. (Or, in relational databases, people typically use "foreign key" term.) We often refer to it as **field edge**; traversing such edges is typically straightforward: you simply load another Ent by the ID obtained from a field of the current Ent. For example, `EntComment#topic_id` or `EntTopic#creator_id` are field edges. + +From a different perspective, traversing a field edge can be seen as "going from a child Ent to a parent Ent" (for example, from `EntComment` to its owning `EntTopic`). In other words, it’s a **child-to-parent traversal**, or a **many-to-one relationship:** + +{% @mermaid/diagram content="classDiagram +direction BT +class EntComment\["EntComment
(child)"] +EntComment : topic\_id +EntComment : creator\_id +class EntTopic\["EntTopic
(parent)"] +EntTopic : creator\_id +class EntUser\["EntUser
(grandparent)"] +EntComment --> EntTopic : field
edge +EntTopic --> EntUser : field
edge +EntComment --> EntUser : field
edge" %} + +Nothing too new yet, right? Just a regular relational theory so far. + +How do we go in the opposite direction, performing a **parent-to-children traversal** in a **one-to-many relationship**? + +To accomplish this, Ent Framework provides (surprise!) a `select()` primitive. It allows you to fetch Ents from the database using any arbitrary expression, including those that specify constraints on which parent Ent's ID the selected Ents should have: + +```typescript +const comments = await EntComment.select( + vc, + { topic_id: "123" }, // "load all children comments of topic 123" + 100, // limit + [{ created_at: "ASC" }] // order by +); +``` + +In production databases with millions of Ents, it's assumed that the relevant table has a necessary index to run such queries efficiently; in the above example, + +```sql +CREATE INDEX ON comments_topic_id_created_at ON comments(topic_id, created_at); +``` + +Nothing new again. Or there is something?.. + +{% @mermaid/diagram content="classDiagram +direction BT +class EntComment1\["EntComment
(child)"] +EntComment1 : topic\_id +class EntComment2\["EntComment
(child)"] +EntComment2 : topic\_id +class EntComment3\["EntComment
(child)"] +EntComment3 : topic\_id +class EntTopic\["EntTopic
(parent)"] +EntTopic : id +EntComment1 <-- EntTopic : ??? +EntComment1 --> EntTopic : field
edge +EntComment2 <-- EntTopic : ??? +EntComment2 --> EntTopic : field
edge +EntComment3 <-- EntTopic : ??? +EntComment3 --> EntTopic : field
edge" %} + +Let's think about those `???` on the diagram. To traverse edges in a graph in both directions, the edges must be bi-directional (or, there should be pairs of edges, which is the same). In the graph with bi-directional edges we discussed earlier, the child-to-parent direction of an edge is represented by an "Ent field edge". But what corresponds to `???`, the opposite **parent-to-children direction** of that edge? + +This `???`, dear friends, is the **automatic database index** (or an index prefix, which is `topic_id` in the example). In fact, as we hinted above, without such an index, the queries will just blow up. + +This distinction between graph edge directions is crucial to understand: for free traversal of the graph, both **field edges** and **indexes** are absolutely essential. + +* By defining a DB foreign key on an Ent, you define a field edge, which represents child-to-parent direction in the graph. +* By defining a DB index, you define the opposite direction of that edge, which is parent-to-children direction. + +Modern database engines are pretty good at managing indexes. You can add them without acquiring write locks on the tables (`CREATE INDEX CONCURRENTLY`), and you can also add more field edges (aka fields with foreign keys) on a table with no downtime, to refer some other Ent from an existing one. + +## What About Microsharding and Horizontal Scaling? + +The point of view described above works straightforwardly when your database is monolithic. Scaling your app introduces more complexity though due to the involvement of microshards in the traversal process. + +Luckily, we can still rely on the parent-to-children indices mainly. And this is where the theory pays off. + +When loading children of a parent Ent, the children might be distributed across multiple microshards. A naive way would thus be to just query all microshards using the exact same query (Ent IDs are globally unique) and then merge the results, but of course it would blow up the DB nodes. + +Therefore, before Ent Framework executes the actual SELECT queries in parallel on multiple nodes to merge their results later, it first determines the **minimal set of microshards** that needs to be queried; in the vast majority of cases, this is just **one microshard**. Those mechanisms are known as [Inverses](/scalability/inverses-cross-shard-foreign-keys) and [Ent Colocation](/scalability/shard-affinity-ent-colocation) correspondingly, and we’ll explore them in detail later, in advanced sections. + +For now, all you need to know is that there is a magical subsystem in Ent Framework called Inverses which, given a parent ID (e.g. EntTopic ID), returns the list of microshards where the children Ents (e.g. EntComment) may **or may not** reside. This "may not" is important: cross-shard writes are not transactional, so sometimes (rarely), slightly more candidate microshards may be returned, but **never less**. In reality it produces no problems for business logic: the "excess" microshards, when queried, will just return 0 children Ents. + +## Ent.select(vc, { ... }, limit, order): Ent\[] + +The `select()` API uses a simple query language. + +If a plain object is passed, it combines all the specified field constraints using an AND operation: + +```typescript +const comments = await EntComment.select( + vc, + { + topic_id: ["123", "456"], + created_at: { $gte: new Date(Date.now() - 1000 * 3600 * 24) }, + }, + 100, // limit + [{ created_at: "ASC" }] // order by +); +``` + +The full list of operations include: + +* equality and "one of array element" implicit operators (see examples above) +* logical: `$or`, `$and`, `$not` +* binary: `$lte`, `$lt`, `$gte`, `$gt` +* `$overlap` (useful for array fields, typically backed by a PostgreSQL GIN index) +* `$isDistinctFrom` (for NULL-safe comparisons) +* `$literal` (to run a custom SQL sub-expression) + +These operations can be nested in any way, but it's important to ensure that the actual SQL engine uses an appropriate database index for efficiency. + +If your project uses microsharding, one of the top-level fields in the `select()` expression must match a parent ID or an array of parent IDs to help Ent Framework identify the relevant microshards. Notice that we used `topic_id` for this purpose in the example above. There’s no magic here: sometimes, it has to determine, which microshards are involved. Alternatively, you can use the special `$shardOfID` operator to explicitly provide this hint in the query. + +For illustrative purposes, below is a giant `select()` expression from one of Ent Framework's unit tests. It is generally obvious, how the operations work (as opposed to e.g. Elasticsearch query language BTW): + +```typescript +const ents = await EntSome.select( + vc, + { + name: ["aa", "bb"], // matches one of + some_flag: true, + $or: [ + { name: "aa" }, + { name: "bb" }, + { url_name: [] }, // will never match + { url_name: [null, "zzz"] }, // null-safe + ], + $and: [ + { name: ["aa", "bb"] }, + { name: { $ne: "kk" } }, + { name: { $isDistinctFrom: "dd" } }, + { url_name: { $isDistinctFrom: null } }, // null-safe != + { url_name: { $ne: ["kk", null] } }, // null-safe too + { url_name: { $ne: [] } }, // will always match + { $literal: ["? > '2'", "5"] }, + { name: { $lte: "y", $gte: "a" } }, + { $overlap: [id1, id2, id3] } // most likely you want a GIN index here! + ], + $not: { + name: "yy", + $literal: ["length(name) < ?", 5], // custom SQL expression + }, + // Optional; it's for the cases when you don't really have a field edge, + // so Ent Framework can't infer microshards from the query. + $shardOfID: "12345", + }, + 100, + [{ name: "ASC" }, { url_name: "DESC" }, { $literal: ["1=?", 2] }] +); +``` + +For more details, see TypeScript `Where<...>` definition in [types.ts](https://github.com/clickup/ent-framework/blob/main/src/types.ts). + +## Batching of select() Calls + +As everything in Ent Framework, when multiple `select()` calls run in parallel, they are batched into one giant SQL `UNION ALL` query. + +The following code will produce only one SQL query: + +```typescript +await Promise.all([ + EntComment.select(vc, { topic_id: "42" }, 10), + EntComment.select(vc, { creator_id: "101" }, 20), +]); +``` + +SQL query produced under the hood (with some simplifications): + +```sql +SELECT * FROM topics WHERE topic_id='42' +UNION ALL +SELECT * FROM topics WHERE creator_id='101' +``` + +Sometimes, `select()` calls are meant to be relatively slow, and we don't want to batch them; instead, we prefer to run them in parallel, in different DB connections. To do so, you can just inject an "event loop spin" barrier: + +```typescript +// Never produces a UNION ALL SQL query. +await Promise.all([ + EntComment.select(vc, { topic_id: "42" }, 10), + new Promise(setImmediate).then( + async () => EntComment.select(vc, { creator_id: "101" }, 20), + ), +]); +``` + +## JOIN, WITH, FROM and Subqueries, Planner Hints + +In addition to database-independent features, `select()` call also supports engine-specific customizations using its last optional argument: + +```typescript +const comments = await EntComment.select( + vc, + { creator_id: "101" }, + 20, // limit + undefined, // order + { joins, ctes, from, hints }, // untyped, but of type SelectInputCustom +); +``` + +Read more in: + +* [PostgreSQL Specific Features](/advanced/postgresql-specific-features) +* [Query Planner Hints](/advanced/query-planner-hints) + + +# Ent API: loadBy\*() Unique Key + +Each Ent usually has an `id` field, serving as its primary key. This enables other Ents to reference it and allows for the use of the highly optimized `loadX(vc, id)` method to load by ID. + +Some Ents may also have a **secondary unique key**. This could be a single text field or a combination of multiple fields. For example, `EntUser` might have an `email` field that must be unique across all `EntUser` rows in the database: + +To use a unique key, define it in the Ent's schema and ensure that the corresponding unique index exists in the database: + +```typescript +const schema = new PgSchema( + "users", + { + id: { type: ID, autoInsert: "nextval('users_id_seq')" }, + email: { type: String }, + }, + ["email"] +); +``` + +Once set up, you can use the following methods to load by a unique key. + +## **Ent.loadByX(vc, { email: "..." }): Ent** + +Loads an Ent by its unique key defined in the schema. + +If no matching row is found in the table, throws an `EntNotFound` error. + +## **Ent.loadByNullable(vc, { email: "..." }): Ent | null** + +**W**orks the same way as the above method, but returns `null` if no matching Ent is found. + +## Batching and by-Prefix Grouping + +As always, if multiple `loadBy*()` calls occur in parallel, Ent Framework batches them into a single SQL query to save on the connections utilization, latency and index usage. + +If the unique key consists of a single field (e.g., `email`), the batched query for: + +```typescript +const [user1, user2] = await Promise.all([ + EntUser.loadByX(vc, { email: "test1@example.com" }), + EntUser.loadByX(vc, { email: "test2@example.com" }), +]); +``` + +looks like this: + +```sql +SELECT * FROM table WHERE email IN('test1@example.com', 'test2@example.com'); +``` + +For a composite unique key (e.g., `creator_id` and `slug`), the batched query for: + +```typescript +const topics = await Promise.all([ + EntTopic.loadByX(vc, { creator_id: "123", slug: "abc" }), + EntTopic.loadByX(vc, { creator_id: "123", slug: "def" }), + EntTopic.loadByX(vc, { creator_id: "456", slug: "ghi" }), + EntTopic.loadByX(vc, { creator_id: "456", slug: "jkl" }), +]); +``` + +is more complex: + +```sql +SELECT * FROM table WHERE + (creator_id='123' AND slug IN('abc', 'def')) OR + (creator_id='456' AND slug IN('ghi', 'jkl')); +``` + +In other words, the engine groups the requests by the **prefix of the unique key, excluding the last field**, and then uses an `IN` clause for the values of the last field. This strategy allows the database to utilize its B-tree unique indexes efficiently. Just make sure that the column order in the database index matches the field order in the unique key in Ent's schema, and that columns with the lowest cardinality appear first in the unique index prefix. + + +# Ent API: update\*() + +Previously, we looked at Ent Framework APIs which were "per Ent class", represented as Ent static methods. + +In contrast, `update*()` calls are Ent instance methods. It means that, to update an Ent, you first need to load that Ent in memory. This achieves 2 goals: + +1. It brings some extra privacy protection, since to load an Ent, the VC needs to have permissions to do so. +2. It enables you to build sophisticated "mutation" privacy checks, since Ent Framework has access to both the old (before update) and the new (after update) Ent fields. + +There is one caveat though: Ent instances are immutable, so `update*()` methods do not change their fields in memory. Instead, they act similarly to "GraphQL mutations" by modifying the data in the database, and then, if you request so, loading the updated rows back and returning them to you. + +This is why all `update*()` methods have vernose suffixes in their names. + +## **ent.updateOriginal({ field: "...", ... }): boolean** + +Updates the row in the database corresponding to `ent.id` ID. Does not modify any fields of `ent` instance, since it's immutable. + +Runs all the needed privacy checks and Ent Framework triggers (we'll discuss both topics later in advanced chapters). In case there are no `privacyUpdate` rules defined in the Ent class configuration, delegates privacy checking to `privacyInsert` rules. + +Returns true if the row existed in the database at the moment of the update and false otherwise. + +As always, when multiple `update*()` calls run in parallel, Ent Framework batches them into a single SQL query: + +```typescript +const [updated1, updated2] = await Promise.all([ + topic1.updateOriginal({ subject: "some" }), + topic2.updateOriginal({ subject: "text" }), +]); +``` + +**This results into the following batched query sent to the database (the actual query is even more complicated actually, but you can see the general idea below):** + +```sql +WITH rows(id, subject) AS (VALUES( + ('123', 'some'), + ('456', 'text')) + UPDATE topics SET subject=rows.subject + FROM rows WHERE topic.id=rows.id + RETURNING rows.id +``` + +{% hint style="info" %} +All `update*()` functions also support a special `$cas` property; read more about it in [Atomic Updates and CAS](/advanced/atomic-updates-and-cas) advanced article. +{% endhint %} + +## **ent.updateReturningX({ field: "...", ... }): Ent** + +Updates the row in the database, then **loads the Ent back** using `loadX()` and returns it to you. In case there was no such row in the database, throws `EntNotFound` error (this is what "X" stands for, "eXception"). + +Since this methods runs 2 database queries under the hood, any side effects applied by e.g. native PostgreSQL triggers will refect in the loaded Ent. + +As of batching, it also results into running just 2 SQL queries, no matter how many Ents are updated in parallel. The first query is the batched `updateOriginal()`, and the second one is the batched `loadX() for the resulting Ents`: + +```sql +WITH rows(id, subject) AS (VALUES( + ('123', 'some'), + ('456', 'text')) + UPDATE topics SET subject=rows.subject + FROM rows WHERE topic.id=rows.id + RETURNING rows.id; + +SELECT * FROM topics WHERE id IN('123', '456'); +``` + +## **ent.updateReturningNullable({ field: "...", ... }): Ent | null** + +Similarly to `updateReturningX()`, updates the row in the database and loads the updated Ent back, but doesn't throw in case you are trying to update a row which doesn't exist at the moment. + +## **ent.updateChanged({ field1: "...", field2: "...", ... }): string\[] | null | false** + +Same as `updateOriginal()`, but updates only the fields which are different in the method's input and in the current Ent instance in memory. + +* If there is no such row in the database, returns false, the same way as `updateOriginal()` does. +* If no changed fields were detected, returns null as an indication (it's still falsy, but is different from the parent `updateOriginal()'s` false). +* Otherwise, when an update happened, returns the list of fields which were different and triggered that change (a truthy value). + +## **ent.updateChangedReturningX({ field: "...", ... }): Ent** + +This is probably the longest method name in Ent API. Acts similarly to `updateChanged()`, but returns the modified Ent back (or the original Ent if no fields were actully changed). + +## Using $literal Instead of Fields + +In addition to updating particular fields by their names, you can also pass an arbitrary SQL piece containing one more comma separated `field = value` expressions: + +```typescript +await topic.updateOriginal(vc, { + subject: "some", + $literal: [ + "tags = ARRAY(SELECT DISTINCT unnest FROM unnest(array_append(tags, ?)))", + "my-tag", + ] +}); +``` + +In the final SQL query generated, what you pass in `$literal` will appear as it is: + +```sql +UPDATE topics +SET + subject = 'some', + tags = ARRAY(SELECT DISTINCT unnest FROM unnest(array_append(tags, 'my-tag'))) +WHERE id = 1004200047373526525 +``` + +There are several downsides of this approach though: + +1. Calls of this kind can't be batched, so if you run multiple of them in parallel, Ent Framework will send independent queries. +2. The syntax is engine-specific; e.g. the above example works for PostgreSQL only. + + +# Ent API: deleteOriginal() + +Similar to `update*()` calls, `deleteOriginal()` is a method of Ent instance. + +## ent.deleteOriginal(): boolean + +Deletes a row in the database whose ID equals to `ent.id`. Returns true if the object was found. + +Before deleting the row, `deleteOriginal()` runs all privacy checks defined in the Ent class configuration, making sure `ent.vc` has permissions to delete the Ent. In case there are no privacy checks defined for deletion (no `privacyDelete`), uses `privacyUpdate` rules for the verification, and if it's also undefined, delegates to `privacyInsert`. + +Since all Ent instances are immutable, the call keeps the current Ent instance unchanged. This is why it's called `deleteOriginal()` and not just `delete()` — because it's basically a mutation of the source. + +And yes, to delete an Ent, you first need to load it (using e.g. `loadX()`, `select()` or any other call). Also, there is intentionally no way to delete Ents in bulk: you can only delete a single Ent (concurrent deletion calls are batched into one SQL query as usual though, so it's efficient). + + +# Ent API: count() by Expression + +Count API is similar to `select()`, but instead of loading the matching Ents, it counts them. + +## **Ent.count(vc, { field: "...", ... }): number** + +Returns the number of Ents matching the `where` condition. Works across multiple microshards. + +As usual, if multiple `count()` calls for the same Ent are run in parallel, they are internally batched into a single SQL query: + +```typescript +const [count1, count2] = await Promise.all([ + EntTopic.count(vc, { creator_id: "123" }), + EntTopic.count(vc, { updated_at: { $gt: new Date("2024-01-01") } }), +]); +``` + +This sends the following SQL query to the underlying database: + +```sql +SELECT count(1) FROM topics WHERE creator_id='123' + UNION ALL +SELECT count(1) FROM topics WHERE created_at>'...' +``` + +As opposed to `select()`, `load*()` and `loadBy*()` calls, `count()` is privacy-unaware: it does not run privacy checks. This is partially a technical limitation (to recheck privacy, one needs to load the actual rows from the database, and count() doesn’t do it). But also, it’s an intended behavior: with `count()`, it’s convenient to build custom privacy checks and avoid "chicken and egg" problem (to build a privacy check, you eventually need to run a privacy-unaware calls at the very bottom of the stack). + + +# Ent API: exists() by Expression + +This is another privacy-unaware API call, similar to `count()`. + +## **Ent.exists(vc, { field: "...", ... })**: boolean + +Returns true if there isat least one Ent in the database matching the `where` condition. Works across multiple microshards too. + +In terms of the logic, `exists()` call is similar to `count() > 0` check, with two performance optimizations: + +1. It uses `EXISTS` SQL clause, which doesn’t read more tuples from the database than needed (as opposed to `count()` aggregate). +2. During the run, it severely reduces the weight (basically, the probability) of seqscan to happen (with `SET enable_seqscan=off` directive merged with the query). I.e. it implies that you must have a good index covering the `where` condition. + +As all API calls in Ent Framework, multiple parallel `exists()` calls are batched into a single SQL query: + +```typescript +const [exists1, exists2] = await Promise.all([ + EntTopic.exists(vc, { creator_id: "123" }), + EntTopic.exists(vc, { updated_at: { $gt: new Date("2024-01-01") } }), +]); +``` + +This sends the following SQL query to the underlying database: + +```sql +SET enable_seqscan=off; +SELECT EXISTS (SELECT true FROM topics WHERE creator_id='123') + UNION ALL +SELECT EXISTS (SELECT true FROM topics WHERE created_at>'...') +``` + +The `exists()` call is even more useful to build custom privacy checks than `count()`, because it’s faster and almost guarantees using an index. + + +# Ent API: selectBy() Unique Key Prefix + +Similar to how `loadBy()` loads a single Ent by its unique key, `selectBy()` call loads *multiple* ents by their **unique key prefix**. + +## Ent.selectBy(vc, { field: "...", ... }): Ent\[] + +Loads the Ents matching the predicate, considering the predicate is a list of fields from your unique key prefix. + +Logically, you can load the same Ents by just `select()` call, but then, while batching, it will produce a `UNION ALL` clause, which is less efficient and may cause performance problems when a large number of calls are batched. In contrast, `selectBy()` never produces a `UNION ALL` clause, but the price we pay for it is the implication that we can only select by the unique key *prefix*, not by an arbitrary predicate. + +All in all, you’ll rarely need to use `selectBy()` in your code. It is used interally though to fetch [Inverses](/architecture/ent-framework-metas-tao-entgo#no-explicit-assocs) efficiently. + +Let’s actually use Inverses to illustrate, how `selectBy()` works. Internally, the Inverses Ent schema looks like this: + +```typescript +const schema = new PgSchema( + name, + { + id: { type: ID }, + created_at: { type: Date, autoInsert: "now()" }, + type: { type: String }, + id1: { type: ID }, + shard2: { type: Number }, + }, + ["type", "id1", "shard2"], +) +``` + +## Simple Batching + +Sometimes, when Ent Framework needs to discover the full list of microshards on the opposite end of some field edge, it internally runs the following calls in parallel: + +
await Promise.all([
+  EntInverse.selectBy(vc, { type: "user2topics", id1: "123" }),
+  EntInverse.selectBy(vc, { type: "user2topics", id1: "456" }),
+]);
+
+ +Notice that in this example, all parallel calls use the same prefix (`type: "user2topics"`), but the very last selection field varies. For such a case (which is actually pretty common), to produce the most optimal PostgreSQL execution plan, Ent Framework builds the following batched SQL query: + +```sql +SELECT * FROM inverses +WHERE type='user2topics' AND id1 IN('123', '456') +``` + +## Complex Batching + +Unfortunately, the above query stops being optimal when the prefix differs across multiple parallel calls. Consider this example: + +
await Promise.all([
+  EntInverse.selectBy(vc, { type: "user2topics", id1: "123" }),
+  EntInverse.selectBy(vc, { type: "user2topics", id1: "456" }),
+  EntInverse.selectBy(vc, { type: "topic2comments", id1: "789" }),
+]);  
+
+ +Assume we try to build the batched query using the same approach as above: + +```sql +-- DON'T DO IT! +SELECT * FROM inverses WHERE + (type='user2topics' AND id1 IN('123', '456')) OR + (type='topic2comments' AND id1 IN('789')) +``` + +In this case, PostgreSQL will often times produce a suboptimal plan with "bitmap index scan" instead of "index scan". This is partially due to the fact that our DB unique index is by `(type, id1, shard2)`, and we only utilize its prefix `(type, id1)`. + +Luckily, there is another query plan which is used by Ent Framework in such a case: + +```sql +-- Good plan! +SELECT * FROM inverses WHERE (type, id1) IN(VALUES( + ('user2topics', '123'), + ('user2topics', '456'), + ('topic2comments', '789') +)) +``` + +It produces an optimal query plan for the cases when prefixes differ. (BTW, it loses in the situations when the prefix is common, for which `AND id1 IN(...)` clause plays better.) + + +# Ent API: upsert\*() + +The `upsert*()` call is a mix of INSERT and UPDATE operation, based on an Ent unique key. + +## Ent.upsert(vc, { field: "...", ... }): string + +This call tries to update an existing row in the database (i.e. a row with the same unique key **defined in Ent schema**). In case there is no such row yet, it inserts the new one. + +Returns ID of the updated (or inserted) row. + +You can rely on the behavior of `autoInsert` and `autoUpdate` fields: they work the same way as in regular `insert*()` and `update*()` calls. + +Upsert can't work if some triggers are defined for the Ent, because we don't know Ent ID in advance (whether the upsert succeeds or skips on duplication). + +Also, `upsert()` will refuse to run if there are Inverses defined on some Ent fields (same reason: Inverses operations run in a different microshard strictly before the main Ent operation, and they must know the row's ID in advance). + +## Ent.upsertReturning(vc, { ... }): Ent + +This call is very similar to `upsert()`, but in the end, it loads the updated (or inserted) Ent back from the datbase using `loadX()`. + +Since `upsert()` is meant to always succeed (except when there is a transport error, or when some database constraint check unrelated to the main Ent's unique key fails), there are no "X" and "Nullable" variations of this method. + +## Batching + +Multiple `upsert*()` calls running in parallel are batched by Ent Framework: + +```typescript +await Promise.all([ + EntTopic.upsert(vc, { + slug: "s1", + creator_id: "123", + subject: "test1", + }), + EntTopic.upsert(vc, { + slug: "s2", + creator_id: "456", + subject: "test2", + }), +]); +``` + +The batched query will look like this: + +```sql +WITH rows(...) AS (VALUES + ('s1', '123', 'test1'), + ('s2', '456', 'test2')), + updates AS ( + UPDATE topics SET ... + FROM rows WHERE topics.slug=rows.slug + RETURNING rows._key, topics.id AS id), + inserts AS ( + INSERT INTO topics (id, ...) + SELECT id_gen(), ... + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) + ON CONFLICT (slug) DO UPDATE SET ... + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +``` + +It is complicated! In fact, the query runs UPDATE-INSERT-UPDATE sequence, to ensure that it doesn't call `id_gen()` in case the row already exists in the database (to not exhaust the sequence). + + +# Privacy Rules + +A crucial reason on why Ent Framework exists at all is its privacy layer. No data exits the API unless it's rechecked against a set of explicitly defined security predicates. In other words, when you have multiple users in your service, you can enforce the strict guarantees that one user can't see other user's data even in theory. + +In relational databases world, the concept of per-row security rechecking is called "row-level security". + +## PostgreSQL Built-in Row Level Security? + +Before we continue, we must mention that some support for row-level security is [built in to PostgreSQL](https://www.postgresql.org/docs/current/ddl-rowsecurity.html), but it has several drawbacks that makes it almost useless in web development: + +1. It is expensive and, at the same time, too "sloppy" and low-level (the amount of DDL code you need to write is large, and there is no framework in place to help you with it). +2. There is no support for "per-transaction variables" in PostgreSQL (no per-session variables as well), so if you want to pass an "acting user ID" (similar to Ent Framework VC's Principal), other than the database DDL user/role, into the query, then you can't. +3. PostgreSQL doesnt't support microsharding, so you basically can't recheck security against the data living in a different microshard. + +## How Ent Framework Privacy Rules Work + +In each Ent class, you need to define an explicit set of rules and determine, can a VC read that Ent (`privacyLoad`), create a new Ent (`privacyInsert`), update the Ent (`privacyUpdate`) and delete the Ent (`privacyDelete`): + +```typescript +const schema = new PgSchema( + "comments", + { + id: { type: ID, autoInsert: "nextval('comments_id_seq')" }, + created_at: { type: Date, autoInsert: "now()" }, + topic_id: { type: ID }, + creator_id: { type: ID }, + message: { type: String }, + }, + [] +); + +export class EntComment extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: ["topic_id"], + privacyInferPrincipal: async (_vc, row) => row.creator_id, + privacyLoad: [ + new AllowIf(new OutgoingEdgePointsToVC("creator_id")), + new AllowIf(new CanReadOutgoingEdge("topic_id", EntTopic)), + ], + privacyInsert: [ + new Require(new OutgoingEdgePointsToVC("creator_id")), + new Require(new CanReadOutgoingEdge("topic_id", EntTopic)) + ], + // privacyUpdate and privacyDelete derive from privacyInsert + // if they are not explicitly specified. + }); + } +} +``` + +### privacyLoad Rules and Graph Reachability + +When you run e.g. `EntComment.loadX(vc, "123")` or any other API call, like `loadBy*()` or `select()`, Ent Framework runs `privacyLoad` rules for each Ent. + +Typically, a **Rule** class used in `privacyLoad` is `AllowIf`: it allows reading the Ent immediately as soon as the passed **Predicate** succeeds. There are several pre-defined Predicate classes, and you can also create your own predicates, or just pass an async boolean function; we'll discuss it a bit later. + +So, the logic in the example is following: + +1. `new OutgoingEdgePointsToVC("creator_id")`: if `comment.creator_id` equals to `vc.principal`, then the read is immediately allowed. It means that you (`vc`) are trying to read a commend which you created (its `creator_id` is your user ID). +2. `new CanReadOutgoingEdge("topic_id", EntTopic)`: if `vc.principal` is able to run `EntTopic.loadX(vc, comment.topic_id)` successfully, then reading of the comment is immediately allowed. This is an extremely powerful construction, the essence of Ent Framework's privacy layer: you can **delegate** privacy checks to other Ents in the graph. And since the engine does batching and caching aggressively, this all will be performance efficient. + +Idiomatically, `privacyLoad` defines access permissions in terms of **graph edges reachability**: typically, if there is **at least one** path in the graph originating from the VC and ending at the target Ent, then this VC is allowed to read the Ent. + +### privacyLoad is a Safety Net, not a Filter + +As opposed to [Meta's Ent Framework](/architecture/ent-framework-metas-tao-entgo), privacy rules *do not post-filter* the loaded Ents. They only recheck and throw. + +I.e. if you run a `select()` call, it will either return you all of the loaded Ents (if they all pass privacy checks) or throw a detailed error (if some of them don't). This applies to all other API calls as well. + +There are several reasons for such behavior: + +1. **Performance.** Ent Framework sits close to the underlying relational database. If you run a call that returns multiple Ents (e.g. `select()`), you most likely want to make sure that the query matches an existing database index. So you *have to* encode the filtering logic in your `where` condition directly, not just "bulk-load everything and then post-filter". It also applies to other aspects of fetching like pagination, `limit` clause etc. +2. **Debugging simplicity.** In Meta (where privacy rules actually *did* filter), it was a severe pain to figure out, why some query returns you an empty (or incomplete) response. This is because privacy rules were implicitly filtering "invisible" Ents, and once the Ents are hidden, you don't even know whether they are filtered out or do not exist. + +Let's consider a common example: an Ent class with `is_archived` boolean field. You obviously want the archived Ents to fail the privacy checks of a regular VC. In Ent Framework, it is not enough: you also have to modify your `select()` calls to explicitly mention `is_archived: false`, otherwise your queries will start throwing `EntNotReadableError` when trying to load an archived Ent. Add a static helper method to your Ent class if you don't want to repear `is_archived` over and over again. (BTW, to still enable archived Ents reading, you may create a `VCReadArchive` [flavor](/advanced/vc-flavors).) + +### privacyLoad and load\*() Calls + +The way privacy rules interact with [load\*()](/getting-started/ent-api-load-by-id) and [loadBy\*()](/getting-started/ent-api-loadby-unique-key) API calls is following: + +* `loadX()` and `loadByX()`: they obviously throw `EntNotReadable` error (derived from `EntAccessError` base class) if the privacy checks fails. +* `loadNullable()` and `loadByNullable()`: they will also throw `EntNotReadableError`, not just return null! This greatly helps with debugging of privacy rules violations. +* `loadIfReadableNullable()`: this is what you want to use in an unlikely case when you *really* need to treat an unavailable Ent as absent. The method name is long intentionally: the best practice is to not use it too often. + +### privacyInsert and Referential Permissions + +As opposed to `privacyLoad`, where a single succeeded rule allows the read, for `privacyInsert` (as well as `privacyUpdate` and `privacyDelete`), **all of them** must pass typically. + +This is because the ability to insert an Ent means that the VC has permissions to reference other Ents in **all** field edges. In reality, for every field edge (foreign key) defined in the Ent, there should be at least one associated **Require** privacy rule. + +Having permissions to insert an Ent is almost always the same as having permissions to reference other Ents in its foreign key fields. If we forget to check some of the field edges, then it is possible that the user will be able to create an Ent "belonging" to someone else (by e.g. referencing someone else's ID). + +The logic in the example above: + +1. `new Require(new OutgoingEdgePointsToVC("creator_id"))`: it is **required** that the value of `comment.creator_id` is equal to `vc.principal`. I.e. you can only reference yourself as a creator of the just inserted comment. +2. `new Require(new CanReadOutgoingEdge("topic_id", EntTopic))`: it is **required** that, to create a comment on some topic, you must have at least read access to that topic. I.e. you can create comments on someone else's topics too, as soon as you can read those topics. + +Notice that here we again use delegation: instead of introducing complicated boilerplate in comments privacy rules, we say: "I fully trust the way how privacy is implemented at EntTopic, and I don't want to know details about it at EntComment level". Basically, you build a **chain of trust**. + +### privacyUpdate and privacyDelete + +`privacyUpdate/Delete` rules are similar to `privacyInsert`, but they are checked by `update*()` and `delete*()` calls correspondingly. + +If there is no `privacyUpdate` block defined, then the rules are inherited from `privacyInsert` array. + +If there is no `privacyDelete` block mentioned in the configuration, then Ent Framework uses `privacyUpdate` rules for it. (And if there are no `privacyUpdate` rules, then `privacyInsert`). + +## Rule Classes + +Item in `privacyLoad/Insert/Update/Delete` arrays are called a **Rules**. There are several built-in rules: + +* `new AllowIf(predicate)`: if `predicate` resolves to true and doesn't throw, allows the access immediately, without checking the next rules. Commonly, `AllowIf` is used in `privacyLoad` rules. It checks that there is **at least one** path in the graph originating at the user denoted by the VC and ending at the target Ent. Also, you may use `AllowIf` in the prefix of `privacyInsert/Update/Delete` rules to e.g. allow an admin VC access the Ent early, without checking all other rules. +* `new Require(predicate)`: if `predicate`resolves to true and doesn't throw, tells Ent Framework to go to the next rule in the array to continue. If that was the last `Require` rule in the array, allows access. This rule is commonly used in `privacyInsert/Update/Delete` blocks, where the goal is to insure that **all** rules succeed. +* `new DenyIf(predicate)`: if `predicate` returns true **or throws an error**, then the access is immediately rejected. This rule is rarely useful, but you can try to utilize it for ealy denial of access in any of the privacy arrays. + +## Predicates + +**Predicate** is like a function which accepts an acting VC and a database row. It returns true/false or throws an error. + +### Custom Functional Predicates + +The simplest way to define a predicate is exactly that, pass it as an async function: + +```typescript +privacyLoad: [ + new AllowIf(new OutgoingEdgePointsToVC("id")), + new AllowIf(async function CommentIsInPublicTopic(vc, row) { + const topic = await EntTopic.loadX(vc, row.topic_id); + return topic.published_at !== null; + }), +] +``` + +Notice that we gave this function an inline name, `CommentIsInPublicTopic`. If the predicate returns false or throws an error, that name will be used as a part of the error message. Of course we could just use an anonymous lambda (like `async (vc) => {}`), but if we did so and the predicate returned false, then the error won't be much descriptive. + +Here, `row` is strongly-typed: you can use Ent data fields. It is not an Ent instance though, which is currently a TypeScript limitation: you can't self-reference a class in its mixin. + +### Custom Class Predicates + +You can also define preticates as classes, to make them more friendly for debugging. In fact, Ent Framework's built-in predicates are implemented as classes. + +As an example, let's see how a built-in predicate `CanReadOutgoingEdge` works: + +```typescript +export class CanReadOutgoingEdge + implements Predicate> +{ + readonly name; + + constructor( + public readonly field: TField, + public readonly toEntClass: EntClass, + ) { + this.name = `${this.constructor.name}(${this.field})`; + } + + async check(vc: VC, row: Record): Promise { + const toID = row[this.field]; + if (!toID) { + return false; + } + const cache = vc.cache(IDsCacheReadable); + if (cache.has(toID)) { + return true; + } + await this.toEntClass.loadX(vc, toID); + // sill here and not thrown? save to the cache + cache.add(toID); + return true; + } +} +``` + +Each predicate class must be defined with `implements Predicate` which requires the method `check(vc, row)` to be implemented, as well as the `name` property to exist. + +In the class constructor, you accept any predicate configuration parameters and build a more descriptive `name` for the predicate instance than just the predicate name. + +And in `check()` method, you implement your predicate's logic, the same way as you would do it in a functional predicate. + +### Built-in Predicates + +For convenience, Ent Framework already includes some of the most useful predicates. This set is constantly growing, so check [src/ent/predicates](https://github.com/clickup/ent-framework/tree/main/src/ent/predicates) for the most up-to-date list. + +#### **new** [**True**](https://github.com/clickup/ent-framework/blob/main/src/ent/predicates/True.ts)**()** + +This is the simplest possible predicate, since it always returns true. It is useful when you want to create an Ent class which can be read by anyone. + +#### **new** [**OutgoingEdgePointsToVC**](https://github.com/clickup/ent-framework/blob/main/src/ent/predicates/OutgoingEdgePointsToVC.ts)**(field)** + +Checks that `ent[field]` is equal to `vc.principal`. This is useful for fields like `created_by` or `user_id` or some similar cases, when you want to make sure that the VC's acting user is mentioned in the Ent field to make this field readable (or writable). + +#### **new** [**CanReadOutgoingEdge**](https://github.com/clickup/ent-framework/blob/main/src/ent/predicates/CanReadOutgoingEdge.ts)**(field, ToEntClass)** + +Delegates the privacy check to another Ent Class (`ToEntClass`) considering that `toEnt.id` is equal to `ent[field]` . Sounds complicated, but in proactice it means the the VC has permissions to read another Ent that is parent to the current Ent, and is pointed by `field` . A good example is a predicate on EntComment: `privacyLoad: [new CanReadOutgoindEdge("topic_id", EntTopic)]` means that, to read this comment, the VC must be able to read its parent topic. + +#### **new** [**CanUpdateOutgoingEdge**](https://github.com/clickup/ent-framework/blob/main/src/ent/predicates/CanUpdateOutgoingEdge.ts)**(field, ToEntClass)** + +Similar to `CanReadOutgoingEdge` above, but delegates the check to the parent Ent's `privacyUpdate` rules. + +#### **new** [**CanDeleteOutgoingEdge**](https://github.com/clickup/ent-framework/blob/main/src/ent/predicates/CanDeleteOutgoingEdge.ts)**(field, ToEntClass)** + +Same as `CanUpdateOutgoingEdge`, but for `privacyDelete` delegation to the parent Ent. + +#### **new** [**IncomingEdgeFromVCExists**](https://github.com/clickup/ent-framework/blob/main/src/ent/predicates/IncomingEdgeFromVCExists.ts)**(EntEdge, entEdgeVCField, entEdgeFKField, entEdgeFilter?)** + +Checks that there is a **child** Ent in the graph (`EntEdge`) that points to both `vc.principal` and to our current Ent. In other words, checks that there is a direct junction Ent sitting in between the VC and our current Ent. Optionally, you can provide an `entEdgeFilter` callback which is fed with that junction Ent (of `EntEdge` class) and should return true or false for filtering purposes. + +Imagine you have `EntUser` and `EntOrganization` Ents, and also `EntEmployment` junction Ent with `(organization_id, user_id)` field edges (foreign keys). You want to check that some `EntOrganization` is readable by a VC: + +```typescript +const employmentsSchema = new PgSchema( + "employments", + { + id: { type: ID, autoInsert: "nextval('employments_id_seq')" }, + organization_id: { type: ID }, + user_id: { type: ID }, + }, + ["organization_id", "user_id"], +); + +export class EntEmployment extends BaseEnt(cluster, employmentsSchema) { + ... +} + +... + +export class EntOrgainzation extends BaseEnt(cluster, organizationsSchema) { + static override configure() { + return new this.Configuration({ + privacyLoad: [ + new AllowIf( + new IncomingEdgeFromVCExists( + EntEmployment, // junction Ent + "user_id", // points to vc.principal + "organization_id", // ponts to this.id + ), + ), + ], + ... + }); + } +} +``` + +You use `IncomingEdgeFromVCExists` just once in `EntOrganization`, and then for all other children Ents, you delegate permission checks to their parent organization, using `OutgoingEdgePointsToVC` typically. + +#### new [Or](https://github.com/clickup/ent-framework/blob/main/src/ent/predicates/Or.ts)(predicate1, predicate2, ...) + +This is a composite predicate, allowing to call other predicates in pallel. It returns true if any of the predicates returned true and no predicates threw an error. + +Notice that you likely don't need this predicate when working with `privacyLoad`, since it's typically a chain of `AllowIf` rules. The `AllowIf` rule already works in an "or-fashion". But for `privacyUpdate/Delete` rules, the `Or` predicate may be useful (`Require` rule is "and-ish" on its nature). + +#### new [VCHasFlavor](https://github.com/clickup/ent-framework/blob/main/src/ent/predicates/VCHasFlavor.ts)(FlaviorClass) + +This predicate returns true if there is flavor of a particular class added to the acting VC. + +[Flavors](/advanced/vc-flavors) will be discussed later in details. For now, we can just mentioned that it's some kind of a "flag" which can be added to a VC instance for later rechecking or to carry some auxiliary information (more precisely, you can derive a new VC with a flavor added to it, since VC itself is an immutable object). + +A very common case is to define your own `VCAdmin` flavor which is added to a VC very early in the request cycle with `vc = vc.withFlavor(new VCAdmin())`, when the corresponding user is an admin and can see any data in the database. Then, in `privacyLoad/Insert/Update/Delete` of the Ent classes, you can add `new AllowIf(new VCHasFlavor(VCAdmin))` to allow an admin to read that Ent unconditionally. + +## Running Privacy Rules Manually + +Every Ent class exposes a special "constant" `VALIDATION` static property that allows you to run privacy rules and fields validators manually if needed. Read more about this in [Ent API: Configuration and Types](/ent-api-configuration-and-types). + + +# Validators + +Validators are predicates, similar to what you use in `privacyInsert/Update` [Privacy Rules](/getting-started/privacy-rules). They are called at the same time, and the error messages (if any) are accumulated to build and throw a compound `EntValidationError` instance. + +## Field Validators + +Field validators are executed on every `insert*()` and `upsert*()` call. + +Also, they are fired when an `update*()` call touches the fields that the validators are attached to. The untouched fields do not trigger re-validation. + +```typescript +export class EntComment extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + privacyLoad: [...], + privacyInsert: [...], + validators: [ + new FieldIs( + "message", + (value, _row, _vc) => value.trim().length > 0, + "Please provide comment text", + ), + new FieldIs( + "topic_id", + async (value, _row, vc) => { + const topic = await EntTopic.loadX(vc, value); + return Date.now() - topic.created_at.getTime() < 1000 * 3600 * 24; + }, + "You can only leave comments on topics created today", + ), + ... + ] + }); + } +} +``` + +If you want to build your own custom validation predicate similar to `FieldIs`, make sure that it implements `AbstractIs` interface. Otherwise, you won't be able to use it in `validators` block. + +Validators have so much in common with privacy rules that internally, the whole Ent Framework's privacy engine is called `Validation`. + +The use case for validators is enforcing some early integrity checks on Ent fields before saving the Ent to the database. Putting this logic as close to the database layer as possible brings expra firmness to the architecture. + +```typescript +try { + const comment = EntComment.insertReturning(vc, { + topic_id: topic.id, + creator_id: vc.principal, + message: request.body.message, + }); + ... +} catch (e: unknown) { + if (e instanceof EntValidationError) { + return res.json({ + errors: e.errors.map((e) => ({ + field: e.field, // null if relates to the whole row + message: e.message, + })), + }); + } else { + throw e; + } +} +``` + +## Whole-Row Validators + +You can also define `RowIs` validators that operate with the entire row to be inserted or updated. As opposed to `FiledIs`, such validators are fired independently on which fields you are modifying. + +```typescript +export class EntComment extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + privacyLoad: [...], + privacyInsert: [...], + validators: [ + new RowIs( + async (row, vc) => checkForSpam(vc, row), + "Comment spam checking failed", + ), + ... + ] + }); + } +} +``` + +## Using with Zod or Standard Schema + +You can also use [Zod](https://zod.dev) or any validation library compatibe with [Standard Schema](https://standardschema.dev): + +```typescript +import { z } from "zod"; + +validators: [ + // Use Zod's default generated message + new FieldIs( + "message", + async (value) => z.string().min(10).safeParseAsync(value), + ), + // Custom error message. + new FieldIs( + "message", + (value) => z.string() + .min(10, "Text must be longer than 10 characters") + .safeParse(value), + ), + // Validation of the entire row. + new RowIs( + (row) => z.object({ + title: z.string().min(1), + message: z.string().min(10), + }).safeParse(row), + ), + ... +] +``` + +Basically, when you omit the last `message` parameter of `FieldIs` or `RowIs` constructors, then it's expected that your validator callback returns an object compatible with Zod's [safeParse()](https://zod.dev/?id=safeparse) or Standard Schema's [validate()](https://standardschema.dev) result shape. + +## Running Validators Manually + +Every Ent class exposes a special "constant" `VALIDATION` static property that allows you to run fields validators manually if needed. Read more about this in [Ent API: Configuration and Types](/ent-api-configuration-and-types). + + +# Triggers + +Triggers are hooks that Ent Framework executes right before or after a mutation (insert, update or delete). They can modify the table row before it's got saved, and also load or update other Ents. + +The word "hook" also draws the analogy with React Hooks (from frontend world), since update-triggers in Ent Framework have several traits in common with React's `useEffect()` hook. + +Triggers are defined in the Ent Class configuration, near [Privacy Rules](/getting-started/privacy-rules). + +## Before-Triggers + +In before-triggers, you can: + +1. Make changes in the fields right before they are saved to the database. +2. Load or even mutate other Ents. + +```typescript +const schema = new PgSchema( + "topics", + { + id: { type: ID, autoInsert: "nextval('topics_id_seq')" }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + slug: { type: String, autoInsert: "NULL" }, + creator_id: { type: ID }, + subject: { type: String, allowNull: true }, + }, + ["slug"] +); + +export class EntTopic extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + ... + beforeInsert: [...], + beforeUpdate: [...], + beforeDelete: [...], + beforeMutation: [...], + }); + } +} +``` + +### beforeInsert Triggers + +Let's start with an example: + +```typescript +... +beforeInsert: [ + async (vc, { input }) => { + let slug = slugufy(input.subject); + if (await EntTopic.exists(vc, { slug })) { + slug += `-${Date.now()}`; + } + input.slug = slug; + }, +], +... +const topic = await EntTopic.insertReturning(vc, { + creator_id: "123", + subject: "My Topic", +}); +``` + +Here, we automatically assign the value to `slug` field of the inserted row based on the topic's subject. + +Notice that it requires a little quirk though: `slug` field in the schema needs to be defined with `autoInsert` attribute, otherwise Ent Framework TypeScript typing will make `slug` as a required property in \``insertReturning()` call. + +As everything in Ent Framework, all arguments of the trigger functions are strongly typed. + +* It will respect the field types defined in the schema exactly. +* Nullability is respected (fields defined with `allowNull: true` will be nullable in the `input` argument). +* It will pay attention to required and optional fields (the optional fields are the ones defined with `autoInsert` or `autoUpdate`). + +### Accessing Ent ID in beforeInsert Trigger + +Despite the insert operation has not yet been applied to the database, in all `beforeInsert` triggers, you can already read the ID of the Ent to be inserted. + +This is very convenient to organize eventually consist logic in your code: in Ent Framework, there are no transactions exposed (and there can be no transactions even in theory when working across microshards or across different storage services), so you must pay attention to the order of the writes, to make sure your don't lose eventual consistency behavior: + +```typescript +... +beforeInsert: [ + async (vc, { input }) => { + await addToKafka(this.name, input.id); + }, +], +... +``` + +Here we assume that you have a `addToKafka()` function which accepts the Ent class name and the Ent ID. After the write to Kafka succeeds, you proceed with saving the Ent to the database. Using this apprpach, you can e.g. implement eventually-consistent pipelining of the Ent data to some other storage using an external bus (like Kafka or Redis Streams), despite this system "bus+PostgreSQL" being not transactionally safe as a whole. + +### beforeUpdate Triggers + +Update is a more complicated operation, since you have the old row and the new row versions at the same time. + +```typescript +... +beforeUpdate: [ + async (vc, { oldRow, input, newRow }) => { + await addToKafka(this.name, newRow.id); + if (newRow.subject !== oldRow.subject) { + // Notice that newRow.subject is a non-optional + // string property, whilst input.slug is optional + // (i.e. string | undefined). + let slug = slugufy(newRow.subject); + if (await EntTopic.exists(vc, { slug })) { + slug += `-${Date.now()}`; + } + input.slug = slug; + } + }, +], +... +await topic.updateReturningX({ subject: "Hello" }); +``` + +Notice that the code here is very similar to the `beforeInsert` trigger we discussed above. To avoid boilerplate in such cases, you can use `beforeMutation` instead; we'll describe it a little later. + +In the trigger functions, Ent Framework gives you the following arguments: + +* `oldRow`: the row with Ent fields right before the update. This object is immutable. +* `input`: properties passed to `update*()` method as they are. Notice that it includes **not** all Ent fields, but only the fields you are mutating (in other words, all properties of `input` object are *optional* in their TypeScript typing). You need to modify this object if you want the trigger to make changes in the Ent before the update happens. +* `newRow`: the result of applying `input` over `oldRow` . This is an immutable object. + +### Immutable Fields + +Using `beforeUpdated`, you can force some Ent field to be *immutable*, so any `update*()` call will not change it: + +```typescript +... +beforeUpdate: [ + function SlugIsImmutable(vc, { oldRow, input }) => { + input.slug = oldRow.slug; + }, +], +... +// This value won't be saved. +await topic.updateReturningX({ slug: "new-value" }); +``` + +### beforeDelete Triggers + +This kind of triggers is the simplest: + +```typescript +... +beforeDelete: [ + async (vc, { oldRow }) => { + await addToKafka(this.name, oldRow.id); + await mapJoin( + await EntComment.select(vc, { topic_id: oldRow.id }, 1000000), + async (comment) => comment.deleteOriginal(), + ); + }, +], +... +await topic.deleteOriginal(); +``` + +In this example, we do two things: + +1. We call `addToKafka()` function to e.g. publish the deletion event to our event bus, so we can replay that deletion to some other data store in an eventually consistent manner. If publishing to Kafka fails, them the trigger will throw an error, and no deletion will happen in the database. If deletion succeeds, then we can be sure that it also got replayed to Kafka (since it's done prior to the deletion). And if deletion fails... then the user will see it and retry later. +2. We delete all children comments when the topic is deleted. This is a kind of `ON DELETE CASCADE` clause in relational database's foreign keys, but with an important difference: it calls Ent Framework triggers on the comments as well. + +### beforeMutation Triggers + +Notice that we have some boilerplate in our triggers: + +* We call `addToKafka()` in 3 places: `beforeInsert/Update/Delete` triggers. +* We have the exact same logic to calculate `slug` field in 2 places: `beforeInsert/Update` . + +To eliminate that, there is a special feature: `beforeMutation` triggers, which are called before *any* mutation (be it insert, update or delete), in a TypeScript-safe way for the arguments. + +```typescript +... +beforeMutation: [ + async (vc, { newOrOldRow }) => { + await addToKafka(this.name, newOrOldRow.id); + }, + async (vc, { op, newOrOldRow, input }) => { + if ( + op === "INSERT" || + (op === "UPDATE" && "subject" in input && newOrOldRow.subject !== input.subject) + ) { + let slug = slugufy(newOrOldRow.subject); + if (await EntTopic.exists(vc, { slug })) { + slug += `-${Date.now()}`; + } + input.slug = slug; + } + }, +], +beforeDelete: [ + async (vc, { oldRow }) => mapJoin( + await EntComment.select(vc, { topic_id: oldRow.id }, 1000000), + async (comment) => comment.deleteOriginal(), + ), +], +... +const topic = await EntTopic.insertReturning(vc, { + creator_id: "123", + subject: "My Topic", +}); +await topic.updateReturningX({ subject: "Hello" }); +await topic.deleteOriginal(); +``` + +There are 2 gotchas here: + +1. We split one big trigger into two independent ones. The triggers are run sequentially, and the next trigger in the list is not called if the previous one throws an error. +2. TypeScript is smart enough to understand that, when you check `op` against `"INSERT"` or `"UPDATE"` strings, the typing of `newOrOldRow` and `input` arguments will be according to the operation types (i.e. it will respect optional properties for instance). + +### Changed Fields Tracking and React's useEffect() Analogy + +But you are probably still not satisfied with that long `if` clause in the example above. We can improve the code: + +```typescript +... +beforeMutation: [ + async (vc, { newOrOldRow }) => { + await addToKafka(this.name, newOrOldRow.id); + }, + [ + (vc, row) => [row.subject], // "deps builder" + async (vc, { op, newOrOldRow, input }) => { + if (op !== "DELETE") { + let slug = slugufy(newOrOldRow.subject); + if (await EntTopic.exists(vc, { slug })) { + slug += `-${Date.now()}`; + } + input.slug = slug; + } + }, + ], +], +... +``` + +Here we pass a tuple with 2 lambdas: + +1. The 1st lambda, `(vc, row) => [row.subject]`, is called "deps builder". It extracts some part of the row, and Ent Framework will call the trigger code **only if** that part has actually changed on an update (and also on insert/delete, since those are also considered as "changes") +2. The 2nd lambda is your trigger code. Ent Framework will run it only if the 1st callback returned a value different between the old and the new rows (or it's an insert or delete operation). If you are familiar with React, you can notice that this mechanism is similar to how its `useEffect()` hook works. + +In the trigger code, you still need to check that the operation is not `DELETE`, but it is way better still than having a boilerplate in the previous example. + +The "deps builder" lambda can be async, so you can run other database queries in it and make decisions based on their results. + +Notice that "deps builder" tuple also works for `beforeUpdate`, as well as for `afterUpdate` and `afterMutation` triggers we'll discuss below. + +## After-Triggers + +After-triggers are called seqentially, as soon as an insert/update/delete mutation succeeds in the database. + +### afterInsert Triggers + +Triggers of this kind act exactly as `beforeInsert`, but they are called after a successful database operation, not before. There, you can do some auxiliary work, but keep in mind that, if this work fails, the Ent will remain created in the database still. There are no (and cannot be) built-in transactions across multiple independent IO services and multiple different microshards. + +```typescript +... +afterInsert: [ + async (vc, { input }) => { + ... + }, +], +... +``` + +### afterUpdate Triggers + +The only difference with `beforeUpdate` triggers here is that there is no `input` argument passed: the only things you have are `oldRow` and `newRow` : + +```typescript +... +afterUpdate: [ + async (vc, { oldRow, newRow }) => { + ... + }, +], +... +``` + +You can also use "deps builder" syntax in `afterUpdate`, to run the trigger code only if some particular fields change: + +```typescript +... +afterUpdate: [ + ... + [ + (vc, row) => [row.subject], // "deps builder" + async (vc, { oldRow, newRow }) => { + ... + }, + ], +], +... +``` + +### afterDelete Triggers + +In `afterDelete`, you can run some optional cleanup of other resources associated to the just-deleted Ent. Keep in mind though that it's all non-transactional: if your cleanup fails, it won't be retried, and the row will already be deleted in the database. + +```typescript +... +afterDelete: [ + async (vc, { oldRow }) => { + ... + }, +], +... +``` + +### afterMutation Triggers + +Similarly to `beforeMutation` triggers, `afterMutation` triggers allow you to react on any of insert/update/delete operations. but only after this operation succeeds in the database. + +There is also no `input` argument available in this kind of triggers, only `newOrOldRow`. + +```typescript +... +afterMutation: [ + async (vc, { op, newOrOldRow }) => { + ... + }, +], +... +``` + +You can use "deps builder" syntax too if you want to react only when some particular fields change on an update (or on insert and delete unconditionally): + +```typescript +... +afterMutation: [ + [ + (vc, row) => [row.subject], // "deps builder" + async (vc, { op, newOrOldRow }) => { + ... + }, + ], +], +... +``` + + +# Custom Field Types + +In addition to [Built-in Field Types](/getting-started/built-in-field-types), you can also defined custom strongly-typed fields. + +The values stored in custom fields will be serialized before storing to the database, and on read, deserealized back. Typically, the serialization format is JSON (so you can use PostgreSQL column types like `jsonc` or `json`), but you can also use other formats (like array of `bigint`, array of `varchar` or anything else). + +## JSON-Serialized Fields + +Let's first consider the simplest and the most common case of custom field types, where a fied is stored as a `jsonc` value in a PostgreSQL Ent table. + +Imagine we want to add a new custom field `actors` to `topics` table, internally stored as a JSON: + +```sql +CREATE TABLE topics( + id bigserial PRIMARY KEY, + ... + actors: jsonc NOT NULL +); +``` + +You define a custom type by providing an object with 3 callbacks: + +```typescript +type Actors = { + editor_ids: string[]; + // will add more fields later +}; + +const ActorsType = { + dbValueToJs(v: unknown): Actors { + // node-postgres already parses jsonc internally, + // so we don't need anything more here + return v; + }, + + stringify(obj: Actors): string { + return JSON.stringify(v); + }, + + parse(v: string): Actors { + return JSON.parse(v); + }, +} +``` + +* `dbValueToJS(v)`: given a value from node-postgres row, converts it to a strongly typed TypeScript value. (Notice that node-postgres already does some conversions internally: e.g. an array field, `v` returned by the engine is already an array of things, so `dbValueToJs` for it will just do nothing.) The return type of this callback will automatically become the custom field's TypeScript type. Ent Framework will execute this callback every time you load an Ent from the database. +* `stringify(obj)`: given a value of your custom type, converts it into a string representation compatible with PostgreSQL value. Ent Framework will run this callback every time you use the custom field in any query (e.g. insert/update/delete or even when selecting Ents). +* `parse(str)`: this callback is the opposite of `stringify()`. Ent Framework doesn't call it (since it uses `dbValueToJs` instead), but for convenience and completeness of the interface, it's still here. + +Once the above 3 callbacks are defined, you can declare a field of custom type in your schema: + +```typescript +const schema = new PgSchema( + "topics", + { + ... + actors: { type: ActorsType }, + }, + ["slug"] +); +... +const topic = await EntTopic.insertReturning(vc, { + ..., + actors: { editor_ids: ["42"] }, +}); +... +console.log(topic.actors.editor_ids); +... +await topic.updateChanged({ + actors: { editor_ids: ["101"] }, +}); +``` + +## Adding an Optional Property to Custom Type + +When you have a custom type, you'll most likely want to modify it in the future. + +The simplest possible modification is adding an optional property: + +```typescript +type Actors = { + editor_ids: string[]; + viewer_ids?: string[]; // <-- added; optional +}; +``` + +You don't need to change anything else: + +* Your existing rows in the database (without `viewer_ids`) will be readable by the new code, since the property is optional. +* When your code assigns a value to `viewer_ids`, it will also be written to the database, and it won't conflict with the old code that can still be running somewhere in the cluster. + +## Adding a Required Property to Custom Type + +Optional propertied are good (and in fact they are the only "officially recommended" way of adding properties in serialization protocols like [protobuf](https://protobuf.dev)), but optionality adds a technical debt spaghetti everywhere in your code where you work with your new properties. A better variant would be to make the property **required**. + +```typescript +type Actors = { + editor_ids: string[]; + viewer_ids: string[]; // <-- added; required +}; + +const ActorsType = { + dbValueToJs(v: /* a little lie */ Actors): Actors { + v.viewer_ids ??= []; // <-- added + return v; + }, + + stringify(obj: Actors): string { + return JSON.stringify(v); + }, + + parse(v: string): Actors { + return this.dbValueToJs(JSON.parse(v)); + }, +} +``` + +The only change you need to make in `ActorsType` is to default-assign `[]` to `viewer_ids` property. Notice that we lie to TypeScript here a little: `v` argument of `dbValueToJs(v)` is in fact of type `Actors & { viewer_ids?: string[] }`, not of type `Actors`. But for simplicity, it's acceptable. + +## Changing the Shape Significantly + +See [Custom Field Refactoring](/advanced/custom-field-refactoring) in Advanced section. + + +# Ent API: Configuration and Types + +Every Ent class exposes several static "constant" properties that you can use to get access to various Ent configuration features. + +## Ent Class Static Properties + +Consider having the following Ent class defined: + +```typescript +const schema = new PgSchema( + "users", + { + id: { type: ID, autoInsert: "nextval('users_id_seq')" }, + email: { type: String }, + }, + ["email"], +); + +export class EntUser extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.id, + privacyLoad: [new AllowIf(new OutgoingEdgePointsToVC("id"))], + privacyInsert: [], + }); + } +} +``` + +### EntClass.SCHEMA + +In the example above, `EntUser.SCHEMA` it is equal to `BaseEnt`'s `schema` parameter. Each Schema has the following properties: + +* `name`: name of the underlying Ent table ("users" in the above example) +* `table`: an stronly typed object that defines the table's shape. In our example it is `{ id: ..., email: ... }` , exactly as defined in `new PgSchema(...)` code above. +* `uniqueKey`: a strongly typed array of fields composing the Ent Schema's unique key. Again, exactly as defined in `PgSchema` above. + +Examples: + +```typescript +// ["id", "email"] +const fields = Object.keys(EntUser.SCHEMA.table); + +// Using the Row type and table name. +const master = await EntUser.CLUSTER.globalShard().client(MASTER); +const rows = await master.query>({ + query: [`SELECT * FROM ${EntUser.SCHEMA.name} WHERE id=?`, userID], + isWrite: false, + annotations: [vc.toAnnotation()], + op: "MY_SELECT", + table: "users", +}); + +// Build a custom WHERE condition. +const where: Where = { + email: { $not: "test@example.com" }, +}; +await EntUser.select(vc, where, 100); +``` + +Notice how we used `typeof EntUser.SCHEMA.table` in the example above: it's a common pattern in Ent Framework. Most of the types it exposes (like `Row`, `Where` etc.) accept a generic `TTable` argument that can be obtained with this construction. + +### Helper (Input) Types + +Ent Framework API methods like `insert*()`, `update*()`, `load*()`, `select*()` etc. accept strongly-typed input and return strongly typed Ents. Here are some examples: + +* `InsertInput`: the shape of the argument that `insert*()` and `upsert*()` methods accept. This type plays nice with e.g. optional fields (the fields that have `autoInsert` in their definition), nulls etc. +* `UpdateInput`: methods like `update*()` accept this shape. Since you can choose, which fields to update, all of the properties of that type are optional. +* `Row` : that's a general shape of Ents returned from `load*()` and `select*()` calls. Notice that the type is very different from `InsertInput`, because it never has any optional fields. Optionality is the concept related to *mutations*; once you load something *existing* from the database, all of the fields are present, so they will all be "required". Don't mix up `Row` and `InsertInput` types in your code! +* `Where`: a query that `select()` call accepts. It supports rich query language features like `$not`, `$and`, `$lt` etc. See more details in [Ent API: select() by Expression](/getting-started/ent-api-select-by-expression). + +There are some other, less frequently, used types as well. See the docblocks in Ent Framework source code for more details and examples. + +### EntClass.VALIDATION + +This static Ent property allows you to manually run privacy and validation rules on an Ent without triggering an insert/update/delete. It is convenient if you want do a "dry-run" before applying an actual operation, to e.g. enable or disable some form controls or buttons in the user interface. + +```typescript +try { + await EntUser.VALIDATION.validateUpdate(vc, user, { + email: "new@example.com" + }); +} catch (e: unknown) { + if (e instanceof EntAccessError) { + // It's a top-level base class for all access related errors. + if (e instanceof EntNotUpdatableError) { + // Privacy rules failure with details. + console.log(e.message); + } else if (e instanceof EntValidatonError) { + // Fields validation error. + console.log(e.errors); + console.log(e.toStandardSchemaV1()); // https://standardschema.dev + } else { + ... + } + } else { + throw e; + } +} +``` + +The methods available on `VALIDATION` property are: + +* `validateInsert(vc, input: InsertInput)`: checks what would happen if you try to insert a new Ent with such properties. +* `validateUpdate(vc, old: Row, input: UpdateInput, privacyOnly: boolean)`: we already mentioned this method in the example above. You can also pass the last `privacyOnly` parameter as `true` if you do not want to run user-defined fields validators and only need to recheck the privacy rules. Otherwise, by default, it runs both privacy rules and fields validators, which is almost always what we want. +* `validateDelete(vc, row: Row)`: rarely used, checks what would happen if you try to delete that Ent. + +Notice that `Row` is not the same as an instance of your Ent (although you can pass an Ent to the functions that accept a `Row` type). Rows are a lower level concept: `Row` represents a plain object, it's basically a strongly-typed TypeScript `Record` of fields and their values (including nullability concept, custom field types etc.). Rows don't have `vc` property, nor do they have any Ent specific methods. + +And as mentioned above, `TTable` is derived from the Ent schema, e.g. `typeof EntUser.SCHEMA.table`. + +### EntClass.CLUSTER + +This static property simply equals to `cluster` parameter of `BaseEnt` you are extending when defining your Ent class. Use it in case you need to access some low-level Cluster API: + +```typescript +const master = await EntUser.CLUSTER.globalShard().client(MASTER); +``` + +### EntClass.SHARD\_AFFINITY and .SHARD\_LOCATOR + +The `SHAR_AFFINITY` static property simply returns the value of `shardAffinity` configuration option. + +The `SHARD_LOCATOR` property is pretty low-level: it exposes an Ent Framework object that allows to infer the affected microshards based on various criteria (like from an ID, or from a `Where` clause, or from a list of IDs etc.): `singleShardForInsert()`, `multiShardsFromInput()`, `singleShardFromID()` etc. Those methods are aware of the Ent's Inverses (see [Inverses and Cross Shard Foreign Keys](/scalability/inverses-cross-shard-foreign-keys)), but we won't discuss them here much. + +It also exposes a useful method `allShards()`: + +```typescript +const userShards = EntUser.SHARD_LOCATOR.allShards(); +for (const shard of userShards) { + // do something with users on this shard +} +``` + +Depending on the Ent's `shardAffinity`, this method will return either one shard (if it's `GLOBAL_SHARD`) or all shards of the cluster (in case it's `RANDOM_SHARD` or some other affinity), thus, allowing you to iterate over all Ents of this type in the cluster. Read more about sharding in [Shard Affinity and Ent Colocation](/scalability/shard-affinity-ent-colocation). + +### EntClass.TRIGGERS + +This static property exposes a `Triggers` objects that allows you to enumerate all of the Ent's trggers. It is almost never used externally, so we'll skip the details (see the source code if you want to learn more). + +## EntClass and Ent Interfaces + +Sometimes you want to write a generic function that accepts *any* Ent of a particular shape, or *any* Ent class. You can use EntClass and Ent interfaces (type shapes) for this. Here are some pretty artificial examples: + +```typescript +async function fancyDelete( + ent: Ent, +): Promise { + ... + await deleteExternalResource(ent.key); + await ent.deleteOriginal(); +} + +async function loadAny( + vc: VC, + EntCls: EntClass, + id: string, +): Promise> { + return EntCls.loadIfReadableNullable(vc, id); +} +``` + +Unfortunately, due to some TypeScript limitations (incomplete mixins support and a lack of class static properties typing), the functionality of EntClass and Ent interfaces is limited. But keep them in mind still, since they may be useful. + + +# Replication and Automatic Lag Tracking + +Replication (vertical scaling) and microsharding (horizontal scaling) are the two key features that define Ent Framework—without them, the library would lose its core purpose. + +In this article, we’ll focus on replication. + +“Replication” means you can write data to a single database machine and, after a short (but noticeable) delay, read the same data from one or more replica machines. PostgreSQL’s built-in replication ensures that all data written to the master database eventually appears on every replica. + +There are 2 main reasons why replicatiomn has to be used in pretty much every serious service: + +1. **Fault tolerance.** The service should survive (ideally with no downtime) in case one single database node goes down. So for every database, you must have at least 2 copies on 2 independent hardware nodes. +2. **Scaling reads.** In most of the projects, there are way more reads than writes happen. Since we need replicas for fault tolerance anyways, it makes sense to also use them for read queries. + +{% hint style="info" %} +By data propagation method, there are asynchronous and synchronous replication approaches. And by node roles, there are singe-master and multi-master configurations. They all have different trade-offs. In this article, by "replication" we mean the most popular setup: "asynchronous single-master replication". +{% endhint %} + +## Terminology + +Before we continue, let's agree on a common terminology. + +* **Master and Replica**: you commit data to the master node, and it eventually appears on all replica nodes. Transactions are remained atomic: multiple rows committed in one transaction to the master also appear "simultaneously" at replicas. +* **Replication lag**: time passed between the moment the data is committed to the master and the moment when this data can be read from a replica. Each replica has its own replication lag, since they all replay transactions from the master independently. +* **Read-After-Write Consistency**: if you write data in some "context" and then can read it back immediately *in the same context*, the read-write API is called "read-after-write consistent". Of course, "write to master, read from replica" workflow is not read-after-write consistent (but "write to master, read from master" is). Despite that, Ent Framework's API *is* read-after-write consistent (we'll discuss it in details below). +* **Eventual consistency**: you write data, and then *eventually*, after some delay (possibly large), you can read it back. "Write to master, read from replica" is an example of an eventually consistent workflow (which is not read-after-write consistent). +* **Write-Ahead Log (WAL)**: when you commit data to the master node, transactional databases (like PostgreSQL) first write it to a special "append-only" file called WAL. Once it's done, they save the rows to the database files. (In practice it's way more complicated, but for simplicity, we can stop on the simple definition.) WAL is also replayed on all replicas, so it's guaranteed that the replicas *eventually* follow the master bit by bit. +* **Log Sequence Number (LSN)**: on master, a position in WAL after some transaction commit; on replica, a position in WAL up to which the replica has already replayed the commits from master. To check that a replica is "good enough" for reading the data previously written to the master, you can compare the replica's current LSN with the master's LSN after the write: if it's greater or equal, then you'll read the data back. + +## Setting up Replication in PostgreSQL + +Ent Framework is just a client library, which means that you need to configure PostgreSQL replication before we continue. + +You have 2 options: + +1. Use low-level tools like [repmgr](https://www.repmgr.org) or [Patroni](https://github.com/patroni/patroni) to connect your master DB with your replica DBs. +2. Pay more money and use a PaaS solution like [AWS RDS for PostgreSQL](https://aws.amazon.com/rds/postgresql/) or [AWS RDS Aurora](https://aws.amazon.com/rds/aurora/). They have replication set up out of the box. + +## Replication Lag + +OK, you have a master database where you write the data to, and you have replica databases where you read from. It's just that simple, right? + +Not so fast. + +Consider the following code: + +```typescript +await query(MASTER, "INSERT INTO comments(text) VALUES('Hello')"); +return res.redirect("/comments"); +``` + +And on your `/comments` page: + +```typescript +const comments = await query(REPLICA, "SELECT * FROM comments"); +return res.render("comments.tpl", { comments }); +``` + +Unfortunately, you won't see the just-added comment on that rendered page, because there is a **replication lag issue**: the data written to a `MASTER` DB doesn't appear on the `REPLICA` DB immediately, there is 10-500 ms latency (and sometimes more, it depends on the database load, network stability etc.). + +This issue appears independently on the database engine you use, be it Aurora, RDS or vanilla PostgreSQL replication. The only difference between the engines is the average lag duration, but the lag always exists. + +To solve the replication lag issue, there are 2 options: + +1. Read from the master DB. The question is, how do we know, should we read from master or from replica at a particular moment. +2. Read from replica, but *if the data is not yet there*, wait a bit and retry. If there is no luck, fallback to master. The main question here is how do we understand that "the data is not there yet". + +Addressing replication lag problem improperly can quickly turn your codebase into a boilerplate mess. + +Luckily, Ent Framework takes care of this all automatically. In most of the cases, you don't need to think about the replication lag at all: the engine will choose, should it read from master or from replicas, transparently for your code. + +## Cluster Configuration + +First, you need to tell Ent Framework, where can it find the master database and all replicas: + +```typescript +export const cluster = new Cluster({ + islands: async () => [ // sync or async + { + no: 0, + nodes: [ + { name: "pg-001a", host: "pg-001a.your-domain.com", ... }, + { name: "pg-001b", host: "pg-001b.your-domain.com", ... }, + { name: "pg-001c", host: "pg-001c.your-domain.com", ... }, + ], + }, + ], + createClient: ({ name, ...config }) => new PgClient({ name, config }), + ..., +}); +``` + +Notice that we don't tell it, what endpoint is master and what endpoints are replicas: Ent Framework will detect it automatically. + +In fact, master and one of replicas may switch roles in real time (when you do some PostgreSQL maintenance, or when a master node fails, and you promote a replica to be the new master). Ent Framework handles such switches automatically and with no downtime. + +### AWS RDS Writer and Reader Endpoints + +If you use Amazon's RDS or Aurora, it provides you with 2 hostnames: + +* **Writer** (master) endpoint. When there is an outage on the master node, RDS automatically promotes one of the replicas to be a new master, and changes the writer endpoint routing to point to the new master. +* **Reader** (random replica) endpoint. If there are multiple replicas in the cluster, RDS routes the connections to a "random" replica (i.e. it's unpredictable, to which one). + +From the first glance, it looks like having just 2 endpoints is a pretty useful feature. There are several downsides though: + +* **Writer endpoint switch latency**: if there is a master outage, then, even after the new master is promoted in the cluster, the writer endpoint switches to it not immediately: there is some artificial latency, +* **Reader endpoint routing is unpredictable**: often times, one replica can already be "in sync" with the master (relative to the current user; we'll talk about it a bit later), whilst another replica is not yet. The engine like Ent Framework needs to know exactly, which replica does it connect to, to properly track its replication lag and metrics. + +So, although you can use writer and reader endpoints in your `Cluster` instance (especially when you don't need Ent Framework's built-in mechanism for replication lag tracking), it's discouraged. Instead, you'd better tell the engine the exact list of nodes in the cluster, and let it decide the rest. + +In Ent Framework, you can even modify the list of nodes in real time, without restarting the Node app. I.e. if you have a periodic timer loop that reads the up-to-date list of cluster nodes and returns it to Ent Framework, it will work straight away and with no downtime. Nodes may appear and disappear from the cluster, and the master may switch roles with replicas: Ent Framework will take care of it all and do the needed transparent retries. + +This is why in `Cluster` configuration, the list of islands (nodes) is returned by a callback. You can tell this callback to return a different list once the cluster layout changes: + +```typescript +export const cluster = new Cluster({ + islands: async () => [ // <-- sync or async callback + { + no: 0, + nodes: [ + { + name: "abc-instance-1", + host: "abc-instance-1.abcd.us-west-2.rds.amazonaws.com", + ..., + }, + { + name: "abc-instance-3", + host: "abc-instance-2.efgh.us-west-2.rds.amazonaws.com", + ..., + }, + ], + }, + ], + ..., +}); +``` + +## Automatic Replication Lag Tracking + +Once you set up the `Cluster` instance, Ent Framework is able to automatically discover, which exact node is master and what nodes are replicas. + +Imagine you run the following series of calls: + +```typescript +await EntComment.insert(vc, { ... }); +... // short delay (like 10 ms) +const comments = await EntComment.select(vc, {...}, 100); // <-- master or replica? +``` + +The 1st call will be executed against the master node, but will a replica be used for the 2nd call? No, it won't: the 2nd call will also run against the master node. 10 ms is a too short time interval for the replica to receive the update from master. If it was queried from a replica, we would not receive the just-inserted comment in the list of all comments returned by `select()` call. + +Ent Framework knows that in should use the master for reading, because for the VC used, it remembers the LSN (write-ahead log position) after each write. For replicas, it also knows their LSNs, so before sending a query to some replica, Ent Framework compares the master LSN at the time of the last write **in this VC** with the LSN at the replica. + +### Timelines, Einstein and Special Relativity + +So, Ent Framework provides a "read-after-write consistency" guarantee within the context of the same VC's principal. + +The context within which a read-after-write consistency is guaranteed is called a **Timeline**. Timeline is a special property of VC which remembers, what were LSNs on the master node after each write to each microshard+table. It's like a temporal state of the database related to the operations in a particular VC (basically, by a particular user). + +Here is a physics analogy to help you better understand, what a timeline is: **frame of reference in special relativity.** It is well known that the order of 2 events happened in one frame of reference [is not necessarily the same ](https://en.wikipedia.org/wiki/Ladder_paradox)as the order of the same exact events in another frame of reference. E.g. events "light bulb A blinked" and "bulb B blinked" separated by 1 mln miles may happen at the same time in one frame of reference, or "first A then B" in another frame or reference, or "first B then A" in a 3rd frame of reference. The order is strictly defined only in case when the light (the fastest speed of signal propagation possible) is able to travel between A and B (then, it will be "first A then B"). I.e. some information needs to be passed from A to B, and only then we can tell for sure that "B happened after A" and not vice versa. + +The same thing applies to timelines in Ent Framework: read-after-write consistency is only guaranteed within the same timeline. Also, one timeline can send a "signal" to another timeline propagating the knowledge about the change (which is called "causality"). After that signal is received, the read-after-write consistency will apply across those timelines. + +### Propagating Timelines via Session + +Consider the following pseudo-code: + +```typescript +app.post("/comments", async (req, res) => { + await EntComment.insert(req.vc, { ... }); + req.session.timelines = req.vc.serializeTimelines(); + return res.redirect("/comments"); +}); + +app.get("/comments", async (req, res) => { + req.vc.deserializeTimelines(req.session.timelines); + const comments = await EntComment.select(req.vc, {...}, 100); + return res.render("comments.tpl", { comments }); +}); +``` + +The browser sends a `POST /comments` request, so a new comment is inserted in the database, and the browser is immediately redirected to a `GET /comments` endpoint. Since we serialize all VC's timelines in the POST endpoint ("1st frame of reference") and then deserialize them in the GET endpoint ("2nd frame of reference"), the second VC receives a "↯-signal" from the first VC, and it establishes a strong read-after-write consistency between them. Thus, the 2nd request will be served by the master node and read the recent data. + +

The changes happened at "W" will be visible at "R"

+ +### Independent Timelines Use Case + +Notice that the above way of timelines propagation (via session) only works in the context of a single user (single session), when we're able to send a "↯-signal" from the write event to the read event moments. + +Now let's see what happens when we have two independent users, Alice and Bob. + +1. Alice calls `POST /comments` and adds a comment to the master database. +2. Immediately "after" that, Bob calls `GET /comments` to see the list of comments. + +

At "R", Bob will likely not see Alice's changes made at "W"

+ +Since the timelines of Bob are in another "frame of reference" than Alice's, and we did not send any signal from Alice to Bob, the request will likely be served from a replica node (not from the master), which means that Bob will likely see the old data. + +The same way as there is no absolute sequentiality in special relativity, there is also no guarantee regarding read-after-write consistency between different timelines. And it is generally fine: we don't care whether Bob loaded the old or the new data. Even if he is lucky and got the new data, he could instead have had just a little higher network latency, or pressed Reload button a little earlier, so he could have seen the old data even in the case there was no replicas in the cluster at all, and all requests would have been served by the master node only. + +### Propagating Timelines via a Pub-Sub Engine + +There are still cases where we want one user to immediately see the data modified by another user, i.e. establish some cross-user read-after-write consistency. + +If we think about it, we realize that it happens only in one use case: when a data modification made by Alice causes other users (Bob, Charlie etc.) to "unfreeze and re-render". I.e. we must already have a transport to propagate that "fanout-unfreeze" signal. So all we need is to just add a payload (with serialized timelines) as a piggy-back to this signal, and then, Bob, Charlie etc. will establish a read-after-write consistency with Alice's prior write. + +```typescript +// Ran by Alice who adds a comment. +app.post("/:topic_id/comments", async (req, res) => { + const topicID = req.params.topic_id; + const commentID = await EntComment.insert(req.vc, { topic_id: topicID, ... }); + await pubSub.publish(topicID, { + commentID, + timelines: req.vc.serializeTimelines(), + }); +}); + +// Ran by each user (Bob, Charlie etc.) to receive updates related +// to a particular topic (rough pseudo-code). +wsServer.on("subscribe", (ws, message) => { + return pubSub.subscribe(message.topicID, async (payload) => { + ws.vc.deserializeTimelines(payload.timelines); + const comments = await EntComment.select( + ws.vc, + { topic_id: message.topicID, ... }, + 100, + ); + ws.send(comments); + }); +}); +``` + +

Bob and Charlie at "R" will see Alice's changes made at "W"

+ +VC's method `deserializeTimelines()` **merges** the received timelines signal into the current VC's timelines. You can call call it as many times as needed, when you receive a pub-sub signal. + +### Saving Timelines in TimelineStorage + +If you can't easily use sessions in your app, there is a `TimelineStorage` abstraction to store the timelines in an external database. + +Ent Framework has one built-in implementation to save timelines on the master nodes in PostgreSQL (microsharded by `vc.principal`): + +```typescript +export const cluster = new Cluster({ ... }); +export const timelineStorage = new PgTimelineStorage({ cluster }); +... +// In the beginning of the request processing: +vc = await vc.loadTimelines(timelineStorage); +// Right before the response is sent to the browser: +await vc.saveTimelines(timelineStorage); +``` + +When using `PgTimelineStorage`, the timelines are saved in the following table in all microshards (you need to create it using [Database Migrations and pg-mig Tool](/advanced/database-schema-migrations) or somehow else): + +```sql +CREATE UNLOGGED TABLE timelines( + id bigserial PRIMARY KEY, + principal text NOT NULL, + data text NOT NULL, + created_at timestamptz NOT NULL +); +CREATE INDEX timelines_principal ON timelines (principal); +``` + +The table is append-only (with idempotent compaction step happening from time to time), so multiple clients can write to it using `saveTimelines()` without having any race conditions. + +You can also build your own `TimelineStorage` implementation (e.g. based off Redis) by extending `TimelineStorage` base class. Just make sure about taking care of race conditions when saving. + +```typescript +export class RedisTimelineStorage extends TimelineStorage { + ... + async load(principal: string): Promise { ... } + async save(principal: string, dataStr: string): Promise { ... } +} +``` + +### What Data is Stored In a Timeline + +VC timelines are basically an array of the following structures: + +* **shard**: microshard number where a write happened; +* **table**: the name of the table experienced a write in that microshard; +* **LSN**: a write-ahead log position after the above write; +* **expiration time**: a timestamp when the above information stops making sense, so Ent Framework can forget about it (typically, 60 seconds; defined in `maxReplicationLagMs` option of `PgClient`). + +There is no magic here: to propagate the minimal read-after-write consistency signal, we must know, which table at which microshard experienced a write. + +Notice one important thing: since there are no JOINs in Ent Framework, we read data from different microshards and track their timelines independently. That allows to assemble a read-after-write consistent snapshot from multiple microshards when reading. + +## Forced Master + +Besides the automatic replication lag tracking, you can also tell Ent Framework explicitly that you want to use only the master nodes for some particular calls. + +```typescript +const topic = await EntTopic.loadX( + vc.withTransitiveMasterFreshness(), + topicID, +); +const comments = await EntComment.select( + topic.vc, + { topic_id: topic.id }, + 100, +); +``` + +Calling to `vc.withTransitiveMasterFreshness()` derives you a new VC that, when used, will force the utilization of a "special" timeline. All the Ents loaded in this VC will be read from the corresponding island's master node. This mechanism is called "VC freshness"; in our case, `vc.freshness` equals to `MASTER` (the default freshness is `null` meaning "use timelines tracking engine"). + +Also, in the example above, `topic.vc` keeps master freshness, so any other Ents loaded with it will be read from master nodes as well. This is why it's called "transitive": once enabled on a VC, the freshness propagates to the further loaded Ents if you use your Ent's `vc` property. Be careful to not abuse the VC master freshness, otherwise you may introduce bottlenecks in your app's performance. + +## Forced Replica + +You can ask Ent Framework to always use a replica for a particular call: + +```typescript +const topic = await EntTopic.loadX( + vc.withOneTimeStaleReplica(), + topicID, +); +const comments = await EntComment.select( + topic.vc, // <-- it does NOT remember withOneTimeStaleReplica()! + { topic_id: topic.id }, + 100, +); +``` + +Notice that the replica freshness is *not transitive (*"one time"*)*: in the example above, you tell Ent Framework to load EntTopic with it, but `topic.vc` will have a regular (default) freshness. I.e. comments will be loaded using the regular timelines engine (from a replica if there were no recent writes to EntComment, or from master if there were). + +To highlight that you may likely read out-of-date rows from the database (replica is always lagging behind master), the method is named `withOneTimeStaleReplica()`: notice the word "stale". + +If you try to write some Ent using a VC with `STALE_REPLICA` freshness, then an interesting thing will happen: the write will still go to the master node, but the Ent's timeline won't remember that. This is convenient when you want to have some "insignificant write in background": i.e. you need your write to not affect the further reads going to replicas. + +## Conclusion + +In the examples above, we called `serializeTimelines()` and `deserializeTimelines()` methods manually on every endpoint. In real projects though, you most likely don't want to call them explicitly, since it produces too much boilerplate in the code. + +Instead, a recommended approach is to embed the above calls into your higher-level framework. E.g. a middleware can call `deserializeTimelines()` very early in the request processing lifecycle (and for all requests), and another middleware may call `serializeTimelines()` right before the response is flushed back to the browser. You may also want to store the timelines not in the session, but in some other ephemeral storage, like Redis. Each framework has its own way of processing the requests, so it's up to you, how you want to use those low-level Ent Framework methods. + + +# Sharding and Microsharding + +[Replication and Automatic Lag Tracking](/scalability/replication-and-automatic-lag-tracking) is not a silver bullet: you get fault tolerance and linear reads scaling, but there are limitations too: + +1. **You can't scale writes.** Eventually, your single master CPU will become a bottleneck. +2. **Single master disk throughput and IOPS have their limits.** Even in AWS, you can't scale them infinitely, there are hard caps on both. +3. **Physical replication in PostgreSQL is single-threaded.** Which means that at some point, single core CPU utilization on replicas will become a bottleneck: the master will still happily cope with writes, but the replicas won't catch up, having 100% CPU utilization in WAL replay process. +4. **You can't upgrade PostgreSQL across major versions** (e.g. from v16 to v17) without stopping the entire cluster or using logical replication (which is slower and is hard to manage). + +Microsharding (horizontal scaling) solves all of the above downsides. And it is an Ent Framework's built-in feature. + +## Sharding + +Sharding means that your table (including its structure, indexes etc.) exists on multiple PostgreSQL nodes (typically, on multiple master+replicas groups, which we call "islands" in Ent Framework). The data is split across the nodes though: no two islands share the same data. + +* When you insert a new row, the engine first needs to decide, what will be the destination island. It may be a random selection, or a selection based on some heuristics (e.g. we may want all the data of a particular customer live on the same island). +* When you update or delete a row, you also first locate its island, and then route the update/delete request there. + +## Microsharding + +Microsharding is a practical approach to do sharding: + +1. There are way more microshards in the cluster than islands or even physical nodes. For instance, in Ent Framework, each microshard is a PostgreSQL schema. Each schema (microshard) has identical tables structure, but the data in different microshards differ. +2. At logical level, island is a group of microshards. And at physical level, island is a set of master + replica nodes serving that group of microshards. +3. Microshards are typically small, so they can migrate from one island to another with no downtime. This allows to rebalance the load evenly, plus enables PostgreSQL upgrades across major releases with no downtime (i.e. you add new islands to the cluster and then tell the migration tool to evacuate the microshards from the old nodes). + +In Ent Framework, the words "shard" and "microshard" mean the same thing, we will use them interchangeably. + + +# Sharding Terminology + +Naming things is [one of two hardest problems in computer science](https://martinfowler.com/bliki/TwoHardThings.html), so before we continue, let's agree on terminology. + +## Node + +We use the word "node" to mean "a dabase server running on some machine, available in the network via a separate host:port pair". It may be a physical computer, a virtual machine, an AWS instance, an AWS RDS or Aurora PostgreSQL instance. + +## Island + +In Ent Framework, "island" is a group of nodes (machines): 1 master and N replicas (where N may be zero). Each node on some island effectively holds the same set of data as all other nodes on that island. Data replication across the nodes may be done using the standard database physical replication mechanisms: + +* managed by [repmgr](https://www.repmgr.org/) or other high level tools, to introduce failover/switchover; +* managed by AWS RDS/Aurora (in RDS terminology, "island" is called "database"); +* ... + +Every vendor uses different words to name what we call "island" here: + +* in PostgreSQL documentation, there is no common term; the closest one is probably "replication cluster" +* in AWS RDS and Aurora, they call it "database" +* in AWS Elasticsearch or OpenSearch services, it is "domain" +* in AWS Elasticache Redis, they call it "cluster" + +The name "island" is a common way to refer any of the above concepts. We also emphasize the logical nature of the island and that microshards can be migrated from one island to another (the same way as people sail between islands in the ocean). + +## Microshard (Shard) + +Microshard is a minimal unit of data rebalancing. Each shard is a PostgreSQL schema, example naming: `sh0001`, `sh4242`, `sh0000`. Typically, there are multiple microshards (say, \~50) on each island, and microshards are randomly distributed across islands (uniformly by the total size). + +* Once some data rows are written to a microshard, those data rows never move to another microshard. I.e. microshard is first determined at row creation time and then never changes. (This denotes a small flavor difference between "microshard" and "shard" terms: typically, rows are allowed to change their "macro shard", but are always nailed down to their microshards.) +* Microshards can be moved **as a whole** from one island to another without downtime. Since each microshard is small, it's a fast and granular process. +* Additional microshards can be added to the cluster with no downtime. E.g. if we have 300 microshards already, we can add 200 more and distribute them across the existing islands uniformly, so the newly created Ents will start being put there. You can't delete microshards once they are allocated though, because otherwise you'll lose the data. +* There can be up to 10000 microshards (the limit is arbitrary, you can make it larger if needed). The maximum number of microshards is determined by the PostgreSQL schemas naming convention: e.g. `sh1235` or `sh0012` names mean that there may only be up to 10000 microshards. +* A microshard schema in the database can be **inactive** or **active**. If it's inactive, it is in the process of allocation, or it has just been moved from one island to another. The schema gets "activated" on the new island and gets inactivated on an old island. + +## **Cluster** + +Cluster is a set of islands used by the same app. E.g. there can be a cluster which consists of 2 islands, and each island has 1 master and 2 replica nodes: 2\*(1+2) = 6 PostgreSQL machines. New island can be added to the cluster, or existing island can be removed from the cluster (after all microshards are moved out of it). + +
+ +It's important to understand that there are 2 points of view on a cluster: + +* **Physical:** cluster is a set of islands (it doesn't matter, which microshards are on each island). +* **Logical:** cluster is a set of microshards (it doesn't matter, which islands the microshards live on). + +{% hint style="info" %} +Notice that in PostgreSQL documentation, "cluster" means a smaller thing (a PostgreSQL installation on a particular machine, what we call "node" above). Even "replication cluster" is smaller there (a group of master + replica nodes, what we call "island"). In Ent Framework, "cluster" is a more overall concept: "group of islands" and "group of shards". +{% endhint %} + +## **Global Microshard, Shard 0** + +Typically, there is a "special" global microshard, which lives on a separate (with more CPU and more replicas) **island 0**. Tables in shard 0 do not exist in other microshards and have a low number of rows with rare writes and frequent reads (e.g. organizations, workspace metadata etc.). + +
+ +This setup is not mandatory though: it's perfectly fine to have the global microshard 0 located on an island together with other microshards; it's just a matter of load balancing. + +## **Logical and Physical Tables** + +Imagine our cluster has 3 "logical tables": `users`, `topics` and `comments` . + +A logical table (e.g. `users`) can be "sharded": in this case, there are effectively M physical tables in M microshards, all having the same DDL structure. Thus, physical table is a regular PostgreSQL table living on some master+replicas of an island in some microshard. + +To ensure that all M physical tables for the same logical table have the identical schema, a database migration tool (such as pg-mig) needs to be used; that is beyond the scope of Ent Framework, but the tools work in tandem. + +
+ +On the picture, there are 3 logical tables (`users`, `topics` and `comments`). The corresponding physical tables live in 4 microshards, and the microshards live on 2 islands. E.g. logical table `users` is represented by 4 physical tables `users` in 4 microshards. Since microshards lives on some island, and island consists of master and replica nodes, there are essentially replicas for every physical table in the cluster. + +## **Discovery Workflow** + +In Ent Framework, the engine needs to determine, which island each microshard is located on. This process is called "shards discovery". It happens at run-time, automatically. If a microshard has just been moved to another island, then the framework picks up this information immediately (with retries if needed). + +Another kind of workflow is "master-replica discovery". Master node of some island may fail at any time, and in this case, one of the replicas will be promoted to become the new master. Although the failover and replica promotion is not a part of Ent Framework (it's a feature of the replication toolset, like repmgr or AWS RDS), Ent Framework needs to react on the promotion promptly and with no downtime. + +## **Microshards Rebalancing** + +During the reblancing, the tool (such as pg-microsharding) determines, what would be an optimal distribution of microshards across islands (based on microshard sizes), and then performs a set of moves, in a controlled parallel way. + +* Rebalancing needs to run when you add a new island to the cluster, to evenly distribute microshards among islands. +* Rebalancing is used to upgrade between the major PostgreSQL versions. First, all microshards are moved away from an island, then an empty island gets re-created from scratch, and then microshards are rebalanced back. (Or both of the above processes run in parallel.) + +Although shards rebalancing is not a part of Ent Framework (you can use e.g. pg-microsharding tool), the engine still needs to be aware of a temporary "blip" which appears when a shard is deactivated on an old island, but is not yet activated on a new one. + + +# Locating a Shard and ID Format + +To enable microshardig support, we first need to configure the instance of `Cluster` class: + +```typescript +export const cluster = new Cluster({ + islands: async () => [ // sync or async + { + no: 0, + nodes: [ + { name: "abc-instance-1", config: { host: "...", ... } }, + { name: "abc-instance-2", config: { host: "...", ... } }, + ], + }, + { + no: 1, + nodes: [ + { name: "abc-instance-3", config: { host: "...", ... } }, + { name: "abc-instance-4", config: { host: "...", ... } }, + ], + }, + ], + createClient: (node) => new PgClient(node), + shardNamer: new ShardNamer({ + nameFormat: "sh%04d", + discoverQuery: + "SELECT unnest FROM unnest(microsharding.microsharding_list_active_shards())", + }), + ..., +}); +``` + +## Shards Discovery + +Notice the `shards` configuration property above. + +* `nameFormat`: this sprintf-style template defines, how Ent Framework should build the microshard schema name when it knows the microshard number. In our case, the schema names will look like `sh0123` or `sh0000`, and there will be up to 10000 microshards allowed. +* `discoverQuery`: Ent Framework will run this query on all islands from time to time to figure out, what shards are located where. It will also run this query immediately in several conditions, like "table not found" error (which may mean that a microshard has just been moved from one island to another, so Ent Framework needs to rediscover). + +There is also pg-microsharding library which allows you to manipulate microshard schemas: create them, activate, move and rebalance microshards across islands. When this library is used, you can utilize `SELECT * FROM unnest(microsharding.list_active_shards())` as a value for `discoverQuery`. + +As of the islands in the cliuster, just enumerate them and their nodes. Ent Framework will figure out, what nodes are masters and whan nodes are replicas. You can also change the list of islands and nodes in real-time, without restarting the app: Ent Framework is smart enough to pick up the changes if `islands` callback returns a different value (it is called from time to time). + +## Format of IDs + +Assume we have the following call: + +```typescript +const user = EntUser.loadX(vc, id); +``` + +When users are distributed across multiple microshards, Ent Framework decides, which microshard should it query the data from. The decision is made based on the ID prefix: + +
+ +To use the default microshard location strategy, there is a convention on ID format, it must consist of 3 parts: + +* `"1"` (Environment Number): you may want to make your IDs globally unique across the entire world, so all IDs in dev environment will start from e.g. 1, IDs in staging with 2, and IDs in production with 3. +* `"0246"` (Shard Number): this is where the microshard number reside in the ID structure. In the code, it is also referred as "Shard No". +* `"57131744498804"` (Entropy): a "never-repeating random-looking" part of the ID. It may not necessarily be random (other strategies are "auto-incremental" and "timestamp-based"), i.e. the concrete generation algorithm it's up to the library which generates the new IDs. + +Ent ID (and thus, its microshard number) is determined once, at the time when the Ent is inserted. Typically, each microshard schema has its own function that build the IDs, fills the environment and shard number, generates the "never-repeating random-looking" part: + +```typescript +const schema = new PgSchema( + "users", + { + id: { type: ID, autoInsert: "id_gen()" }, + email: { type: String }, + }, + ["email"] +); +``` + +Here, we use `id_gen()` function from [pg-id](https://www.npmjs.com/package/@clickup/pg-id) library, which by default generates the IDs in the format we mentioned above: + +``` +EssssRRRRRRRRRRRRRR + ^ ^^^^^^^^^^^^^^ + 4 14 +``` + +## Stored Functions in pg-id Library + +The complete list of `id_gen*()` functions in [pg-id](https://www.npmjs.com/package/@clickup/pg-id) library are: + +* `id_gen()`: generates next globally-unique randomly-looking id. The main idea is to not let external people infer the rate at which the ids are generated, even when they look at some ids sample. The function implicitly uses a sequence to get the information about the next available number, and then uses [Feistel cipher](https://en.wikipedia.org/wiki/Feistel_cipher) to generate a randomly-looking non-repeating ID based off it. +* `id_gen_timestampic()`: similar to `id_gen()`, but instead of generating randomly looking ids, prepends the "sequence" part of the id with the current timestamp. +* `id_gen_monotonic()`: the simplest and fastest function among the above: generates next globally-unique monotonic id, without using any timestamps as a prefix. Monotonic ids are more friendly to heavy INSERTs since they maximize the chance for btree index to reuse the newly created leaf pages. +* `id_gen_uuid()`: returns an ID in UUID format (PostgreSQL `uuid` type) with first several digits assigned to `Essss` prefix as in all other functions above. + +## Using UUID v4 for ID Fields + +You can also use `id_gen_uuid()` function if you want your primary keys to be in UUID v4 format (or utilize the built-in PostgreSQL function [`gen_random_uuid()`](https://www.postgresql.org/docs/current/functions-uuid.html) in case you don't need microsharding support). + +The UUID generated by that function looks like this: + +``` +10246xxx-xxxx-4xxx-Nxxx-xxxxxxxxxxxx +``` + +Here, as in the previous examples, `1` is environment number (e.g. production), `0246` is microshard number, `4` is the UUID version field, and `N` is a so-called "variant". All other digits are randomly generated. + +Notice that `id_gen_uuid()` replaces the first several digits in the string representation of UUID with the information regarding environment and microshard numbers. This trick doesn't cut too much of the UUID's entropy (UUID is 16 bytes; compare it to 8 bytes of `bigint`), but allows to use UUIDs in microsharded environment. + +Also, you need to use type `String` and not `ID` for the fields that hold an UUID data. This applies to `id` property and to all "foreign key like" fields. + +## Why Using Database Generated IDs? + +Let's get back to the previous example of an ID field definition: + +```typescript +const schema = new PgSchema( + "users", + { + id: { type: ID, autoInsert: "id_gen()" }, + ... + }, + ... +); +``` + +Also, the corresponding SQL table schema in every microshard is: + +```sql +CREATE TABLE users( + id bigint PRIMARY KEY DEFAULT id_gen(), + + ... +) +``` + +(In case you want IDs as UUID, use the built-in PostgreSQL type `uuid` instead of `bigint`.) + +### id\_gen() is Mentioned in Two Places + +Technically, you don't have to include `DEFAULT id_gen()` clause in your SQL table definition. For Ent Framework to operate, it's fully enough to define just `autoInsert="id_gen()"`. + +But we strongly advise to have both. Otherwise, you won't be able to e.g. connect to a node with `psql` and run `INSERT INTO users ...` safely, without thinking of IDs generation. It will also be hard to build database triggers if they insert `users` rows. + +### autoInsert is a String Property, not a Callback + +You probably wondered, why doesn't Ent Framework support `autoInsert` being a TypeScript callback? Why do we always ask the *database* to generate IDs, why don't we support application code ID generation (especially for UUIDs)? + +There are several reasons for this. + +1. As mentioned above, the best practice is to have the `autoInsert` expression defined in both Ent Framework schema and in the SQL table definition. Thus, we need an approach available in both TypeScript and SQL worlds; that is using an SQL expression as a string. (BTW, for non-ID fields, other available values for `autoInsert` are: `"now()"`, `"NULL"` or even `"'{}'"` for e.g. an empty array.) +2. When building batched INSERTs, Ent Framework uses the expression from `autoInsert` directly in the batched SQL queries. +3. If an Ent class has `beforeInsert` triggers, Ent Framework runs the expressions from `autoInsert` in a separate query, so the generated IDs are available in `beforeInsert` triggers early, even though the row is not yet inserted into the table. This allows to build "eventually consistent" logic without transactions. See more details about this in [Triggers](/getting-started/triggers) article. + + +# Sharding Low-Level API + +In [Locating a Shard and ID Format](/scalability/locating-a-shard-id-format) article we discussed, how Ent Framework automatically determines, which shard to use for a particular Ent, based on the Ent ID. + +But there is also a lower level set of methods in `Cluster` class, for the following use cases: + +* when you want to manipulate the shards manually; +* when you don't want to encode the shard number in an ID for some reason; +* when you need to use transactions (`acquireConn()` API). + +The API described below is exposed by `Cluster` class, see [Locating a Shard and ID Format](/scalability/locating-a-shard-id-format). + +```typescript +import pg from "pg"; + +export const cluster = new Cluster({ + islands: async () => [ // sync or async + { + no: 0, + nodes: [ + { name: "abc-writer-1", config: { host: "...", ... } }, + { name: "abc-reader-2", config: { host: "...", ... } }, + ], + }, + { + no: 1, + nodes: [ + { name: "abc-writer-3", config: { host: "...", ... } }, + { name: "abc-reader-4", config: { host: "...", ... } }, + ], + }, + ], + createClient: (node) => new PgClient({ + ...node, + // you can use your own Pool class here instead of node-postgres + createPool: (config) => new pg.Pool(config), + } satisfies PgClientOptions), + shardNamer: new ShardNamer({ + nameFormat: "sh%04d", + discoverQuery: + "SELECT unnest FROM unnest(microsharding.microsharding_list_active_shards())", + }), + ..., +}); +``` + +## Substituting the Default node-postgres Pool + +Instead of the default [node-postgres Pool](https://node-postgres.com/apis/pool) class, you can tell `PgClient` to use the custom one you built. Create that class and derive it from `pg.Pool` , then pass `createPool()` property that creates an instance of it. + +This is a rarely used feature: in most of the cases, you can just skip passing the `createPool` property, and `PgClient` will use the built-in node-postgres `Pool` class. + +## cluster.shardByNo(): Get a Shard by its Number + +This is the simplest way to get an instance of `Shard` class (representing a microshard) by its number: + +```typescript +import { MASTER } from "ent-framework"; + +const shard = cluster.shardByNo(42); +const pgClient = await shard.client(MASTER); +``` + +{% hint style="info" %} +Notice that `shardByNo()` is synchronous: it doesn't even check that such shard exists in the cluster. Instead, all errors are processed later, at the time when `shard.client()` is called. +{% endhint %} + +Having a `Shard` object, you obtain a `Client` instance (in our case, `pgClient`) which enbles access to one of the nodes backing that shard. + +As of the client's node role, you can pass `MASTER` (to access the master database client) or `STALE_REPLICA` (to access a random and arbitrarily lagging replica). + +You can also pass an instance of `Timeline` object to utilize the automatic replication lag tracking and let Ent Framework decide, whether it can choose a replica, or it should use the master this time: + +```typescript +const timeline = vc.timeline(shard, "users"); +const pgClient = await shard.client(timeline); +``` + +## Sending SQL Queries via a Shard Client + +`PgClient` class exposes 2 ways of sending queries to the database. (This applies to PostgreSQL; for other databases, especially non-SQL, the API is up to that `Client` class implementation.) + +Internally, `PgClient` maintains a pool of open connections and reuses them automatically. It also works great together with [pgbouncer](https://www.pgbouncer.org) (or any other conections pooler for PostgreSQL) in both "transaction pooling" and "connection pooling" modes. (In real projects, you'll most likely want to use "transaction pooling".) + +### pgClient.query(): Send a Single SQL Query + +You can use `query()` method of `PgClient` to send singular SQL queries: + +```typescript +const rows = await pgClient.query({ + query: ["SELECT email FROM users WHERE id=?", userID], + isWrite: false, + annotations: [vc.toAnnotation()], + op: "MY_SELECT", + table: "users", + // Optional properties. + hints: { enable_seqscan: "off" }, + batchFactor: 1, +}); +``` + +Notice that `query()` API is pretty verbose: it is not meant to be used in the code directly, introduce your own wrapper if you find yourself sending raw SQL queries frequently. (But better use Ent Framework's calls which hide all of the complexity behind a graph-like language.) + +Before the query is executed, Ent Framework basically prepends it with `SET search_path TO sh0123` clause within the same "implicit transaction" of the "simple multi-query protocol". I.e. if you access some table without providing its schema name, then the table will be searched in the current shard's schema (`sh0123` in the above example). + +Some properties like `annotations`, `op` and `table` are used for instrumentation purposes only. It is highly recommended to pass them, since it will make the built-in Ent Framework logging meaningful. + +{% hint style="info" %} +Overall, `query()` works similarly to "session pooling" mode in popular PostgreSQL poolers like pgbouncer. It's the exact method which Ent Framework higher level calls (like `loadX()` or `insert()`) use. +{% endhint %} + +### pgClient.acquireConn(): Get a Low-Level node-postgres Client + +If you want to access the native [node-postgres](https://node-postgres.com) library API (Node module: "pg") to use transactions, streaming etc., use the following code: + +```typescript +import { PoolClient } from "pg"; + +const conn: PoolClient = await client.acquireConn(); +try { + await conn.query("BEGIN"); + const res = await conn.query( + "INSERT INTO users(email) VALUES($1) RETURNING id", + ["test@example.com"], + ); + await conn.query( + "INSERT INTO photos(user_id, photo_url) VALUES ($1, $2)", + [res.rows[0].id, "s3.bucket.foo"], + ); + await conn.query("COMMIT"); +} catch (e) { + await conn.query("ROLLBACK"); + throw e; +} finally { + // Don't forget to ALWAYS call release() to put the connection + // back to the pool, including when an error happened, otherwise + // it will all explode badly. + conn.release(); +} +``` + +This example is also pretty verbose: try not to use this API in your code directly; instead, introduce some higher-level wrappers. + +## Other Ways of Accessing Shards + +There are other methods in `Cluster` that allow you to access shards. + +### cluster.globalShard(): Access a Global Shard + +There is a special microshard in the cluster with number 0. It is called "global shard". Typically, the global shard lives on a separate island with more replicas, since it is used to store shared low cardinal data (like organizations, workspaces, user accounts etc.) that doesn't need to be sharded. + +Calling `globalShard()` is the same as calling `shardByNo(0)`. + +### cluster.nonGlobalShards(): Get the List of All Shards + +This async method returns all microshard instances except the global shard: + +```typescript +const shards = await cluster.nonGlobalShards(); +``` + +### cluster.randomShard(): Get a Random Shard in the Cluster + +When you insert a new row to the database, Ent Framework calls this method to choose a shard for the insertion. This happens for Ents with `shardAffinity` equals to `RANDOM_SHARD` (i.e. when the Ent is not colocated with some other parent Ent). + +### cluster.shard(id): Get a Shard from the ID prefix + +Earlier in [Locating a Shard and ID Format](/scalability/locating-a-shard-id-format) article we disussed, what format an ID should have to work in microsharding environment: + +
+ +If you have such an ID in a variable, a call to `cluster.shard(id)` will parse it and return a shard instance which you can then use to send low-level SQL queries to that shard. + +## Working with Islands + +Sometimes you want to work with even lower primitive than a microshard, with an island itself. + +This is helpful when your app has some background worker (or crawler) that needs to traverst through all records of a particular table, in all shards, and you want to control the processing parallelism: not more than 1 worker process per each island (to not overload the database with concurrent queries). + +### cluster.islands(): Get All Islands of the Cluster + +The method allows you to enumerate all islands of the cluster to e.g. spawn worker processes per each of them: + +```typescript +const islands = await cluster.islands(); +for (const island of islands) { + await spawnWorkerIfNotRunningAlready(island.no); +} +``` + +Since Ent Framework supports real-time reconfiguration, the list of islands may change after the call to `islands()`, so be careful to run the above code from time to time. + +### cluster.island(no): Get One Island by its Number + +Then, in each worker process, you may want to get an instance of an isand with the number corresponding to that worker: + +```typescript +async function worker(islandNo: number) { + const island = await cluster.island(islandNo); + const shardsOfIsland = island.shards(); + const master = island.master(); + const aliveReplica = island.replica(); + ... +} +``` + +### island.master(): Get a Client for Island Master Node + +Previously, we learned that the queries sent to a "shard client" are delivered in the context of that shard's PostgreSQL schema (i.e. they run as if they are prefixed with `SET search_path TO sh0123` clause). + +The queries sent to an "island client" are executed in the context of PostgreSQL schema `public`. In most of the cases, you'll want to override this and provide a particular schema name as a prefix of the table name: + +```typescript +const master = island.master(); +await master.query({ + query: ["SELECT email FROM sh0123.users WHERE id=?", userID], + // ^^^^^^ + ... +}); +``` + +### island.shards(): Get the Currently Known Shards on an Island + +Island clients are typically used to build "cross-shard" queries on a particular island. The most common example is building a UNION ALL query that allows to load the data from multiple shards on the same island more effectively than going "shard after shard": + +```typescript +const shards = island.shards(); +const masters = await mapJoin( + shards, + async (shard) => shard.client(MASTER), +); +const query = masters + .map((client) => ` + (SELECT id FROM ${client.shardName}.users + WHERE needs_processing + LIMIT 100) + `.trim()) + .join("\n UNION ALL\n`); +const ids = await island.master().query({ + query: [query], + ... +}); +``` + +Here, we ask the database to return us the users that "need to be processed" by the background job, from all shards of a particular island. From each shard, we return not more than 100 IDs. Considering that we have an index on the `WHERE` condition, such approach of crawling the users will be more effective than going "shard after shard". + + +# Shard Affinity and Ent Colocation + +When designing your Ent graph, it's important to think in advance, how will sharding strategy look like for those Ents: what microshard will be chosen once a new Ent is created. + +The strategy is defined by the reqired `shardAffinity` configuration property: + +```typescript +export class EntTopic extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: ..., + ... + }); + } +} +``` + +You have 3 options described below. + +## Global Shard: shardAffinity=GLOBAL\_SHARD + +This is the simplest strategy possible: all new Ents created are just placed to "shard 0" (aka "global shard"). By doing this, you essentially disable sharding for the Ent. + +Using the global shard works best for the Ents which have relatively low cardinality in the database (like workspaces, sometimes user accounts etc.). That Ent must also experience not too many writes (comparing to the number of reads), i.e. it should have no strict needs for horizontal scaling. + +## Colocate With Parent: shardAffinity=\["parent1", "parent2", ...] + +Another commonly used strategy is to place the newly created Ent to the same microshard as some of this Ent's parent have. By doing this, you tell Ent Framework that your child Ent is located "near" its parent (or parents). + +E.g. we may want to put all comments of a particular topic to the same microshard as this topic has: + +```typescript +const schema = new PgSchema( + "comments", + { + id: { type: ID, ... }, + topic_id: { type: ID }, + ..., + }, + [] +); + +export class EntComment extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: ["topic_id"], + ..., + }); + } +} + +// Creates the comment in the microshard whose number +// is parsed out of topicID prefix. +await EntComment.insert(vc, { topic_id: topicID }); +``` + +Sometimes Ents have nullable field edges. If you still want to use parent colocation in this case, you can provide multiple field names in `shardAffinity`: Ent Framework will try to infer the microshard number from them in order of appearance, from the first non-null field. + +Since Ent Framework does batching of the calls by microshards, having more children Ents in the same microshard significantly improves performance when e.g. doing `select()` calls. + +Also, if you have colocation based on some parent pointing field edge (foreign key), Ent Framework is able to infer the microshards to query: + +```typescript +// The queries will be sent only to the microshards +// where topicID1, topicID2 etc. live. +const comments = await EntComment.select( + vc, + { topic_id: [topicID1, topicID2, ...] }, + 100, +); +``` + +I.e. the main reason to use colocation is that you don't need to have inverses (cross-shard foreign keys) defined in the Ent to query, if the call arguments includes knowledge about the parent Ent IDs. + +## Random Shard on Insert: shardAffinity=\[] + +This strategy creates Ents in a randomly chosen microshard. It works best only for a small number of Ent classes, the ones that define "roots" of "colocation hierarchies" (like users). + +The idea is that you may e.g. defined EntWorkspace as having `shardAffinity=[]`, so it will be created in a random shard. Then, all other Ents that are "children" to EntWorkspace (e.g. EntUser), may have `shardAffinity` pointing to the parent's field edge (like `workspace_id`). Consequently, other may be colocated to their own parents (e.g. EntTopic colocated to EntUser, and EntComment colocated to EntTopic). This way, all the data related to the same EntWorkspace will appear in the same microshard, and no inverses will be needed. (Of course, such approach only works for relatively small workspaces.) + + +# Inverses and Cross Shard Foreign Keys + +We have already touched the topic of inverses and loading Ents across multiple microshards in [Ent API: select() by Expression](/getting-started/ent-api-select-by-expression) article. We also noted that in many cases, it's better to colocate "related" Ents in one microshard: [Shard Affinity and Ent Colocation](/scalability/shard-affinity-ent-colocation). + +Now, it's time to discuss how inverses work in details. + +## Ents with Random Shard Affinity + +Let's first build a pretty artificial "family" of the Ents (EntUser—EntTopic—EntComment), where each Ent is created in a random shard at insert time. (In real life, you'll likely want most of your Ents to be [colocated](https://docs.ent-framework.net/scalability/shard-affinity-ent-colocation) to their parents, but for the best illustration, we'll make the opposite assumption). + +```typescript +export class EntUser extends BaseEnt( + cluster, + new PgSchema("users", { + id: { type: ID, autoInsert: "id_gen()" }, + ... + }), +) { + static override configure() { + return new this.Configuration({ + shardAffinity: [], + ... + }); + } +} + +export class EntTopic extends BaseEnt( + cluster, + new PgSchema("topics", { + id: { type: ID, autoInsert: "id_gen()" }, + // Reference to parent 1. + creator_id: { type: ID }, + // Reference to parent 2 (optional). + last_commenter_id: { type: ID, allowNull: true }, + ... + }), +) { + static override configure() { + return new this.Configuration({ + shardAffinity: [], + inverses: { + creator_id: { name: "inverses", type: "topic2creators" }, + last_commenter_id: { name: "inverses", type: "topic2last_commenters" }, + }, + ... + }); + } +} + +export class EntComment extends BaseEnt( + cluster, + new PgSchema("comments", { + id: { type: ID, autoInsert: "id_gen()" }, + // Reference to parent. + topic_id: { type: ID }, + ... + }), +) { + static override configure() { + return new this.Configuration({ + shardAffinity: [], + inverses: { + topic_id: { name: "inverses", type: "comment2topics" }, + }, + ... + }); + } +} +``` + +Notice the following: + +1. `shardAffinity=[]` for all of the above Ents. It means that, at insert time, the target microshard will be chosen randomly. +2. There is `inverses` configuration property, which tells Ent Framework, how it can find children Ents located in other microshards than the parent Ents (e.g. how to find all topics related to a particular creator). + +## SQL Tables for Inverses + +Before we continue, let's look at the tables structure we need to have in **all microshards** of the database. + +```sql +-- All of sh0001, sh0002 etc. schemas must have this: +CREATE TABLE users( + id bigint PRIMARY KEY DEFAULT id_gen(), + ... +); + +CREATE TABLE topics( + id bigint PRIMARY KEY DEFAULT id_gen(), + creator_id bigint NOT NULL, + last_commenter_id bigint, + ... +); +CREATE INDEX topics_creator_id ON topics(creator_id); +CREATE INDEX topics_last_commenter_id ON topics(last_commenter_id); + +CREATE TABLE comments( + id bigint PRIMARY KEY DEFAULT id_gen(), + topic_id bigint, + ... +); +CREATE INDEX comments_topic_id ON comments(topic_id); + +-- And the main table for this article... +CREATE TABLE inverses( + id bigint PRIMARY KEY DEFAULT id_gen(), + created_at timestamptz NOT NULL DEFAULT now(), + type varchar(64) NOT NULL, + id1 bigint, + id2 bigint, + UNIQUE(type, id1, id2) +); +``` + +Tables `users`, `topics` and `comments` are not much special, except that their `*_id` fields are not declared as `FOREIGN KEY`. No surprise: there can be no SQL-enforced foreign keys across microshards, so we just keep the fields being of the regular `bigint` type. We still define indexes for those fields though, for faster selection. + +Now, notice the `inverses` table. It is treated by Ent Framework in a special way. + +When you e.g. insert an EntTopic row, Ent Framework first chooses the target microshard randomly and then creates a row in `inverses` table in the parent's shard. It then inserts the row to the destination microshard. Thus, Ent Framework remembers, what are the children microshards (encoded in `inverses.id2`) for each parent ID (`inverses.id1`). + +## An Example of What's Actually Inserted + +Probably the simplest way to understand inverses is to look at a particular example, what's inserted and where when creating the Ents. + +Consider the following "family" Ents creation. + +```typescript +// +// Remember that we had the following inverses defined for EntTopic: +// inverses: { +// creator_id: { name: "inverses", type: "topic2creators" }, +// last_commenter_id: { name: "inverses", type: "topic2last_commenters" }, +// }, +// +// And for EntComment: +// inverses: { +// topic_id: { name: "inverses", type: "comment2topics" }, +// }, +// +const creatorID = await EntUser.insert(vc, { ... }); +const commenterID = await EntUser.insert(vc, { ... }); +const topicID = await EntTopic.insert(vc, { + creator_id: creatorID, + last_commenter_id: commenterID, + ... +}); +const commentID = await EntComment.insert(vc, { + topic_id: topicID, + ... +}); +``` + +Internally, Ent Framework will run the following SQL queries (pseudo-code): + +```sql +-- Microshard for the user is randomly chosen as sh0888. +INSERT INTO sh0888.users(id) VALUES(id_gen()) + RETURNING id INTO $creatorID; + +-- Microshard for anotjer user is randomly chosen as sh0999. +INSERT INTO sh0999.users(id) VALUES(id_gen()) + RETURNING id INTO $commenterID; + +-- Microshard for the topic is randomly chosen as sh0123. +$topicID := sh0123.id_gen(); +INSERT INTO sh0888.inverses(type, id1, id2) VALUES + ('topic2creators', $creatorID, $topicID); +INSERT INTO sh0999.inverses(type, id1, id2) VALUES + ('topic2last_commenters', $commenterID, $topicID); +INSERT INTO sh0123.topics(id, creator_id, last_commenter_id) VALUES + ($topicID, $creatorID, $commenterID); + +-- Microshard for the comment is randomly chosen as sh0456. +$commentID := sh0456.id_gen(); +INSERT INTO sh0123.inverses(type, id1, id2) VALUES + ('comment2topics', $topicID, $commentID); +INSERT INTO sh0456.comments(id, topic_id) VALUES + ($commentID, $topicID); +``` + +Notice that, because of `{ name: "inverses", type: "topic2creators" }` inverse specifier, Ent Framework knows that the inverses table name is `inverses`, and the value of the `type` field there is `"topic2creators"`. You can choose your own values for both of those options: the above example is just a convention. + +
+ +Inverses are always inserted to the parent's shard (not to the child's shard) and **before** the actual row (`id_gen()` defined in `autoInsert` option is called in a separate database query). This guarantees that there will be no situation when there is a row in some shard, but its corresponding inverse does not exist in another shard. I.e. there are always **not less** inverses in the cluster than the Ent rows. + +And to maintain the "not less" invariant, when an Ent is deleted, the corresponding inverses are deleted **after** it (not before). So even if the inverse deletion query fails, it will still be okay for us: we'll just have a "hanging inverse". + +As a result, the following rows will appear in the database tables: + +``` +sh0888 - creator's shard: +- users(id:10888001) +- inverses(type:topic2creators id1:10888001 id2:10123002) + $creatorID $topicID +sh0999 - commenter's shard: +- users(id:10999001) +- inverses(type:topic2last_commenters id1:10999001 id2:10123002) + $commenterID $topicID +sh0123 - topic's shard: +- topics(id:10123002 creator_id:10888001 last_commenter_id:10999001) +- inverses(type:comment2topics id1:10123002 id2:10456003) + $topicID $commentID +sh0456 - comment's shard: +- comments(id:10456003 topic_id:10123002) +``` + +## Children Ent Microshard Hints + +So, inverses allow Ent Framework to find all children Ents related to a particular parent ID, like all topics by a last commenter ID: + +```typescript +const topics = await EntTopic.select( + vc, + { last_commenter_id: "10999001" }, + 100, +); +``` + +This would send the following SQL queries to the database (pseudo-code): + +```sql +SELECT id2 FROM sh0999.inverses + WHERE type='topic2last_commenters' AND id1='10999001' + INTO $id2s; + +FOR $shard IN shards_from_ids($id2s) LOOP + SELECT * FROM $shard.topics + WHERE last_commenter_id='10999001' + LIMIT 100; +END LOOP; +``` + +(Of curse, as everything in Ent Framework, the above will be batched into compound SQL queries when multiple `select()` calls are happening in parallel.) + +Notice that, despite inverses hold the full ID of the child Ent in `id2` field, only the microshard number from `id2` is used in Ent Framework logic (see `shard_from_id()` in the above pseudo-code). I.e. `id2` is used to get "covering hints" about what microshards should be queried when loading the data. + +This is a really powerful approach. It solves the problem of cross-shard consistency. Since we have not less inverses than the children rows, Ent Framework just collects all unique microshards from those inverses `id2` fields and then queries them using a regular SELECT. If there were some hanging (excess) inverses in the database, it is not a big deal: the engine will just query a little bit more microshards than it needs to, but the SELECTs from those extra microshards will just come empty. + +It makes sense to delete the hanging inverses in background from time to time ("inverses fixer" infra), since they may accumulate slowly. Currently, Ent Framework doesn't have any logic to help with this, but in the future, such functionality may appear. Since inverses are treated as just covering hints, not cleaning them up is generally fine. + +## Unique Keys + +When `shardAffinity=[]`, the microshard is chosen randomly at insert time. + +But if the Ent has a unique key defined in its schema, *this randomness is not absolute*. The seed of the random number generator is assigned based on the value of the Ent's unique key, so all `insert()` calls with the same unique key will essentially choose the same microshard for insertion. + +Let's consider an example. + +```typescript +export class EntTopic extends BaseEnt( + cluster, + new PgSchema( + "topics", + { + id: { type: ID, autoInsert: "id_gen()" }, + creator_id: { type: ID }, + slug: { type: String }, + }, + ["slug"], // <-- unique key + ), +) { + static override configure() { + return new this.Configuration({ + shardAffinity: [], + ... + }); + } +} +``` + +Imagine you have a web server route which creates a new topic with a particular slug: + +```typescript +app.post("/topics", async (req, res) => { + const topicID = await EntTopic.insert(req.vc, { + creator_id: req.vc.principal, + slug: String(req.body.slug), + }); +}); +``` + +The user clicks the button in the browser, and the topic creation POST request is sent to the server. + +Now imagine that there is some network issue happened between the backend and the database after the INSERT query is sent, but before the response is received. Or the database is overloaded, so it accepted the INSERT query, but failed to respond within the allocated time. In this case, there will be a "non-idempotent query timeout": the backend will see the error, but it will not know, whether the INSERT query succeeded (and thus, the topic is created), or it failed internally. + +In case of an error, the user will likely press the button again, and it will cause another POST request to insert the topic with the same slug. + +If we chose the microshard trully randomly, then the 2nd INSERT would create the topic row in another shard, and there would be a possiblilty to create 2 topics with the same slug in 2 different microshard. This would be really bad: we would violate the unique key constraint across shards (there is nothing in inverses engine preventing from creating duplicates). + +This is why the microshard is chosen pseudo-randomly, based on the Ent's unique key (it it is defined). Thus, an attempt to create a duplicated Ent would fail with a unique key constraint error, and the app would be able to handle it. In the above example, `insert()` would've thrown an error, but you could use `insertIfNotExists()` call which would just return null; see [Ent API: insert\*()](/getting-started/ent-api-insert). + + +# Shards Rebalancing and pg-microsharding Tool + +Ent Framework by itself does not include a tool that allows to create new microshards or move them from one island to another. As we earlier discussed in [Locating a Shard and ID Format](/scalability/locating-a-shard-id-format), for the engine, a microshard is just a PostgreSQL schema located on some island. You, as a user, define the naming convention to be used for such schemas: + +```typescript +export const cluster = new Cluster({ + shardNamer: new ShardNamer({ + nameFormat: "sh%04d", + discoverQuery: + "SELECT nspname FROM pg_namespace WHERE nspname ~ 'sh[0-9]+'", + }), + ... +}); +``` + +The above means that your microshard schemas are named like `sh0000`, `sh0123` etc., and also you provided a query that enumerates all microshard schemas available on a particular island. + +To manage the actual schemas, an external tool needs to be used. + +Below is the README content of [pg-microsharding](https://www.npmjs.com/package/@clickup/pg-microsharding) tool. + +## pg-microsharding: Microshards Support for PostgreSQL + +See also [TypeScript API documentation](https://github.com/clickup/pg-microsharding/blob/master/docs/globals.md). + +[![CI run](https://github.com/clickup/pg-microsharding/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/clickup/pg-microsharding/actions/workflows/ci.yml/badge.svg?branch=main) + +The [pg-microsharding](https://www.npmjs.com/package/@clickup/pg-microsharding) CLI tool enables microshard schemas management across multiple PostgreSQL servers. You can do the following: + +* Add and remove microshard schemas. +* Move a microshard from one PostgreSQL server to another with no downtime. +* Automatically rebalance microshards among multiple servers, so that each server will become of approximately the same size. +* Copy regular PostgreSQL schemas (unrelated to microshards) from one host to another without downtime (think of no-downtime dump-restore workflow). +* Activate and deactivate microshards. +* Enumerate active microshard schemas. +* View the entire cluster layout: what microshard schemas are where, of what size, and how many reads/writes do the experience. +* Weighted rebalancing: when one server looks overloaded, you can "dissolve out" some shards from it to other servers to achieve equal load. + +Each microshard is a PostgreSQL schema with numeric suffix. Microshard schemas have the same set of tables with same names; it's up to the higher-level tools to keep the schemas of all those tables in sync (e.g. see [pg-mig](https://www.npmjs.com/package/@clickup/pg-mig) tool). + +## Usage + +``` +pg-microsharding install + [--schema-name-fmt=SCHEMA_NAME_FMT] + [--dsn=DSN | --dsns=DNS1,DSN2,...] + +pg-microsharding list | ls + [--weight-sql='SELECT returning weight with optional unit'] + [--verbose] + [--dsn=DSN | --dsns=DNS1,DSN2,...] + [--json] + +pg-microsharding allocate + --shard=N | --shards=N-M + --migrate-cmd='shell command to run migrations' + --activate={yes | no} + [--dsn=DSN | --dsns=DNS1,DSN2,...] + +pg-microsharding factor + --shard=N | --shards=N,M,... | --shards=DSN-PREFIX + --factor=P|+P.Q|-P.Q|"*P.Q" + [--dsn=DSN | --dsns=DNS1,DSN2,...] + +pg-microsharding move + --shard=N + --from=DSN-OR-DSN-PREFIX-OR + --to=DSN-OR-DSN-PREFIX + --activate-on-destination={yes | no} + [--wait] + [--validate-fks] + [--max-replication-lag-sec=N] + [--dsns=DNS1,DSN2,...] + [--deactivate-sql='SQL $1 SQL'] + +pg-microsharding rebalance + --activate-on-destination={yes | no} + [--wait] + [--validate-fks] + [--deactivate-sql='SQL $1 SQL'] + [--weight-sql='SELECT returning weight with optional unit'] + [--decommission=DSN1,DSN2,...] + [--max-replication-lag-sec=N] + [--parallelism=N] + [--dsn=DSN | --dsns=DNS1,DSN2,...] + +pg-microsharding cleanup + [--dsn=DSN | --dsns=DNS1,DSN2,...] + +pg-microsharding copy + --schema=SCHEMA-NAME + --from=DSN-OR-DSN-PREFIX-OR + --to=DSN-OR-DSN-PREFIX + [--wait] + [--validate-fks] + [--max-replication-lag-sec=N] + [--dsns=DNS1,DSN2,...] +``` + +## Environment Variables + +The tool receives parameters from command line option, but allows to set defaults for most of them using environment variables. + +Some variables are standard for psql command: + +* `PGUSER`: default database user +* `PGPASSWORD`: default database password +* `PGHOST`: default database host (or multiple hosts, comma-separated) +* `PGPORT`: default database port +* `PGDATABASE`: default database name +* `PGSSLMODE`: default SSL mode (e.g. "prefer") +* `PGOPTIONS`: options (like `-c maintenance_work_mem=500MB`) used when creating indexes + +Custom variables of the tool itself: + +* `DSNS` or `PGDSNS` or `PGHOST`: default value for `--dsns` option, comma-separated list of DSNs (see below) +* `MIGRATE_CMD`: default value for `--migrate-cmd` option +* `WEIGHT_SQL`: default value for `--weight-sql` option +* `DEACTIVATE_SQL`: default value for `--deactivate-sql` option +* `PARALLELISM`: default value for `--parallelism` option +* `MAX_REPLICATION_LAG_SEC`: default value for `--max-replication-lag-sec` option +* `SCHEMA_NAME_FMT`: default value for `--schema-name-fmt` (applies only\ + to install action; if not passed, `"sh%04d"` is used) + +## Configuration File: pg-microsharding.config.ts + +Instead of setting the environment variables, you can export the same exact values in `pg-microsharding.config.ts` file by e.g. deriving them directly from the [Ent Framework](https://ent-framework.org/) cluster configuration: + +```javascript +import { cluster } from "ents/cluster"; + +export default async function(action: "apply" | "undo" | string) { + const islands = cluster.options.islands(); + return { + PGDSNS: islands + .map((island) => island.nodes.map(({ host }) => host) + .flat() + .join(","), + PGPORT: 5432, // we don't want to use pgbouncer port here + PGUSER: firstNode.user, + PGPASSWORD: firstNode.password, + PGDATABASE: firstNode.database, + PGSSLMODE: firstNode.ssl ? "prefer" : undefined, + MIGRATE_CMD: "yarn -s pg-mig", + }; +} +``` + +Unless `--skip-config` flag is passed, the file `pg-microsharding.config.ts` is searched in all parent folders starting from the current working directory when `pg-microsharding` is run (typically you want to have it in the root of your project, near the other configuration files). + +You can export-default a regular function, an async function, or even a plain constant object. + +## DSN and Addressing Databases + +Option `--dsns`, if required, should be a comma separated list of DSNs. + +Also, you may pass duplicated DSNs and even DSNs of replicas: the tool will filter them out and remain only master DSNs in the list. + +DSN format examples (parts defaults are from environment variables): + +* `postgresql://user:pass@hostname/db?options` (all parts are optional) +* `hostname:port/db` (all parts except the hostname are optional + +## Command Line Tool + +The `pg-microsharding` library consists of 2 parts: + +1. A CLI tool allowing you to manipulate with microshards. +2. A set of PostgreSQL stored functions to call them from your app. + +### Show Cluster Layout: pg-microsharding list + +```bash +pg-microsharding list +``` + +This action prints the list of all PostgreSQL islands (pointed by DNSn), microshards and some statistics. + +If `--json` flag is passed, prints a JSON version of the output. Otherwise, prints a pseudo-graphic representation. + +In `--verbose` mode, also prints detailed statistics about insert/update/delete, index scans and seqscans. + +
+ +### Allocate New Microshards: pg-microsharding allocate + +```typescript +pg-microsharding allocate --shards=301-309 --activate=yes +``` + +This action allows you to create more microshard schemas in the cluster. The microshards are created on PostgreSQL the host pointed by the 1st DSN, so after it's done, run `pg-microsharding rebalance` to spread that new schemas across other nodes. + +Each microshard can either be "active" or "inactive". When you create them, you tell the tool, should the microshards become active immediately (and thus, visible to `microsharding_list_active_shards()` API) or not. You can always activate the schemas later using the same exact command (it is idempotent). + +The tool runs `--migrate-cmd` command right after creating the inactive microshards, assuming that your migration tool will initialize them properly. + +
+ +
+ +### Move One Microshard: pg-microsharding move + +```bash +pg-microsharding move \ + --shard=42 --from=host1 --to=host2 \ + --activate-on-destination=yes \ + --max-replication-lag-sec=20 +``` + +Microshards can be moved from one PostgreSQL node to another. There is no need to stop writes while moving microshards: the tool uses PostgreSQL logical replication to stream each microshard table's data, and in the very end, acquires a quick write lock to finalize the move. + +There are many aspects and corner cases addressed in the move action, here are some of them: + +* The move is fast even for large microshards. The tool internally uses the same approach for data copying as `pg_dump`. First recreates the tables structure on the destination, except most of the indexes and foreign key constraints (only the primary key indexes or REPLICA IDENTITY indexes are created at this stage, since they are required for the logical replication to work). Then, it copies the data, utilizing the built-in PostgreSQL tablesync worker; this process is fast, since it inserts the data in bulk and doesn't update indexes. In the end, the tool creates the remaining indexes and foreign key constraints (this is where you may want to increase [maintenance\_work\_mem](https://www.postgresql.org/docs/current/runtime-config-resource.html) and [max\_parallel\_maintenance\_workers](https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAX-PARALLEL-MAINTENANCE-WORKERS) for the role you pass to pg-microsharding, or use `PGOPTIONS` environment variable, since it directly affects the indexes creation time). Overall, this approach speeds up the copying by \~10x comparing to the naive way of using logical subscriptions. +* At each long running step, the tool shows a descriptive progress information: how many tuples are copied so far, what is the elapsed %, how much time is left, what are the SQL queries it executes (dynamically updatable block in console) etc. +* It also shows replication lag statistics for all physical replicas of the source and the destination, plus the logical replication lag of the temporary subscription. +* By default, foreign keys on the destination node are created with `NOT VALID` clause. This speeds up the move severely, because we assume that the foreign keys are valid on the source, and we don't want to spend time re-validating them. You can amend this behavior by passing `--validate-fks` flag: the move will take more time though. +* In the end, the tool activates the microshard on the destination and deactivates on the source, but it does it only when the replication lag in seconds dropped below some reasonable threshold (defaults to 20 seconds, but you can pass a lower value to be on a safe side). So the write lock is guaranteed to be acquired for only a brief moment. +* The tool runs it all in an automatically created tmux session. If you accidentally disconnect, then just connect back and rerun the same command line: instead of running another move action, if will jump you back in the existing session. + +If you're unsure, you can practice with the move without activating the microshard on the destination (and without deactivating it on the source) by passing `--activate-on-destination=no` option. This is like a "dry-run" mode, where the tool does all the work, except the very last step. The moved schema on the destination won't be activated, and it will also be renamed using some long descriptive prefix (including the move date). + +At any moment, you can abort the move with ^C. It is safe: half-moved data will remain on the destination, but the microshard schema will remaim invisible there for e.g. `microsharding_list_active_shards()` API (see below). If you then rerun the `move` action, it will start from scratch. + +
+ +In addition to printing to console, `move` action also writes the same output to a log file in `/tmp/pg-microsharding.$uid` directory. Each individual move creates its own file prefixed with a timestamp and including the microshard number moved. + +### Clean Old Moved Copies: pg-microsharding cleanup + +```bash +pg-microsharding cleanup +``` + +When you move one or more microshards, pg-microsharding doesn't delete the old copy from the source host. Instead, it renames the schema (using some long descriptive prefix with date) and deactivates it. + +Later, when you are sure that everything went well, you can remove such "backup" copies by running the cleanup action. It is interactive: it will tell you, what it wants to delete, and ask for an explicit confirmation. + +### Rebalance All Islands: pg-microsharding rebalance + +```bash +pg-microsharding rebalance --activate-on-destination=yes +``` + +This action runs multiple "move" sub-actions in parallel, utilizing [tmux](https://github.com/tmux/tmux/wiki) panes. Notice that tmux is required: it allows to resume the rebalancing if your SSH console on the server gets disconnected (in this case, just run the `rebalance` action again, and you'll "jump into" the existing session). + +Before running the moves, the action calculates weights of each shard (by default, the weight is the microshard tables size in bytes, mutuplied by per-shard "weight factor"; see below). Then, it estimates, which microshards need to be moved to what islands, to achieve a more or less uniform distribution. The algorithm is complicated: among other heuristics, it tries to make sure that each island gets approximately the same number of microshards with comparable sizes (e.g. if you allocate 100 new empty microshards, then rebalancing will spread them across islands uniformly). + +
+ +Once the rebalancing plan is ready, the tool will print it to you and ask for your confirmation. You can always run `pg-microsharding rebalance` and then press ^C to just see, what *would* happen if you rebalance. + +After rebalancing, the result may look like: + +
+ +At any time, you can abort the rebalancing with ^C in any of the tmux panes. It is as safe as aborting the `move` action. + +### Evacuate All Microshards from an Island (Decommissioning) + +```bash +pg-microsharding rebalance \ + --decommission=host2 --activate-on-destination=yes +``` + +This mode of "rebalance" action allows you to remove a PostgreSQL host from the cluster, or even upgrade PostgreSQL to the next major version with no downtime. It moves all the microshards from the provided DSN, so the host becomes "empty". + +E.g. after decommissioning, the result may look like (notice that one node became empty): + +
+ +Now, you can remove the host from the cluster or upgrade PostgreSQL, then rebalance the microshards back (rebalancing works fine across different major PostgreSQL versions). + +### Tweak Island Weights: pg-microsharding factor + +```bash +pg-microsharding factor --shards=host1 --factor="*1.2" +``` + +Imagine you have 10 PostgreSQL islands with rebalanced microshards, and you see on your monitoring charts that some island is loaded more than all other islands. E.g. it may experience higher CPU load, higher disk throughput etc. + +Such situation typically happens, because one microshard became too large, or there is a customer data in some microshard that causes more load than the data of an average customer. In case you don't want to investigate this case too much, you can "duct tape" it by artificially "dissolving" a fraction of microshards from that overloaded island to other islands. + +When you run `pg-microsharding factor --factor="*1.2"`, the tool artificially increases the "weight" of each microshard on the provided host (in this example, the increase is by 1.2, i.e. by 20%). This information is then remembered in the microshards themselves (and is displayed in `list` action), so you can run rebalancing and "dissolve" some of the microshards among other hosts. As a result, your target island will become less loaded (on average), and by repeating this step several times, you can achieve a more fair load distribution. + +The "weight increase factor" is technically stored as a SQL comment on the microshard schema, and it travels along with the microshard when you move it. + +### Dump-Restore a Schema with no Downtime: pg-microsharding copy + +```bash +pg-microsharding copy \ + --schema=my-schema --from=host1 --to=host2 \ + --wait \ + --max-replication-lag-sec=20 +``` + +This action works very similarly to the move action, but it doesn't deactivate/activate microshards, so it may be used for other regular PostgreSQL schemas. Unlike when using regular `pg_dump` and restoring, you may continue sending writes to the source host, and the changes will eventually be replayed on the destination while the action is running. + +It is very convenient to run this action with `--wait` flag: in this case, it will ask for the user confirmation right before removing the logical subscription, so you'll have a chance to stop writes on the source host and quickly reroute the traffic to destination before finishing the action and removing the logical subscription. + +### Replication Lag Prevention + +The tool tries hard to not affect the replication lag of the destination nodes when moving or rebalancing microshards. It waits until the lag drops below `--max-replication-lag-sec` seconds before running heavy operations (or until the user presses Shift+S to force-continue). + +Also, if you want the tool to pause explicitly and wait until the user presses Shift+S before activating the shard on the destination node, you can use the `--wait` option. + +### Tuning Indexes Creation Performance + +When moving or rebalancing microshards, indexes creation may take a substantial amount of time. You can tell pg-microsharding to use more memory and run faster by passing `PGOPTIONS` environment variable to the tool, similar to how you would do it in psql: + +```bash +PGOPTIONS="-c maintenance_work_mem=500MB -c max_parallel_maintenance_workers=4" \\ + pg-microsharding rebalance ... +``` + +Some options you may want to increase: + +* [maintenance\_work\_mem](https://www.postgresql.org/docs/current/runtime-config-resource.html) +* [max\_parallel\_maintenance\_workers](https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAX-PARALLEL-MAINTENANCE-WORKERS) +* [work\_mem](https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-WORK-MEM) (not clear whether it affects index creation time, but you can try) + +It's generally safe to set high values in those options, because they are applied only in the session that runs queries like `CREATE INDEX` and do not affect other database sessions/queries. + +## PostgreSQL Stored Functions API + +This is the second part of pg-microsharding tool: a set of stored functions you add to your database. + +### Installing into the Database + +To manually install or upgrade the library's stored functions in the database without a migration tool, run the `install` action: + +```bash +pg-microsharding install +``` + +
+ +Otherwise, run the following SQL files in your up- and down-migrations to install (or upgrade) and uninstall the tool: + +* sql/pg-microsharding-up.sql: to install/upgrade the library +* sql/pg-microsharding-down.sql: to uninstall + +E.g.: + +```sql +-- mig/20250628100000.add-pg-microsharding.public.up.sql +CREATE SCHEMA microsharding; +SET search_path TO microsharding; +\ir ../pg-microsharding/sql/pg-microsharding-up.sql +``` + +```sql +-- mig/20250628100000.add-pg-microsharding.public.dn.sql +SET search_path TO microsharding; +\ir ../pg-microsharding/sql/pg-microsharding-down.sql +DROP SCHEMA microsharding; +``` + +In the above example, we create a separtate schema for the library, but it is not mandatory: you can also install it into schema `public` (all of the API functions have `microsharding_` prefix). + +### List Active Shards: microsharding.list\_active\_shards() + +This function returns the list of active microshard schemas in the current PostgreSQL database. When using the tool with [Ent Framework](https://ent-framework.org/), mention it in your `Cluster` object: + +```typescript +export const cluster = new Cluster({ + shards: { + nameFormat: "sh%04d", + discoverQuery: + "SELECT unnest FROM unnest(microsharding.microsharding_list_active_shards())", + }, + ... +}); +``` + +### Microsharding Debug Views + +The `microsharding_migration_after()` function creates so-called "debug views" for each sharded table in your cluster: + +```sql +SELECT microsharding.microsharding_migration_after(); +``` + +For instance, it you have `sh0001.users`, `sh0002.users` etc. tables, then it will create a debug view `public.users` with the definition like: + +```sql +-- This is what pg-microsharding creates automatically. +CREATE VIEW public.users AS + SELECT * FROM sh0001.users + UNION ALL + SELECT * FROM sh0002.users + UNION ALL + ...; +``` + +Even more, if you pass the list of all PostgreSQL hosts, and those hosts can access each other without a password (e.g. they have `/var/lib/postgresql/N/.pgpass` files), then those debug views will work **across all shards on all nodes, including the remote ones** (using [foreign-data wrapper](https://www.postgresql.org/docs/current/postgres-fdw.html) functionality): + +```sql +SELECT microsharding.microsharding_migration_after('host1,host2,host3'); +``` + +So **for debugging purposes**, you'll be able to run queries *across all microshards on all hosts* in your `psql` sessions. + +Of course those **debug views are not suitable for production traffic**: cross-node communication in PostgreSQL, as well as query planning, work not enough inefficiently. Do not even try, use application-level microshards routing, like e.g. [Ent Framework](https://ent-framework.org/) provides. + +``` +$ psql +postgres=# SELECT shard, email FROM users + WHERE created_at > now() - '1 hour'::interval; +-- Prints all recent users from all microshards, including +-- the microshards on other PosgreSQL nodes! Use for +-- debugging purposes only. +``` + +As of `microsharding_migration_before()`, you must call it before any changes are applied to your microsharded tables: + +```sql +SELECT microsharding.microsharding_migration_before(); +``` + +The function drops all of the debug views mentioned above. E.g. if you remove a column from a table, PostgreSQL would not allow you to do it it this column is mentioned in any of the views, so it's important to drop the views and re-create them afterwards. + +Typically, you just call `microsharding_migration_before()` in your pre-migration sequence and then call `microsharding_migration_after()` in your post-migration steps. + + +# Connection Pooling + +When `min` client option is provided in the [cluster configuration](/getting-started/connect-to-a-database), Ent Framework maintains up to this number of established database connections ("pre-warmed"), even when there are no queries coming. This allows for the new queries to execute quickly: establishing a new connectioon is an expensive process that may take tens of milliseconds and involve multiple round-trips to the server (especially when using SSL-encryption). + +At the same time, having many persistent connection in some databases is expensive as well. For instance, PostgreSQL architecture implies that there is one independent OS process behind every single active connection. So setting PostgreSQL config's [max\_connections](https://www.postgresql.org/docs/current/runtime-config-connection.html) to a value larger than \~100 (varies depending on the number of CPU cores on the server and available memory) is not the best idea. + +## Direct Connections + +Imagine you have one database server and 20 Node app processes running in your cluster. If each app opens 5 persistent connections to the database, you'll have 20\*5 = 100 database processes, idle most of the time, which exhausts that 100 cap mentioned above. And it's not even considered a large cluster. + +
+ +It's obvious that this picture doesn't scale: you can't launch more Node apps in case you experience heavier traffic, because those processes will overload the database by too many connections. + +## Connection Pooler + +A classical solution for this is to inject a so-called "connection pooler" (like PgBouncer for PostgreSQL). + +* It handles thousands of incoming app's connections, so you can have as many Node processes as needed. +* It keeps a small and limited number of open outgoing connections to the database server. +* It receives client queries (transactions) from the incoming connections, then figure out, is there an "idle" outgoing database connection. If so, it proxies the transaction to that connection; if not, then it waits toll some database connections become available again. + +
+ +Now, you can have thousands (ane even tens of thousands) persistent incoming connections per database, and if they are mostly idle (which is the case), then it will all work the right way. + +Often times, when the database server is installed on a regular machine, people setup the connection pooler (such as PgBouncer) on the same machine, just to protect the database from overloading with the number of connections (processes). If you use AWS RDS or Aurora, then it may make sense to setup PgBouncer on a separate set of the machines (or containers) in the cluster, in the same availability zone as the RDS/Aurora databases. Despite Aurora is more tolerant to the number of incoming connections than vanilla PostgreSQL, it still does not handle tens of thousands of them, so a separate connection pooler is always preferred. + +## Ent Framework and PgBouncer + +Generally, Ent Framework doesn't care, what you put to the nodes configuration, be it a direct PostgreSQL connection or a reference to some connection pooler like PgBouncer. If you use a pooler, then make sure to: + +1. Set it up in "transaction pooling" mode. In this mode, if a transaction is started in some connection, then the pooler makes sure that all of the queries within this transaction are proxied to the same PostgreSQL server connection as the transaction `BEGIN` statement itself. +2. Prefer PgBouncer. This is because PgBouncer sometimes emits its own error codes (related to connections or queries time out, disconnects etc.), which Ent Framework understands in addition to the vanilla PostgreSQL client library error codes. This improves the engine behavior in case it loses a node due to network errors or a restart, and speeds up the master/replica rediscovery process. + + +# Database Migrations and pg-mig Tool + +As opposed to classical ORMs (like [Prisma](https://www.prisma.io) or [Drizzle](https://orm.drizzle.team)), Ent Framework does not include any built-in database migration tool, and it doesn't infer SQL tables schema for you from the TypeScript schema definition. + +In terms of the storage layer, Ent Framework operates at a lower level than the ORMs mentioned above. Ent abstraction is in fact very close to PostgreSQL layer. Such approach is the exact sweet spot and the exact trade-off between being flexible (e.g. to expose all bleeding edge PostgreSQL features without hiding them) and being useful in practice. + +Database migration is a complicated process with many details. You can use any existing tools (like Liquibase) to organize it, or you can plug in Ent Framework to your existing database (considering you are already doing migrations for that database somehow). + +There is one important aspect though: the most popular solutions you may have heard of do not support microsharding out of the box. They are targeting just the most common use case: a single database on a single host. + +## Migrations in Microsharding Environment + +When working with [microsharding](https://docs.ent-framework.net/scalability/sharding-microsharding), you'll have hundreds of PostgreSQL schemas (with names like `sh01234`) living on multiple islands and PostgreSQL nodes. All those schemas (microshards) have exactly the same set of tables, indexes, stored functions etc. At the same time, you don't want to sacrifice any of the PosrgreSQL built-in features when adding microsharding to your project. + +So, database migration gets several important aspects: + +1. It should track database schema version per each microshard individually. I.e. imagine that you add a column to some table, run the migration to apply it to all physical tables in microshards, and it fails in the middle. Next time you run the migration process, it must continue from the microshard where it left off. +2. It shoud apply the changes to multiple PostgreSQL nodes and microshards in a controlled-parallel way, otherwise the migration will take forever to finish. I.e. the migration tool must know the entire cluster configuration, not only one PostgreSQL node. + +The [pg-mig](https://www.npmjs.com/package/@clickup/pg-mig) tool solves all of the above, and it helps to organize the migration for databases backed by Ent Framework. + +Below is the content of pg-mig tool README file. + +## pg-mig: PostgreSQL schema migration tool with microsharding and clustering support + +See also [Full API documentation](https://github.com/clickup/pg-mig/blob/master/docs/globals.md). + +
+ +The [pg-mig](https://www.npmjs.com/package/@clickup/pg-mig) tool allows to create a PostgreSQL database schema (with tables, indexes, sequences, functions etc.) and apply it consistently *across multiple PostgreSQL nodes*, also *across multiple microshard schemas* on multiple hosts. The behavior is transactional per each microshard per migration version ("all or nothing"). + +In other words, pg-mig helps to keep your database clusters' schemas identical (each microshard schema will have exactly the same DDL structure as any other schema on all other hosts). + +In case your project doesn't have microsharding and uses just 1 database, you can use pg-mig too: just tell it to target only 1 schema (e.g. `public`). + +## Usage + +``` +pg-mig + [--migdir=path/to/my-migrations/directory] + [--hosts=host1,host2,...] + [--hosts=host[:port][/db],...] + [--hosts=postgres://[user][:password][@]host[:port][/db],...] + [--port=5432] + [--user=user-that-can-apply-ddl] + [--pass=password] + [--db=my-database-name] + [--createdb] + [--undo=20191107201239.my-migration-name.sh] + [--make=my-migration-name@sh] + [--list | --list=digest] + [--parallelism=8] + [--dry] +``` + +All of the command line arguments are optional, the tool uses defaults from environment variables or `pg-mig.config.ts` (or `.js`) file; see details below. + +## Environment Variables + +There are variables standard for `psql` tool: + +* `PGHOST`: database server hostname, or `host:port/db` string, or `postgres://user:password@host:port/db` string (all parts except of `host` are optional). When the cluster has multiple nodes in it, separate them here with commas. You may also include both master and replica hosts in the list: the tool is smart enough to only use the master nodes and ignore everything else. +* `PGPORT`: default database servers port. +* `PGUSER`: default database user. +* `PGPASSWORD`: default database password. +* `PGDATABASE`: default database name. + +Other variables: + +* `PGMIGDIR`: the default value for `--migdir` option, a directory with migration version files. +* `PGCREATEDB`: if 1, assumes that `--createdb` flag is passed. It forces pg-mig to try creating the database if it doesn't exist, which is very convenient in dev (and test) environment. Also, it PostgreSQL servers are down or booting, this flag will tell pg-mig to wait for them. + +## Configuration File + +Instead of setting the environment variables, you can export the same exact values in `pg-mig.config.ts` (or `pg-mig.config.js`) file by e.g. deriving them directly from the [Ent Framework](https://ent-framework.org/) cluster configuration: + +```javascript +import { cluster } from "ents/cluster"; + +export default async function(action: "apply" | "undo" | string) { + const islands = cluster.options.islands(); + const firstNode = islands[0].node[0]; + return { + PGHOST: islands + .map((island) => island.nodes.map(({ host }) => host) + .flat() + .join(","), + PGPORT: 5432, // we don't want to use pgbouncer port here + PGUSER: firstNode.user, + PGPASSWORD: firstNode.password, + PGDATABASE: firstNode.database, + PGSSLMODE: firstNode.ssl ? "prefer" : undefined, + PGMIGDIR: `${__dirname}/mig`, + PGCREATEDB: process.env.NODE_ENV === "development", + after: async () => { + // Will be called after the migrations succeed. Here, you can e.g. + // upsert some initial objects in the database if they don't exist. + }, + }; +} +``` + +The file `pg-mig.config.*` is searched in all parent folders starting from the current working directory when `pg-mig` is run (typically you want to have it in the root of your project, near the other configuration files). + +You can export-default a regular function, an async function, or even a plain constant object. + +## Migration Version Files + +When running in default mode, `pg-mig` tool reads (in order) the migration versions `*.up.sql` files from the migration directory and applies them all on the hosts passed (of course, checking whether the version file has already been applied before or not). + +The migration version file name has the following format (examples): + +``` +mig/ + before.sql + 20231017204837.do-something.sh.up.sql + 20231017204837.do-something.sh.dn.sql + 20241107201239.add-table-abc.sh0000.up.sql + 20241107201239.add-table-abc.sh0000.dn.sql + 20241201204837.change-other-thing.sh.up.sql + 20241201204837.change-other-thing.sh.dn.sql + 20251203493744.install-pg-extension.public.up.sql + 20251203493744.install-pg-extension.public.dn.sql + after.sql +``` + +Here, + +* The 1st part is a UTC timestamp when the migration version file was created. +* The 2nd part is a descriptive name of the migration (can be arbitrary). Think of it as of the commit title. +* The 3rd part is the "schema name prefix" (microshard name prefix). The SQL operations in the file will be applied only to the schemas whose names start with that prefix. +* The 4th part is either "up" ("up" migration) or "dn" ("down" migration). Up-migrations roll the database schema version forward, and down-migrations allow to undo the changes. +* There are 2 optional special files: `before.sql` and `after.sql`. They are executed on every PostgreSQL hosts once per each pg-mig run, in independent transactions. It is convent to run some common initialization or maintenance there, especially when working with microsharding (there will be an example provided later in Advanced section). + +It is your responsibility to create up- and down-migration SQL files. Basically, you provide the DDL SQL queries on how to roll the database schema forward and how to roll it backward. + +You can use any `psql`-specific instructions in `*.sql` files: they are fed to `psql` tool directly. E.g. you can use environment variables, `\echo`, `\ir` for inclusion etc. See [psql documentation](https://www.postgresql.org/docs/current/app-psql.html) for details. + +## Apply the Migrations + +To run the up migration, simply execute one of: + +``` +pnpm pg-mig +yarn pg-mig +npm exec pg-mig +deno run pg-mig +``` + +(To run in Deno v2+, you first need to add `"pg-mig": "pg-mig"` to `scripts` section in your `package.json`.) + +Technically, pg-mig doesn't know anything about microsharding; instead, it recognizes the databasde schemas. Each migration version will be applied (in order) to all PostgreSQL schemas (aka microshards) on all hosts. The schema names should start from the prefix provided in the migration version file name. + +If multiple migration files match some schema, then only the file with the **longest prefix** will be used; in the above example, prefix "sh" effectively works as "sh\* except sh0000", because there are other migration version files with "sh0000" prefix. + +E.g. imagine you have the following migration version files: + +``` +before.sql +20231017204837.do-something.sh.up.sql # .sh. +20241107201239.add-table-abc.sh0000.up.sql # .sh0000. +20241201204837.change-other-thing.sh.up.sql # .sh. +20251203493744.install-pg-extension.public.up.sql # .public. +after.sql +``` + +Then, the following will happen in parallel on all hosts and for all microshards: + +* On every PostgreSQL host, `before.sql` will run. Until it succeeds, no other migration versions will even start running. +* For every `shNNNN` schema (basically, all schemas starting with "sh" prefix) except `sh0000`, the version `do-something.sh` will be applied first, and then, if it succeeds, the `change-other-thing.sh` will be run. Notice that `sh0000` is excluded, because there exist other migration file versions targeting `sh0000` precisely (and "sh0000" prefix is longer than "sh"). +* For `sh0000` schema, `add-table-abc.sh0000` will be run. +* For `pubic` schema, `install-pg-extension.public` will be run. +* In the end, on each host, `after.sql` will run (in case the migration succeeds). + +All in all, the behavior here is pretty intuitive: if you want to target a concrete schema, just use its full name; if you want multiple schemas to be considered, then use their common prefix. + +If the migration file application succeeds, it will be remembered on the corresponding PostgreSQL host, in the corresponding schema (microshard) itself. So next time when you run the tool, it will understand that the migration version has already been applied, and won't try to apply it again. + +Each migration version file is applied atomically, in a single transaction. Also, it't the same exact transaction where pg-mig remembers that the version has been applied, so there is no chance that your version will run out of sync with the database. + +When the tool runs, it prints a live-updating information about what migration version file is in progress on which host in which schema (microshard). In the end, it prints the final versions map across all of the hosts and schemas. + +If you have multiple PostgreSQL hosts and/or multiple target schemas, you can control the level of parallelism with `--parallelism=N` command line option (defaults to 10). + +## Undo the Migrations + +With e.g. `--undo=20231017204837.do-something.sh` argument, the tool will run the down-migration for the corresponding version on all nodes. If it succeeds, it will remember that fact on the corresponding node in the corresponding schema. Only the very latest migration version applied can be undone, and you can undo multiple versions one after another of course. + +Undoing migrations in production is not recommended (since the application code may rely on its new structure), although you can do it of course. The main use case for undoing the migrations is **during development**: you may want to test your DDL statements multiple times, or you may pull from Git and get someone else's migration before yours, so you'll need to undo your migration and then reapply it. + +## Create a New Migration Version File + +With `--make=my-migration-name@sh` argument, pg-mig creates a new pair of empty files in the migration directory. E.g. if you run: + +``` +pg-mig --make=my-migration-name@sh +``` + +then it will create a pair of empty files which looks like `my-dir/20251203493744.my-migration-name.sh.up.sql` and `my-dir/20251203493744.my-migration-name.sh.dn.sql` which you can edit further. + +Of course, you can also create such a pair of files manually. + +New migration version files can only be appended in the end of the list (lexicographically, or by timestamp, which is the same). If pg-mig detects that you try to apply some migrations conflicting with what's remembered in the database, it will print the error and refuse to continue. This is similar to "fast-forward" mode in Git, and we'll talk about it in details later in the article. + +## The Initial Migration + +When you start using pg-mig tool, run it with `--make` to create your initial migration: + +``` +pg-mig --make=initial@public +``` + +Since almose every PostgreSQL database has schema `public` pre-created, it's convenient to target this schema in your initial migration version. If you need microsharding support, then that initial (or the following) migration version may create the desired number of microshard schemas. + +### In a New Project + +If you use pg-mig in a brand new project, then just edit the created `*.initial.public.up.sql` and \``*.initial.public.dn.sql` files in your text editor and run `pg-mig` to apply the versions. + +```sql +-- 20251203493744.initial.public.up.sql +CREATE TABLE users( + id bigserial PRIMARY KEY, + email varchar(256) NOT NULL +); +``` + +And the corresponding down-file: + +```sql +-- 20251203493744.initial.public.dn.sql +DROP TABLE users; +``` + +During the execution of the above files, pg-mig will set the corresponding shema as current; in the above example, with the implicit `SET search_path=public` query. + +For debugging purposes, while building the SQL DDL statements, it's convenient to undo and apply the version in one command line: + +```bash +pg-mig --undo=20251203493744.initial.public; pg-mig +``` + +### In an Existing Project + +If you plug in pg-mig to an existing project, to start using the tool for all further database migrations, just use `pg_dump` and put its output to the initial version file: + +```bash +pg_dump --schema-only --schema=public your-db-name \ + > mig/20251203493744.initial.public.up.sql +``` + +If your database has multiple schemas, but no microsharding, you have 2 options: + +1. If the schemas are completely independent on each other, so the changes may apply in parallel, use `*.schema1.{up,down}.sql` and `*.schema2.{up,down}.sql` files. +2. Otherwise (and it would be the most frequent case), just use `*.public.{up,down}.sql` version files targeting schema `public` . In most of PostgreSQL databases, schema `public` pre-exists and is mentioned in the default `search_path`, so targeting this schema in your migration version files will guarantee that the versions will be applied strictly sequentially, with no parallelism. + +## Use Standard psql Meta-Commands + +The real power of pg-mig that many other migration tools don't have is that the migration version files are processed through the [standard `psql` tool](https://www.postgresql.org/docs/current/app-psql.html), so you can use its [meta-commands](https://www.postgresql.org/docs/current/app-psql.html) there. Each file is also applied atomically: "all or nothing", in a single transaction. + +Below are several examples (see `psql` documentation for more). + +### Include Other \*.sql Files + +You can include other `*.sql` files (even using relative paths): + +```sql +-- mig/20231017204837.do-something.sh.up.sql +\ir ../vendor/path/to/another/file.sql +ALTER TABLE ... +``` + +### Echo Diagnostics + +Use `\echo` for debugging + +```sql +-- mig/20231017204837.do-something.sh.up.sql +\echo Running a dangerous query... +ALTER TABLE ... +``` + +### Assign and Use Variables + +The `psql` tool allows to define macros and use them as variables: + +```sql +-- mig/20231017204837.do-something.sh.up.sql +SELECT 'hello' AS var1 \gset +\echo :var1 +UPDATE my_table SET some=:'var1'; +``` + +### Use Environment Variables + +If you assign e.g. `process.env.VAR = "a,b,c"` (or return it) in your `pg-mig.config.ts` file, you can use that value in all of the version files using the standard `psql` feature: + +```sql +-- mig/20231017204837.initial.public.up.sql +\set HOSTS `echo "$VAR"` +SELECT my_function(:'VAR'); +``` + +### More Meta-Commands + +See the [official psql documentation](https://www.postgresql.org/docs/current/app-psql.html) for more meta-commands. + +## Transactions, CREATE INDEX CONCURRENTLY + +Every migration version file is executed in a separate transaction, but sometimes you'll want to make an exception. + +E.g. it is highly discouraged to create indexes in transactions using the plain `CREATE INDEX` query, especially when the table is large. The query acquires a write lock on the table, so no data can be written to it until the index creation finishes, which may take many minutes. + +Luckily, PostgreSQL supports a non-blocking version of this query, `CREATE INDEX CONCURRENTLY`. It allows writes when the index is creating. The query has its downsides though: + +1. It may be 2 times slower than the regular `CREATE INDEX`. +2. In rare cases, it may fail and leave the index in a "broken" state. Nothing too bad will happen in terms of the database health though: you'll just need to drop that broken index and retry. +3. It must run outside of `BEGIN...COMMIT` transaction block. + +Use the following up-migration file to deal with the downsides above: + +```sql +-- $parallelism_per_host = 2 +COMMIT; +DROP INDEX IF EXISTS users_email; +CREATE UNIQUE INDEX CONCURRENTLY users_email ON users(email); +BEGIN; +``` + +And the down-migration file: + +```sql +COMMIT; +DROP INDEX CONCURRENTLY IF EXISTS users_email; +BEGIN; +``` + +Here, we first tell pg-mig that it should not run this script with concurrency higher than `$parallelism_per_host=2` (for instance, if you have multiple microshard schemas `shNNNN` on that host, then it will apply the query not to all of them simultaneously, but slower). This is a good practice to not max out the database server CPU (PostgreSQL also has a built-in protection against running too many maintenance queries in parallel, but often times it's better to be explicit). + +Then, we close the transaction that pg-mig automatically opens for each migration version file, run `CREATE INDEX CONCURRENTLY` and, in the end, open a new transaction to let pg-mig commit the new version update fact to the database. It makes this migration version non-transactional, so there is a nonzero chance that it may fail. Also, as `CREATE INDEX CONCURRENTLY` may legally fail as well and produce a "broken index", we use `DROP INDEX IF EXISTS` query before creating the index, to remove any leftovers in case you manually retry. + +In a rare case when the migration fails, you'll be able to just rerun pg-mig: it will just continue from the place where it failed. (In fact, when using microsharding, it will only continue with the schemas that failed, so the rerun will be way quicker than the initial run.) + +## Parallelism Limiting Options + +Here is the complete list of `-- $` pseudo comments that pg-mig supports in the migration version files: + +* `$parallelism_per_host=N`: as mentioned above, this option forces the parallel migrations for schemas on the same host to wait for each other, not allowing to run more than N of then at the same time. +* `$parallelism_global=N`: limits parallelism of this particular version *within the same schema prefix* across all hosts. +* `$delay=M`: introduces a delay (in ms) between each migration. You can use it with `$parallelism_global` to reduce load on the database even further. +* `$run_alone=1`: if set to 1, no other migrations, *including other schema prefixes*, will run on any other host while this one is running. I.e. it introduces a global ordering of the migration files application across schemas. This option is useful when you want to e.g. install a PostgreSQL extension used in other schemas, so you want all other schemas to wait until the installation finishes. + +## Advanced: Use With pg-microsharding Library + +Overall, there are many popular alternatives to pg-mig for managing a single database without sharding. But when it comes to migrating an entire cluster with multiple nodes or working with microsharding, pg-mig shines. + +The recommended library to manage the microshards schemas is [pg-microsharding](https://www.npmjs.com/package/@clickup/pg-microsharding). To couple it with pg-mig, create the following before/after scripts: + +```sql +-- mig/before.sql +CREATE SCHEMA IF NOT EXISTS microsharding; +SET search_path TO microsharding; +\ir ../pg-microsharding/sql/pg-microsharding-up.sql +SELECT microsharding.microsharding_migration_before(); +``` + +```sql +-- mig/after.sql +\set PG_MIG_HOSTS `echo "$PG_MIG_HOSTS"` +SELECT microsharding.microsharding_migration_after(:'PG_MIG_HOSTS'); +``` + +The `PG_MIG_HOSTS` environment variable is automatically assigned by pg-mig tool with the value like: `host1:port/db,host2:port/db,...`. + +The `microsharding_migration_after()` function from pg-microsharding creates so-called "debug views", which are giant `UNION ALL` across all tables in all shards on all PostgreSQL nodes. This allows to query the sharded tables for the data, as if they are not sharded. Of course, it is slow and should only be used for debugging purposes (don't use the debug views from your app). See more details in pg-microsharding documentation. + +## Advanced: Merge Conflicts + +Migration version files are applied in strict order per each schema, and the same way as Git commits, they form a dependency **append-only** chain. + +### Happy Path: Version is Appended + +The scenario below will happen most of the times. + +Imagine that on your local dev environmant (e.g. on your laptop) you have already applied the following migration versions to particular schemas in your local database: + +``` +20231017204837.do-something.sh +20241201204837.change-other-thing.sh +20241202001000.and-one-more-thing.sh +``` + +Then, another developer pushes the code with a new version file: + +``` +20241202002000.their-thing.sh.up.sql +``` + +And you pull it to your local working copy: + +``` +20231017204837.do-something.sh.up.sql +20241201204837.change-other-thing.sh.up.sql +20241202001000.and-one-more-thing.sh.up.sql +20241202002000.their-thing.sh.up.sql <-- new version pulled +``` + +Here, if you run pg-mig tool, it will happily apply that new version, since its timestamp comes after all of the versions you already have in your database. + +Since the changes in the database are relatively rare, in most of the cases, you'll experience this "happy" behavior. + +### Unhappy Path: Explicit Merge Conflict + +Now imagine you still had the same versions in your local database: + +``` +20231017204837.do-something.sh +20241201204837.change-other-thing.sh +20241202001000.and-one-more-thing.sh <-- you work on this +``` + +But when you pulled, you got the new version file sitting in the middle: + +``` +20231017204837.do-something.sh.up.sql +20241201204837.change-other-thing.sh.up.sql +20241202000000.middle-thing.sh.up.sql <-- new version pulled +20241202001000.and-one-more-thing.sh.up.sql +``` + +If you then run pg-mig tool locally, it will refuse to work: + +``` +Migration timeline violation: you're asking to apply +version 20241202000000.middle-thing.sh, although +version 20241202001000.and-one-more-thing.sh has already +been applied. Hint: make sure that you've rebased on +top of the main branch, and new migration versions are +still the most recent. +``` + +So what you'll need to do is to undo your latest migration version and then rerun pg-mig: + +``` +pg-mig --undo=20241202001000.and-one-more.sh +pg-mig +``` + +### Unhappy Path: Implicit Conflict + +Imagine you added a new version file: + +``` +20231017204837.do-something.sh.up.sql +20241202001000.your-new-thing.sh.up.sql <-- not yet pushed +``` + +You tested everything locally and are now ready to push to Git. But right before, you must pull from Git and ensure that your new verson file is still in the very end of the list of migration version files. Because if you don't do it, and some other developer appended another version file, the following trouble will appear: + +``` +20231017204837.do-something.sh.up.sql +20241202001000.your-new-thing.sh.up.sql <-- not yet pushed +20241202002000.other-dev-thing.sh.up.sql <-- just pulled +``` + +If you blindly push this, then there is a risk that anyone (or any environment) where `20231017204837.do-something.sh` and `20241202002000.other-dev-thing.sh` are already applied, will not be able to migrate anymore: they will get the error mentioned above. + +I.e. right before pushing, you must ensure that all migration version files you add within the new commits really appear in the end of the versions in Git. If not, then you'll need to rename your files using the latest timestamp: + +``` +mv 20241202001000.your-new-thing.sh.up.sql \ + 20241202002200.your-new-thing.sh.up.sql +``` + +In practice, the situation is not as bad as it sounds: + +1. If it breaks, it's easy to fix: just rename one file and push a new commit. +2. Database changes are relatively rare, and deployments also don't typically happen immediately after each pushed commit (unless you are very lucky), so the chance of catching such an ordering conflict are low. + +## Advanced: Migration Versions Digest + +Often times, when deploying the application code, we need to ensure that the database cluster schema is "good enough" for this code. Which means that all of the migrations have fully succeeded before the further code deployment (it's assumed that migrations produce backward compatible database schemas, otherwise it all makes not a lot of sense). + +To help with this check, pg-mig exposes the concept called "version digest". It's like a database schema version, but appended with a hashcode of the migration version files combined. + +Digest is a string, and by comparing 2 digests lexicographically, you may make a decision, which one is "better" (or, if you don't want "better/worse" comparison, you can also compare them for strict equality). If the database's digest is **greater or equal to** the application code's digest, then the code is compatible to the currently existing cluster schema, so the code can be deployed ("your database is ahead of the code"). + +* Run `pg-mig --list=digest` to print the digest of the current migration files on disk (i.e. "code digest"), like `20250114061047.1552475c743aac01`. +* Use `loadDBDigest()` function exported by pg-mig Node library to extract the digest from the current databases used by the app (i.e. "database digest"). E.g. you can call it from your app's `/health` endpoint to allow querying the current database version digest from a deployment script or pipeline. + +Every time the whole migration process succeeds, the digest is saved to *all database nodes*, so it can be later retrieved with `loadDBDigest()`. If you have multiple nodes managed by pg-mig, and they happen to appear out of sync, then this function will take care of choosing the most "sane" digest from those nodes, to let you compare it with the "code digest", with no single point of failure. + +If you initiated an undo process earlier, and your database is currently in an "intermediate" state, then the digest loaded will be "lower" than any "code digest", so it will conflict with any code digest you're trying to deploy; you'll need to fix and finish the DB migration before you could proceed with any code deployment. + + +# Ephemeral (Symbol) Fields + +Sometimes we want to pass auxiliary information into [Triggers](/getting-started/triggers), but there is really no field in the Ent schema corresponding to it. We need some temporary place to put the data to, to let the trigger read it and run some additional logic (like another Ent creation or update). + +At the same time, you may want that "temporary place" to be non-optional on inserts. I.e. if you create a new Ent in the database, that piece of information must be treated as required. + +**Ephemeral fields** provide such kind of a storage. + +As an example, let's consider that you want to store EntComment's message encrypted, and in a separate Ent named EntText, with the same ID as EntComment's ID (possibly in a separate DB cluster). You don't want to deal with EntText directly though, and you don't want to let the developer forget about EntText creation as well. You need to incapsulate the encryption logic in EntComment class completely and let the trigger do encryption work on insert/update. + +```typescript +const $MESSAGE = Symbol("$MESSAGE"); + +const schemaComments = new PgSchema( + "comments", + { + id: { type: ID, autoInsert: "nextval('comments_id_seq')" }, + topic_id: { type: ID }, + // This becomes a required and non-nullable ephemeral field. + [$MESSAGE]: { type: String }, + }, + [] +); + +export class EntComment extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + ... + beforeMutation: [ + async (vc, { op, newOrOldRow }) => { + if (op !== "DELETE") { + const text = await encrypt(newOrOldRow[$MESSAGE]); + await EntText.upsertReturning( + vc, + { id: newOrOldRow.id, text }, + ); + } + }, + ], + afterDelete: [ + async (vc, { oldRow }) => { + const text = await EntText.loadNullable(vc, oldRow.id); + await text?.deleteOriginal(); + } + ], + }); + } +} +... +const comment = await EntComment.insertReturning(vc, { + topic_id: "123", + [$MESSAGE]: "Hello", // required property! +}); +``` + +From Ent Framework's point of view, `$MESSAGE` is a regular field: you can provide a type for it in the schema and, if there is no `autoInsert` specified, the property will be required (non-optional). Also, `allowNull` plays its regular role here. + +But since `$MESSAGE` is an ephemeral (symbol) field, it won't be stored in the database. The data is **only available in your triggers**. The analogy on why it won't be stored is simple: + +* When you run `JSON.stringify(obj)`, it skips all of the symbol fields. +* By default, `Object.keys(obj)` also doesn't return symbol keys. + +(The above is just an analogy and a convention of course.) + +TypeScript doesn't let you forget passing `$MESSAGE` field: it will raise an error saying that `$MESSAGE` is a required property of the `insertReturning()` argument. Also, in your triggers, the type of `input[$MESSAGE]` will be `string` and not `string | undefined`, so you can assume that the value is always passed. + + +# Atomic Updates and CAS + +Ent Framework does not expose transactions in its top-level APIs: it's a trade-off made towards supporting automatic queries batching and built-in microsharding. Of course, each individual write to the database is still transactional, but you can't have a notion of "all or nothing" when updating multiple Ents. (Those Ents may also reside in different microshards, so a robust transactional update for them is impossible even in theory.) + +Except when you build a billing solution or a banking app, transactions are rarely needed in practice: probably in less than 1% of cases. Occasionally, you may want to transactionally update multiple Ents still, or (more often) update a single Ent in "read-modify-write" fashion, when it's guaranteed that there are no concurrent writes happened in between. To do this, you have the following options: + +1. Use [PgClient's low level API](/scalability/sharding-low-level-api), `acquireConn()` and `release()`. It exposes the vanilla [node-postgres](https://www.npmjs.com/package/pg) client object, which you can use directly: run transactions, streaming operations etc. You'll have to write raw SQL in this case though. +2. Incapsulate your multi-table update logic in a PostgreSQL stored procedure or in a [PostgreSQL trigger](https://www.postgresql.org/docs/17/sql-createtrigger.html). (In fact, out of the 1% mentioned above, probably 80% can be covered with a native PostgreSQL trigger.) Ent Framework is very friendly to allowing the developer use built-in underlying database's features. The main idea is that both stored functions and triggers are atomic in PostgreSQL, so if you call them, you'll get the transactional behavior without using `BEGIN...COMMIT` statements. Of course, it's possible only for Ents living in the same microshard. +3. For "read-modify-write" cases, use Ent Framework's `$cas` feature which is described below. + +The truth is that, even without full `BEGIN...COMMIT` transactions, it's surprising how far can you go with just having a CAS primitive. (People typically tend to over-use classical transactions.) + +### Compare And Swap ($cas) in update\*() Calls + +{% hint style="info" %} +Strictly speaking, this feature is not a classical "compare and swap": there is no "swap" step, it's more like a "conditional assignment" or "compare and update" pattern. But in the industry, the group of algorithms like that are typically named CAS, so we name it the same way in Ent Framework. +{% endhint %} + +Let's start with a classical read-modify-write code example: + +```typescript +while (true) { + const topic = await EntTopic.loadX(vc, topicID); + const newTags = uniq([...topic.tags, "my-tag"]); + const didUpdate = await topic.updateOriginal({ + tags: newTags, + $cas: { tags: topic.tags }, + }); + if (didUpdate) { + break; + } +} +``` + +Here, we want to append a tag to EntTopic, but we want to be protected against concurrent overwrites: if someone else is adding another tag to the same topic right now, in between our own read and write, we don't want to lose that update. + +Here, `$cas` allows us to perform the update only if the values passed to its properties have not changed **in the database** since our earlier read. If they did in fact change, then we do an in-app retry of the read-modify-write sequence. + +### Short Syntax: Pass Only Field Names + +The pattern above is so common that the code can be shortened: + +```typescript +const didUpdate = await topic.updateOriginal({ + tags: newTags, + $cas: ["tags"], // same as $cas: { tags: topic.tags } +}); +``` + +Here, we tell Ent Framework that it should grab the "original" values for the `$cas` fields from the Ent's properties: in this case, from `topic.tags`. + +### Short Syntax: All Updating Fields + +The practice shows that in the absolute most of the cases, you are updating exactly the same set of fields that you want to protect with `$cas`. It is not mandatory (e.g. you may `$cas` on a timestamp or version field and update way more than that), but often, it's what you want. + +There is a "shorter" syntax for this: + +```typescript +const didUpdate = await topic.updateOriginal({ + tags: newTags, + $cas: "skip-if-someone-else-changed-updating-ent-props", +}); +``` + +Don't worry that `skip-if-someone-else-changed-updating-ent-props` is so intentionally long: it's type-safe (enforced by TypeScript). It is so long just to be explicitly-descriptive. + +### Don't Mix Up $cas and updateChanged()! + +In [Ent API: update\*()](/getting-started/ent-api-update) article we mentioned another call, `updateChanged()`: + +```typescript +const result = topic.updateChanged({ + title: newTitle, +}); +``` + +Be aware that its behavior is very different from what `$cas` feature provides: it does not guarantee read-after-write consistency, it just compares the updating fields with the original values in the Ent (not in the database)! The catch is that in the Ent, the field values may be outdated in comparison what's stored in the database right now. + +Let's compare those 2 calls side by side. + +Without `$cas`: + +```typescript +const result = topic.updateChanged({ + tags: newTags, +}); +``` + +1. Cancels the update in case `newTags === topic.tags` in memory (i.e. may send 0 or 1 SQL query to the database). +2. Does not protect from concurrent changes made by some other Node process in the cluster. +3. After the update, `result` is either an array of actually updated field names (truthy value), `null` if the update got cancelled (no changed fields), or `false` if there is no such Ent in the database anymore. + +And with `$cas`: + +```typescript +const result = topic.updateOriginal({ + tags: newTags, + $cas: "skip-if-someone-else-changed-updating-ent-props", // <-- added $cas +}); +``` + +1. Always sends 1 UPDATE query to the master node, since `$cas` checks the condition on the destination database, not in memory. This means that it almost always doesn't matter, whether the Ent was initially loaded from master or from a replica: in both cases, `$cas` will protect against any improper change. +2. Protects againt concurrent changes made by some other Node process in the cluster. +3. Returns `true` if the update succeeded and `false` in case `$cas` comparison failed or there was no Ent in the database (it's easy to see that both variants can be considered as CAS expectation failures). + +Notice that `updateChanged()` can also work with `$cas`: + +```typescript +const result = topic.updateChanged({ + tags: newTags, + $cas: "skip-if-someone-else-changed-updating-ent-props", // <-- added $cas +}); +``` + +It will give you the mix of both worlds: cancelling the query in case `newTags` is unchanged in comparison to `topic.tags` (which is more a syntax sugar in your code), plus protecting against the concurrent changes. + +## Using $literal in updateOriginal() Call + +There is another way of doing a conflict-free update (like appending to an array field) in Ent Framework: + +```typescript +await topic.updateOriginal(vc, { + $literal: [ + "tags = ARRAY(SELECT DISTINCT unnest FROM unnest(array_append(tags, ?)))", + "my-tag", + ] +}); +``` + +Here, we use `$literal` feature that enables you to pass a raw SQL expression, so the query will look like: + +```sql +UPDATE topics +SET tags = ARRAY(SELECT DISTINCT unnest FROM unnest(array_append(tags, 'my-tag'))) +WHERE id = 1004200047373526525 +``` + +There are several downsides in this approach though: + +1. Calls of this kind can't be batched, so if you run multiple of them in parallel, Ent Framework will send independent queries. +2. It is engine-specific and uses PostgreSQL stored functions under the hood. + + +# Custom Field Refactoring + +In [Custom Field Types](/getting-started/custom-field-types) article, we discussed, how you can add Ent fields of an arbitrary shape to your Ent class. + +You also learned, how easy it is to modify the custom type when you add new properties. + +Although adding optional and required properties to custom types covers the absolute most of cases, sometimes we want to do a larger refactoring, changing the *shape* of the data *entirely*. It's harder to do, since you need to deal with both the old DB format and the new format at all times (unless you want to rewrite all the rows in your database). + +There are some best practices still, and TypeScript helps here a lot. + +## Backward/Forward Compatibility Aspects + +When modifying custom types, it's crucial to think about the database schema migration and backward compatibility aspects, especially when you add non-optional properties to your type, or when you change inner types of the properties. + +The hardest thing here is that you need to care not only about backward compatibility (when you must be ready to read the old data format from the existing database rows), but also about forward compatibility (i.e. be ready to **write** the data in an old format), because there may still be the readers in the cluster running the old code and expecting the old data format. + +Let's get back to the type which we defined previously: + +```typescript +type ActorsV1 = { + editor_ids: string[]; + viewer_ids: string[]; +}; +... +const topic = await EntTopic.insertReturning(vc, { + ..., + actors: { editor_ids: ["42"], viewer_ids: [] }, +}); +``` + +Here, we stored a row to the database, so it remains there: + +``` +ROW(id="123", ..., actors='{"editor_ids":["42"],"viewer_ids":[]}') +``` + +Imagine now that we want to significantly change the type: instead of storing just user IDs, we also want to store the timestamps when those users performed an action last time: + +```typescript +type Actors = { + editor_ids: Array<{ id: string; ts: number; }>; + viewer_ids: Array<{ id: string; ts: number; }>; +}; +``` + +## Deployment 1: New Format in Code, Old Format in Database + +As a preliminary step, we need to rename `Actors` to `ActorsV1`, to declare it as an "old data format". This, newest format that we'll introduce will always be named as just `Actors`. + +To transition between the custom type formats, we then need to update the code to let it work with `Actors`. But the code must still **write** the data in the old `ActorsV1` format: the deployment is not an immediate process, so there are periods of time when Node processes with the new code and Node processes with the old code run at the same time. + +```typescript +function typecheck(v: T): T { + return v; +} + +const ActorsType = { + // Accepts BOTH the old format and the new format. Returns new format. + dbValueToJs(obj: ActorsV1 | Actors): Actors { + return { + editor_ids: obj.editor_ids.map( + (v) => typeof v === "string" ? { id: v, ts: Date.now() } : v, + ), + ... + }; + }, + + // Accepts only new format. Stringifies to the old format. + stringify(obj: Actors): string { + return JSON.stringify(typecheck({ + editor_ids: obj.editor_ids.map((v) => v.id), + ... + })); + }, + + // Auxiliary counter-part to stringify(). + parse(v: string): Actors { + return this.dbValueToJs(JSON.parse(v)); + }, +} +``` + +The idea is following: + +1. In our code, we always work with the new format, `Actors`. +2. When writing to the database, we use the old format, `ActorsV1`. +3. When reading from the database, we are able to recognize both the old format `ActorsV1` and the new format `Actors`. This behavior will remain with us forever, becuse we'll keep having the data stored in the database in old format. + +Notice how much TypeScript does help us here: it ensures that we won't return nor accept a mismatched type in both `dbValueToJs()` and `stringify()` (try returning some different shape, and you'll see a compile-time error): + +* `dbValueToJs(obj: ActorsV1 | Actors)` allows us to work with a union type, which is safer than working with e.g. `any`. +* `return JSON.stringify(typecheck({ ... }))` doesn't let us to return data in a wrong format and ensures that it conforms the `ActorsV1` shape. + +This change in the code needs to be deployed, and we must be sure that there is no old code running anywhere before continuing. + +## Deployment 2: New Format in Writes, Ability to Read Old Format Still + +Once we're sure that the code can read both the old data format `ActorsV1` and the new format `Actors`, we can proceed with the 2nd step: switch to writing the new data in the new format. We can do so, because there are no old readers in the cluster anymore. + +The final permanent code will be: + +```typescript +function typecheck(v: T): T { + return v; +} + +// Internal to this file (not exported). +type ActorsV1 = { + editor_ids: string[]; + viewer_ids: string[]; +}; + +export type Actors = { + editor_ids: Array<{ id: string; ts: number; }>; + viewer_ids: Array<{ id: string; ts: number; }>; +}; + +export const ActorsType = { + // Accepts BOTH the old format and the new format. + // This code about ActorsV1 will remain here forever. + dbValueToJs(obj: ActorsV1 | Actors): Actors { + return { + editor_ids: obj.editor_ids.map( + (v) => typeof v === "string" ? { id: v, ts: Date.now() } : v, + ), + ... + }; + }, + + // Accepts only new format. Stringifies to the new format. + stringify(obj: Actors): string { + return JSON.stringify(obj); + }, + + // Auxiliary counter-part to stringify(). + parse(v: string): Actors { + return this.dbValueToJs(JSON.parse(v)); + }, +} +``` + +In the future, if we need to change the format one more time in an incompatible way, we'll need to introduce `ActorsV2` (as an initial copy of `Actors`) and do 2 deployments again. + + +# VC Flavors + +VC (stands for "Viewer Context") is one of Ent Framework's core abstractions. As described in [VC: Viewer Context and Principal](/getting-started/vc-viewer-context-and-principal) article, it represents an "acting user". More precisely, it is actually an "acting principal", since it may not necessarily be a user: for e.g. background jobs, people often use other "owning" objects, like a company or a workspace, depending on the app's business logic. + +## VC Principal + +Early in a request cycle, you create an instance of VC and then use it everywhere else in the code to load Ents: + +```typescript +// Early in your request processing lifecycle: +const guestVC = VC.createGuestPleaseDoNotUseCreationPointsMustBeLimited(); +const user = await EntUser.loadX(guestVC.toOmniDangerous(), { + email: session.user.email, +}); + +// Every Ent carries the VC that was used to load it. In case +// we used an omni VC, then it is "downgraded" to a "less +// powerfull" VC right after loading; see privacyInferPrincipal +// configuration option on Ent classes. +vc = user.vc; + +// Later in all other code: +const user = await EntUser.loadX(vc, vc.principal); +const topic = await EntTopic.loadX(user.vc, topicID); +const comments = await EntComment.select(topic.vc, ...); +``` + +Every VC instance has `principal` property, a raw string that identifies, who's acting. Here are some common values for it: + +1. `"10042000123456789"`, i.e. some Ent's ID: used in absolute most of the cases (like user ID or company ID). It is more a convention rather than a rule though. +2. `"omni"`: if you call `vc.toOmniDangerous()`, the returned VC will have that value in its `principal` property. (The original VC remains immutable.) Omni VCs bypass all privacy rules. +3. `"guest"`: such VC is created by `vc.toGuest()` call or with `createGuestPleaseDoNotUseCreationPointsMustBeLimited()` static method. It cannot load or update anything by default, unless explicitly allowed with e.g. `AllowIf(new True())` privacy rule. + +When you want to get s VC with particular principal in your code, you typically *derive* it from some existing VC by using the methods mentioned above. This enables keeping the knowledge about the derivation chain. + +## Flavors + +In addition to `vc.principal` property, it is often times convenient to store some auxiliary information in a VC. You can do it by adding *flavors*, instances of classes derived from `VCFlavor`: + +```typescript +/** + * A flavor that carries an auxiliary email. + */ +export class VCEmail extends VCFlavor { + constructor(public readonly value: string) { + super(); + } + + override toDebugString() { + return this.value; + } +} + +/** + * A flag-like flavor that enables reading of soft-deleted + * Ents (e.g. Ents with deleted_at set to non-null). + */ +export class VCCanReadSoftDeletedEnts extends VCFlavor { + override toDebugString() { + return "read-soft-deleted"; + } +} + +/** + * A flag-like flavor that our app may check to allow reading + * or writing of any Ent. + */ +export class VCAdmin extends VCFlavor { + override toDebugString() { + return "admin"; + } +} +``` + +Typically, you store any arbitrary properties in your flavor instance and then derive a new VC by attaching the flavor: + +```typescript +const derivedVC = vc.withFlavor( + new VCEmail("test@example.com"), + new VCCanReadSoftDeletedEnts(), +); +const topic = EntTopic.loadX(derivedVC, softDeletedTopicID); +``` + +You can then read the flavor back in your code (e.g. in privacy rule predicates) to make decisions: + +```typescript +import { VCHasFlavor } from "ent-framework"; +... +privacyLoad: [ + new AllowIf(new VCHasFlavor(VCAdmin)), + new AllowIf(async function CanReadSoftDeletedEnt(vc, row) { + const flavor = vc.flavor(VCCanReadSoftDeletedEnts); + return row.deleted_at !== null && flavor !== null; + }), + ... +], +... +``` + +Notice that `vc.flavor(Class)` returns an instance of `Class` flavor associated with the VC, or `null` if there was no such flavor attached. + +## VC#toString() and Flavors + +Each class derived from `VCFlavor` may have a `toDebugString()` method overridden. When you call `vc.toString()` or `vc.toAnnotation()` , all the flavors in the VC are enumerated, and the values returned by `toDebugString()` are glued together, so the final result looks like: + +```typescript +console.log(derivedVC.toString()); +console.log(derivedVC.toAnnotation().vc); +// Both print: +// vc:10042000123456789(test@example.com,read-soft-deleted) +``` + +This is extremely convenient: in your query logs, you likely save the result of `vc.toAnnotation()`, so with e.g. `VCEmail`, you immediately see, who is sending the queries. + +## Example: Attaching Flavors in a Next App + +In [VC: Viewer Context and Principal](/getting-started/vc-viewer-context-and-principal) article, we provided the code for `getServerVC()` helper function that can be used in a Next app to derive the request VC. Let's amend it to include `VCEmail` helper flavor. + +```typescript +import { VC } from "ent-framework"; +import { getServerSession } from "next-auth"; +import { headers } from "next/headers"; +import { EntUser } from "./EntUser"; + +const vcStore = new WeakMap(); + +export async function getServerVC(): Promise { + const [heads, session] = await Promise.all([headers(), getServerSession()]); + let vc = vcStore.get(heads); + if (!vc) { + vc = VC.createGuestPleaseDoNotUseCreationPointsMustBeLimited(); + if (session?.user?.email) { + const vcOmni = vc.toOmniDangerous(); + let user = await EntUser.loadByNullable(vcOmni, { + email: session.user.email, + }); + if (!user) { + // User did not exist: upsert the Ent. + await EntUser.insertIfNotExists(vcOmni, { + email: session.user.email, + is_admin: false, + }); + user = await EntUser.loadByX(vcOmni, { + email: session.user.email, + }); + } + // Thanks to EntUser's privacyInferPrincipal rule, user.vc is + // automatically assigned to a new derived VC with principal + // equals to user.id. We also attach flavors here. + vc = user.vc.withFlavor( + new VCEmail(user.email), + user.is_admin ? new VCAdmin() : undefined, + ); + } + vcStore.set(heads, vc); + } + return vc; +} +``` + +## Flavors and Security + +Flavors engine is not limited to auxiliary or debug purposes only: it may also be used on the app's privacy checking critical path. + +E.g. a flavor can be used as a *proof of identity*. In all previous examples, we used `vc.toOmniDangerous()` to load the very first EntUser in our request lifecycle, to avoid the "chicken and an egg" problem ("to load a user, you need a VC that can do it, and to derive that VC, you need an EntUser instance loaded"). Once the above is done, we wrote the user's ID to `vc.principal` and then *assumed* that the VC is allowed to behave on behalf of that user, fully trusting the value in `vc.principal`. + +It is not the only way to create the initial acting VC though. Ask yourself: what kind of *proof* do we need to load an arbitrary EntUser? How does the backend do it naturally? The answer is that you must have some kind of a *secret* in hands, like the user's password salted hash, or the user's token stored in a cookie, or an OAuth2 token. If you put that "proof" in a favor, then you can use it in EntUser's privacy rules to unlock the loading without ever calling to `vc.toOmniDangerous()`: + +```typescript +class VCIdentityProof extends VCFlavor { + #cookieToken: string; + + constructor( + public readonly email: string, + cookieToken: string, + ) { + super(); + this.#cookieToken = cookieToken; + } + + override toDebugString() { + // Do NOT expose #cookieToken! + return this.email; + } +} + +class EntUser extends ... { + ... + privacyLoad: [ + new AllowIf(async function HasValidIdentityProof(vc, row) { + const flavor = vc.flavor(VCIdentityProof); + return row.email === flavor.email && + row.cookie_token === flavor.cookieToken; + }), + ... + ], + ... +} +``` + +So instead of using `toOmniDangerous()` in your initialization code, you may just attach the proof of identity flavor to a VC: + +```typescript +// Pseudo-code: +const cookieStore = await cookies(); +const guestVC = VC.createGuestPleaseDoNotUseCreationPointsMustBeLimited() + .withFlavor(new VCIdentityProof(email, cookieStore.get("token"))); +const user = await EntUser.loadByX(vc, { email }); +return user.vc; +``` + +If you want even more or security, you may store a HMAC of the cookie token in `VCIdentityProof` flavor instead of the token itself, and then use *HMAC verification* instead of `===` operator. In that case, even if the flavor payload is leaked, you'll face no harm. + +Then, in privacy rules of the rest of your Ents, you delegate checking to the privacy rules of the parent EntUser (or of a parent Ent, considering that it delegates the checks to its owning EntUser). I.e. proceed with utilizing the standard privacy chain supported by Ent Framework. + +A slight downside of this approach is that you'll always be having `vc.principal` equal to `"guest"` in this case, but it also makes sense: until "a guest" really "proves" that the VC has permissions to load an EntUser, it can't load the Ent. + + +# Query Cache and VC Caches + +Ent Framework supports *in-VC LRU query caching* for all read API calls (like `loadX()`, `loadBy*()`, `select()` etc.): + +```typescript +// Somewhere in early stage of the request lifecycle: +// enable Ent query caching. +const vc = user.vc.withFlavor(new VCWithQueryCache({ + maxQueries: 1000, +})); + +const topic1 = await EntTopic.loadX(vc, topicID); +const topic2 = await EntTopic.loadX(vc, topicID); // no DB queries sent! +``` + +By default, the cache is not enabled: to activate it, add a VC flavor `VCWithQueryCache` early in the request lifecycle. + +* Once enabled, all of the read results will be saved in an internal store associated with the VC. +* Since VC is immutable, the store is not inherited when you derive a VC from the current VC, e.g. with `toOmniDangerous()` or `withFlavor()`. I.e. the new VC will appear with empty caches. +* Every write (like `insert*()`, `update*()` or `deleteOriginal()` calls) will also invalidate the caches. Ent Framework tries to do it intelligently: if you e.g. update a particular Ent, only the `load*()` cache related to the same ID will be cleaned. +* Writes happened in one VC do not affect caches stored in other VCs (even derived ones). Be careful. +* You can create a derived VC with empty caches and no other changes, by using `newVC = vc.withEmptyCache()`. + +Overall, query caching works the way you expect it to work. As a concept, reading through the cache is sligtly similar to reading from a replica DB (see [Replication and Automatic Lag Tracking](/scalability/replication-and-automatic-lag-tracking)): you may get the stale data, but Ent Framework does its best to prevent that when possible. + +Notice that the VC caches are very short-lived: they are not stored externally (no files, no Redis etc.) and exist in Node process memory only. + +## Enable Query Cache in a Next App + +Earlier in [VC Flavors](/advanced/vc-flavors), we updated our `getServerVC()` function example to attach additional flavors to the per-request VC. Let's modify it further to enable query caching. + +```typescript +import { VC } from "ent-framework"; +import { getServerSession } from "next-auth"; +import { headers } from "next/headers"; +import { EntUser } from "./EntUser"; + +const vcStore = new WeakMap(); + +export async function getServerVC(): Promise { + const [heads, session] = await Promise.all([headers(), getServerSession()]); + let vc = vcStore.get(heads); + if (!vc) { + vc = VC.createGuestPleaseDoNotUseCreationPointsMustBeLimited(); + if (session?.user?.email) { + const vcOmni = vc.toOmniDangerous(); + let user = await EntUser.loadByNullable(vcOmni, { + email: session.user.email, + }); + if (!user) { + // User did not exist: upsert the Ent. + await EntUser.insertIfNotExists(vcOmni, { + email: session.user.email, + is_admin: false, + }); + user = await EntUser.loadByX(vcOmni, { + email: session.user.email, + }); + } + // Thanks to EntUser's privacyInferPrincipal rule, user.vc is + // automatically assigned to a new derived VC with principal + // equals to user.id. We also attach flavors here and enable + // the built-in query caching. + vc = user.vc.withFlavor( + new VCWithQueryCache({ maxQueries: 1000 }), // <-- + new VCEmail(user.email), + user.is_admin ? new VCAdmin() : undefined, + ); + } + vcStore.set(heads, vc); + } + return vc; +} +``` + +## Custom Caches + +In addition to built-in query caching, you may utilize your own in-VC caches with `VC#cache()` API. This is convenient when Ent Framework's built-in capabilities are not enough, or you want to cache the data related to other databases. + +First, define the *cache store* for your use case. Often times, the simplest way is to just extend the JS built-in `Map` class, but you can use any other store (like [quick-lru](https://www.npmjs.com/package/quick-lru)) or even implement your own store the way you want. + +```typescript +export class MyStore extends Map { + constructor(private vc: VC) { + super(); + } +} +``` + +Once you have a store class with a constructor that accepts a VC insrance, you can use it with `cache()` API: + +```typescript +const store = vc.cache(MyStore); +if (!store.has(myKey)) { + store.set(myKey, myValue); +} +return store.get(myKey)!; +``` + +When you call `vc.cache(MyStore)` the very 1st time for the VC, Ent Framework will create an instance of `MyStore` class and save it in the VC itself. Next time you run `vc.cache(MyStore)`, it will find that store instance and return it to you. + +The store class itself plays the role of the store identification within the VC. So if you have 2 independent store classes, they will not collide. + +If you don't like classes, use a special *tagged* version of `cache()` call: + +```typescript +const $MY_STORE = Symbol("$MY_STORE"); +... +myMethod() { + const store = vc.cache( + $MY_STORE, + () => new Map(), + ); + if (!store.has(myKey)) { + store.set(myKey, myValue); + } + return store.get(myKey)!; +} +``` + +Here, `$MY_STORE` symbol will play the same identification role as `MyStore` class itself in the previous example. + +Overall, `cache()` call does nothing more than "memoizing" an instance of your store container in a particular VC. It's up to you, how to utilize that store instance, be it a key-value container or something else. + +## Privacy Rules Caching + +At this point, it won't be a surprise for you that Ent Framework privacy checking layer (see [Privacy Rules](/getting-started/privacy-rules)) uses the VC caching engine described above. In particular, once some Ent ID is determined to be readable in a VC (`privacyLoad` rules), then all future checks within that VC are bypassed. The same applies to `privacyUpdate` and `privacyDelete`. Since Ents and VCs are immutable, we can safely rely on that machinery. + +In practice, the caching for privacy rules works quite effectively: you'll rarely see too many additional database requests that Ent Framework issues for privacy checking. + +Privacy caching also enables one interesting feature: if you, say, load an Ent in a VC and then soft-delete it (by setting its `deleted_at` field to the current time or to `true`, depending on your business logic), then you will still be able to reload that Ent in the same VC, even if its privacy rules block reading of soft-deleted rows. This is because when you read an Ent, you have already "proven" that you have access to it, so Ent Framework will bypass all the further checks related to the same VC. + + +# Loaders and Custom Batching + +One of the key features of Ent Framework is a holistic [N+1 Selects Solution](/getting-started/n+1-selects-solution). When you address a single row in the database (read or write), the engine batches that calls into compound SQL queries, which allows to save a lot on round trip time. + +The core of that idea lies in Meta's [DataLoader](https://github.com/graphql/dataloader) pattern, which initially was invented for just one case: loading an object by its ID. Ent Framework generalizes DataLoader to *all* read and write operations. + +In Ent Framework, this layer of abstraction is called Loader. It's an advanced concept. You'll rarely need to create your own Loaders, but once you do, your Loader will inevitably appear on the critical path of your app, since it's a performance optimization pattern. + +## Node Event Loop + +Event loop, Promises and I/O in Node are complicated topics with many nuances, best described in the [official documentation](https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick). + +Here, we'll only give a rough and inprecise overview, enough to understand the Loader abstraction better. + +Event loop consists of multiple "phases", each phase is a sequence of Macrotasks, and after each macrotask, all pending Microtasks are executed. + +1. In every phase, Node first picks the oldest pending macrotask (like callbacks waiting for I/O results, timers etc.). +2. Then, once the macrotask finishes, it run the pending microtasks (like `Promise#then()` invocations or callbacks scheduled with `process.nextTick()`), until there are no more microtasks pending. +3. Microtasks may schedule new macrotasks (for the next phase or the next spin) of new microtasks (for the same phase), and then eventually it all starts from the beginning. + +The main idea behind Loader is to accumulate all Ent Framework calls (like `loadX()`, `insert()` etc.) within one microtasks block, group them together and then flush as one large SQL query towards the resolution in the next macrotask I/O processing. Only the calls of the same type are batched together (e.g. load with load, insert with insert); the calls of different types relate to independent I/O macrotasks, and thus, almost always resolve in different microtask blocks. + +
+ +## Simple Loader Example + +Loader class allows you to build your own batching logic, for the cases when Ent Framework internal batching is not enough. + +Let's first build a very simple Loader, similar to what the built-in `loadNullable()` uses internally. + +```typescript +class TopicSimpleLoader { + private ids = new Set(); + private results = new Map(); + + constructor(private vc: VC) {} + + onCollect(id: string): void { + this.ids.add(id); + } + + async onFlush(): Promise { + const topics = await EntTopic.select( + this.vc, + { id: [...this.ids] }, + Number.MAX_SAFE_INTEGER, // limit + ); + for (const topics of topic) { + this.results.set(topic.id, topic); + } + } + + onReturn(id: string): EntTopic | null { + return this.results.get(id) ?? null; + } +} +``` + +The main beauty of Loaders is that your code still looks like you're working with single objects (or sincle IDs), not with lists: + +```typescript +async function getTopic(vc: VC, id: string) { + const topic = await vc.loader(TopicSimpleLoader).load(id); + return topic; +} +... +// The following calls will be batched into 1 SELECT query. +await mapJoin([id1, id2, ...], async (id) => getTopic(vc, id)); +``` + +I.e. Loader is that exact abstraction that allows you to write a "single-object" code and have free I/O batching under the hood. + +Each Loader is a class with at least the following methods: + +* `onCollect(arg1, arg2, ...)`: it's simply called when you run `vc.loader(MyLoader).load(arg1, arg2, ...)`. Your goal here is to accumulate all of the incoming requests in some private property (typically, in a Set or in a Map). +* `onFlush()`: this method is called in the end of microtasks block on the diagram above. By that time, you can assume that all of the incoming requests are accumulated already. So you build a final batched query, read its response and save it to another private property (typically, a Map, where keys are those `arg1`, `arg2` etc. that we used above. +* `onReturn(arg1, arg2, ...)`: it's called right before `vc.loader(MyLoader).load(arg1, arg2, ...)` returns in the caller's code. Here, you just read from your accumulated results and return the value to the client. +* Also, you may defined a constructor, to receive and store a VC. VC is passed to each Loader, for the cases when your `onFlush()` logic requires it. (Your Loader may work with any other I/O service, not necessarily with Ent Framework. E.g. you may read from Redis or DynamoDB directly.) + +So, you can see that the arguments type of `onCollect()` and `onReturn()` methods become the argument types of \``` .load(..)` `` exactly, and the return type of `onReturn()` becomes the return type of `.load()`. The engine uses TypeScript inference, and it will warn you in case some types mismatch somewhere. + +Overall, Loader is not a rocket science: it just splits the lifetime of `.load()` call into 2 phases: `onCollect` and `onReturn`, effectively deferring the response to the caller up to the moment when `onFlush` has a chance to trigger. From the point of view of the caller code, imagine it as a short interruption between calling `.load()` and getting the results back. + +## Real Life Loader Example + +Once you understand, how the above code works, we can move on to a more realistic example. + +Imagine that in your `topics` table, you have `tags text[]` field, which is an array of strings: + +```sql +CREATE TABLE topics( + id bigserial PRIMARY KEY, + tags text[] NOT NULL DEFAULT '{}', + slug varchar(64) NOT NULL UNIQUE, + creator_id bigint NOT NULL, + subject text DEFAULT NULL +); +CREATE INDEX topics_tags ON topics USING gin(tags); +``` + +PostgreSQL has reach support for arrays, JSON and GIN indexes to quickly SELECT from the fields of such types, so no surprise we want to use this little denormalization here. + +In your app utility library API, you often times query for topics with one particular tag: + +```typescript +async function loadTopicsByTag(vc: VC, tag: string) { + return EntTopic.select(vc, { $literal: ["? = ANY(tags)", tag] }, 100); +} +``` + +You prevent an engineer from abusing this API though: + +```typescript +const topicGroups = await mapJoin( + ["tag1", "tag2", ...100 other tags], + async (tag) => loadTopicsByTag(vc, tag), +); +``` + +If someone runs this, then Ent Framework will build a large UNION ALL clause with individual SELECTs, which will be far from efficient. I.e. we need a better batching strategy for this particular case. + +Let's build a Loader: + +```typescript +class TopicsTagLoader { + private tags = new Set(); + private results = new Map(); + + constructor(private vc: VC) {} + + onCollect(tag: string): void { + this.tags.add(tag); + } + + async onFlush(): Promise { + const topics = await EntTopic.select( + this.vc, + { tags: { $overlap: [...this.tags] } }, + Number.MAX_SAFE_INTEGER, // limit + ); + for (const topics of topic) { + for (const tag of topics.tags) { + if (!this.results.has(tag)) { + this.results.set(tag, []); + } + this.results.get(tag)!.push(topic); + } + } + } + + onReturn(tag: string): EntTopic[] { + return this.results.get(tag) ?? []; + } +} +``` + +Now, you can rewrite `loadTopicsByTag()`, so anyone can use it without thinking about parallel calls: + +```typescript +async function loadTopicsByTag(vc: VC, tag: string) { + return vc.loader(TopicsTagLoader).load(tag); +} + +// Now, this works well, only one SQL query: +const topicGroups = await mapJoin( + ["tag1", "tag2", ...100 other tags], + async (tag) => loadTopicsByTag(vc, tag), +); +``` + +## Loader for INSERT/UPDATE/DELETE Queries + +The main purpose of Loader is to do bathing for individual single-row queries, so they work perfectly not only for reads from the database, but also for writes. This abstraction is agnostic on the type of the operation: the only requirement is that it must be *idempotent*: calling it once is no different from calling it several times successively. + +## Loader for Other Databases + +Overall, Ent Framework already has good enough internal batching mechanism, so you won't use Loaders for your man database frequently. + +More often, Loader is useful for external API calls or for querying the external databases. + +Assume that you have `viewCount()` method in EntTopic which queries Redis for the counter value. There is also a `render()` method that returns a text representation of the topic with comments. + +```typescript +class EntTopic extents BaseEnt(...) { + ... + async viewCount() { + const count = redis.get(this.id); + return parseInt(count) || 0; + } + + async render() { + const [viewCount, comments] = await Promise.all([ + this.viewCount(), + EntComment.select(this.vc, { topic_id: this.id }, 10), + ]); + const commentWidgets = await mapJoin( + comment, + async (comment) => comment.render(), + ); + return `${topic.title}: ${viewCount}\n` + commentWidgets.join("\n"); + } +} + +class EntComment extents BaseEnt(...) { + ... + async render() { + const creator = await EntUser.loadX(this.vc, this.author_id); + return `${creator.name}: ${comment.message}`; + } +} +``` + +And then in your code, you have the following logic: + +```typescript +const topics = await EntTopic.select(vc, {}, 10, [{ created_at: "DESC" }); +const widgets = await mapJoin(topics, async (topic) => topic.render()); +``` + +In [#node-event-loop](#node-event-loop "mention") section above, we discussed, how Ent Framework batching works together with Node event loop machinery. **If only `viewCount()` was querying the counter from Ent Framework as well**, then we'd have just 4 queries to the database: + +

If only viewCount() was running against the main database, not Redis...

+ +On the picture above, the horizontal line denotes the "spin" of event loop (more precisely, it's a barrier between one I/O macrotask and another). We can see that `view_counts` and `comments` queries run in parallel, and after they both resolve, the batched `users` query starts to run. I.e. batching for `users` works fine: no matter how many comments there are, there will be just one SQL query to that table. + +But the reality is that `viewCount()` calls into Redis, and Redis client doesn't support query batching by default (it uses *pipelining*, a different concept). Thus, individual `redis.get()` calls will resolve independently on each other, in different macrotasks, so the sequence of "event loop spins" (aka I/O macrotasks) will be this: + +

redis.get() calls resolbe in different macrotasks...

+ +So, Ent Framework batching got completely broken: since tens of `redis.get()` invocations resolve in different macrotasks, the consequent `EntUser.loadX()` calls are also scheduled independently: the engine can only batch the calls issued within the same macrotask. + +To fix it, we need to make sure that all high-frequent calls to other I/O subsystems and databases (like Redis) pass through some Loader. I.e., you may want to build a `RedisGetLoader` or some other similar abstraction, turning multiple Redis [GET](https://redis.io/docs/latest/commands/get/) calls into one [MGET](https://redis.io/docs/latest/commands/mget/) call. Typically, it's even better for performance: you get batching in the places where you used to send individual queries. + + +# PostgreSQL Specific Features + +Earlier we described the engine-independent features of [select() API call](/getting-started/ent-api-select-by-expression). + +The default `select()` call (as all other Ent API calls) is generic and engine independent, be it PostgreSQL or any other database. In addition to that, `select()` allows to pass the last optional engine-specific argument, to let you use the most of PostgreSQL features without falling back to vanilla SQL queries prematurely. + +```typescript +const comments = await EntComment.select( + vc, + { creator_id: "101" }, + 20, + undefined, // order + custom, // untyped, but of type SelectInputCustom +); +``` + +Despite the last optional parameter is an untyped object, it in fact accepts the following structure: + +```typescript +type SelectInputCustom ={ + ctes?: Literal[]; + joins?: Literal[]; + from?: Literal; + hints?: Record; +} +``` + +An artificial usage example: + +```typescript +const comments = await EntComment.select( + vc, + { created_at: { $gt: yesterdayDate } }, + 10, + [{ created_at: "DESC" }], + { + // WITH clauses (Common Table Expressions). + ctes: [ + [ + "recent_topics AS (SELECT * FROM topics WHERE created_at > ?)", + yesterdayDate, + ], + [ + "recent_comments AS (SELECT * FROM comments WHERE created_at > ?)", + yesterdayDate, + ], + ], + // A replacement for the entire FROM clause. + from: ["recent_comments"], + // Clauses after FROM. + joins: [ + [ + "JOIN recent_topics t ON t.id = topic_id AND comment_count > ?", + minComments, + ], + ], + // Parameters like enable_seqscan, enable_bitmapscan etc. + hints: { + enable_seqscan: "off", + } +); +``` + +Of course, this all works only within one microshard. You can't use JOINS or WITH statements targeting different microshards. + +Continue reading: [Query Planner Hints](/advanced/query-planner-hints). + + +# Query Planner Hints + +Another PostgreSQL specific feature (probably the most popular one among the [other custom options](/advanced/postgresql-specific-features)) is giving the PostgreSQL planner some hints on how you prefer the query to be executed. + +## GUC (Grand Unified Configuration) Settings + +E.g. if your table contains data with very different cardinality of a particular field, and you see that PostgreSQL runs a seqscan sometimes, you may try to lower the seqscan priority: + +```typescript +const comments = await EntComment.select( + vc, + { topic_id: topicID }, + 100, + [{ created_at: "DESC" }], + { hints: { enable_seqscan: "off" } }, +); +``` + +The planner settings like [enable\_seqscan](https://www.postgresql.org/docs/current/runtime-config-query.html#GUC-ENABLE-SEQSCAN) do not fully prevent sequential scan, but they greatly reduce its probability, in case you know what you're doing. + +If your query is intentionally a sequential scan or just works with lots of rows in the table, does [JOINs uses WITH custom clauses](/advanced/postgresql-specific-features), then you may also want to increase `work_mem` for it, to lower the chance of using temporary files: + +```typescript +const comments = await EntComment.select( + vc, + { topic_id: topicID }, + 100, + [{ created_at: "DESC" }], + { hints: { work_mem: "100MB" } }, +); +``` + +Of course, Ent Framework changes the GUC settings during the query execution period only, and guarantees that they are restored after. Internally, `SET LOCAL ... RESET` are used for that (running within the same transaction of the multi-query). + +Also, batching is done with respect to the planner hints. Only the SELECT queries having the same set of hints are potentially batched together with `UNION ALL` clause, as explained in [Ent API: select() by Expression](/getting-started/ent-api-select-by-expression) article. + +See [PostgreSQL documentation](https://www.postgresql.org/docs/current/runtime-config.html) for the full list of settings you can customize per query (notice that not all of them allow per-query changing: some require a database restart). + +## Using pg\_hint\_plan Extension + +There is a special hint with `""` (empty string) name. It allows to raw-prepend an arbitrary string in front of the query that Ent Framework sends to the database. + +The raw-prepend hint is useful when e.g. working with PostgreSQL extensions like [pg\_hint\_plan](https://github.com/ossc-db/pg_hint_plan). This extension enables a **way** greater level of planning customization. You can even tell PostgreSQL, which exact index it must use for your query: + +```typescript +const comments = await EntComment.select( + vc, + { topic_id: topicID }, + 100, + [{ created_at: "DESC" }], + { + hints: { + [""]: "/*+IndexScan(comments comments_created_at_idx)*/", + work_mem: "10MB", + statement_timeout: "20s", + }, + }, +); +``` + +The resulting SQL multi-query (one transaction) sent to the server will then look like: + +```sql +/*+IndexScan(comments comments_created_at_idx)*/ +SET LOCAL search_path TO sh0123; +SET LOCAL work_mem TO 10MB; +SET LOCAL statement_timeout TO 20s; +SELECT ... FROM comments WHERE topic_id=? + ORDER BY created_at DESC LIMIT 100; +RESET statement_timeout; +RESET work_mem; +SELECT pg_last_wal_replay_lsn(); +``` + +Since it's a multi-query, there will be only one round-trip to the server and one transaction. + +The use of raw-prepend hint turns off Ent Framework's SELECT query batching. I.e. your query is guaranteed to run alone, not in a `UNION ALL` construction. + + +# Cluster Maintenance Queries + +During its work, Ent Framework cluster runs various queries in background, invisible to the user. + +## Connectons Prewarming + +In [Connect to a Database](/getting-started/connect-to-a-database) article, we briefly mentioned `cluster.prewarm()` call. The goal of this feature is to let Ent Framework keep the minimal number of open database connections per each client pool (e.g. per each `PgClient`). Having ready open connections means that the queries can be processed immediately (connection establishment is expensive, especially when it uses SSL and/or a proxy solution like pgbouncer). + +Prewarming is done by sending a simple `SELECT 1` SQL query to the pool from time to time (`PgClientOptions#prewarmIntervalMs`, defaults to 5 seconds), in controlled parallel bursts. You can customize the query Ent Framework sends with `PgClientOptions#prewarmQuery` property (to e.g. prewarm full-text search dictionaries if you use them). + +When Ent Framework boots, it does not start sending all those prewarm queries immediately. Instead, it waits for a random time period (passed as a parameter to `prewarm()` method). And, Ent Framework first prewarms 1 connection, then 2, then 3 etc. until it reaches the `min` value passed to the client's config. + +Notice that the default value of `prewarmIntervalMs` is chosen intentionally: adding its jitter `prewarmIntervalJitter` (which is +0.5x), the interval is lower than node-postgres'es `idleTimeoutMillis` value that is 10 seconds by default. Thus, Ent Framework is able to keep the desired number of open connections and not risk them being closed on an idle condition. + +## Cluster Islands Reconfiguration + +From time to time (`ClusterOptions#reloadIslandsIntervalMs`, defaults to 500 ms if it is sync function or 10 seconds if it's an async function), Ent Frameworks calls `islands()` callback from Cluster object options, to figure out whether some new islands or nodes were added to the cluster, or some nodes were removed. This allows you to dynamically change the cluster configuration without restarting Node process. + +You may also use an async function for `islands()`: in this case, Ent Framework will take care of proper caching and error handling. If the callback starts failing, it won't cause any downtime: instead, it will be retried, until it succeeds (and the Cluster's configuration will remain unchanged). If your function fails at the very 1st attempt, right after the process boot, then the error will be propagated back to the client code. + +## Shards Rediscovery + +From time to time (`CusterOptions#shardsDiscoverIntervalMs`, defaults to 10 seconds), Ent Framework polls all of the Cluster nodes to get the list of active microshards on those nodes. + +It also runs such a poll immediately in the following rare cases: + +1. When the very 1st query arrives, and there is no yet info about the microshards in the cluster. +2. When a query fails with "table not found" exception. It is often times the case when a microshard has just [moved](/scalability/shards-rebalancing-and-pg-microsharding-tool) from one island to another, so the cluster needs to be rediscovered. + +## Jitter + +All of the periodic maintenance operations are done with slightly different and randomized time intervals between them. This approach is called "jitter". + +Imagine that you have, say, 500 Node processes in the cluster running Ent Framework, and you boot all 500 processes at the same time when deploying your app. (This may easily happen in automatic deployment environment like Kubernetes or AWS ECS.) If not the jitter, then those 500 processes would start hammering all your databases with new connections and maintenance queries at the same time. And worse, they would continue doing it in spike, on each new "tick" of the maintenance loops. + +It is especially deadly when using SSL and [pgbouncer](https://www.pgbouncer.org) : since pgbouncer is single-threaded, when it receives a burst of new connections, it severely overloads the CPU core, which causes the connections and queries to timeout and be retried. + +Having jitter helps in this situation perfectly. + +## Tweaking Maintenance Operations + +Normally, you don't need to tweak any of the parameters (time intervals and jitters) mentioned above, since Ent Framework has sane defaults for them. In case you still have to, then look at `PgClientOptions` and `ClusterOptions` interfaces. + + +# Logging and Diagnostic Tools + +Ent Framework includes reach instrumentation features that allows you to see, what exactly is happening in the cluster right now. + +## Loggers + +In [Connect to a Database](/getting-started/connect-to-a-database) article we defined our cluster the following way: + +```typescript +export const cluster = new Cluster({ + ..., + loggers: { + clientQueryLogger: (props) => console.debug(props.msg), + swallowedErrorLogger: (props) => console.log(props), + }, +}); +``` + +This is of course sub-optimal: e.g. Ent Framework will start printing all SQL queries it runs to the script console, which is not what you want. + +So instead, plug in your own logging solution, like [Datadog](https://www.datadoghq.com), [Elasticsearch APM](https://www.elastic.co/observability/application-performance-monitoring) or any solution based on OpenTelemetry standard (like [Prometheus](https://prometheus.io)). Of course, you also don't have to log every single SQL query: you may just log errors, and even do it to a file or to the console. + +Logger properties are defined in [Loggers.ts](https://github.com/clickup/ent-framework/blob/main/src/abstract/Loggers.ts) file. + +### clientQueryLogger(props: ClientQueryLoggerProps) + +This logger is called after each batched SQL query sent to the database. Here are the most important properties of `ClientQueryLoggerProps`: + +* `msg`: the query sent to the database. +* `error`: in case an error happened, its `string` representation (or `undefined` otherwise). +* `output`: raw results of the database driver library for the query. Don't log it entirely, since it may be too large. +* `elapsed`: an object with the number of milliseconds the query execution took, as 2 sub-properties: `total` (total time) and `acquire` (how much time the engine spent waiting for a connection becoming available in the pool). +* `annotations`: some information related to the [VC](/getting-started/vc-viewer-context-and-principal) that made the request. It's an array of objects, not noticeable sub-properties are: `vc` (text representation of VC principal and flavors) and `trace` (random-looking trace ID which is unique per each VC hierarchy). + +See Loggers.ts for more properties. + +In case an error is delivered to this logger, it means that the error was severe: it caused the Ent Framework to throw an exception to the client code. + +## swallowedErrorLogger(props: SwallowedErrorLoggerProps) + +As opposed to `clientQueryLogger`, this logger is called on non-critical (recoverable) errors, which likely did not cause the engine to throw an exception back to the client code. Such errors include: shard discovery retries, master-replica discovery failed attempts, connection failures, various recoverable timeouts and slowdowns etc. + +Treat such events as "warnings": better log them and monitor them, but to not trigger a panic in case they appear. + +Here are the most important properties of `SwallowedErrorLoggerProps`: + +* `error`: the original error (of type `unknown`) caused this logger to trigger. +* `where`: a `string` hint on where the error happened. +* `elapsed`: the time of the subject operation; if the error is related to some condition and not to action lasting over time, you'll get a null here. +* `importance`: either "low" or "high" strings. + +## Scoreboard Tool + +In [Cluster Maintenance Queries](/advanced/cluster-maintenance-queries) article we mentioned that Ent Framework runs a number of periodic internal queries across the cluster that are invisible to the user. Also, in case one node goes down or times out, the engine runs a series of retries, and it also tries to run cluster rediscovery to find a good replacement node. + +Doesn't it all look too opaque for you? + +The Scoreboard tool is to remove this opacity and expose, what exactly happens under the hood when there is a stream of queries coming to all nodes of the cluster. + +Scoreboard prints all nodes in the cluster (masters and replicas) as lines on the screen. It sends test "ping" queries to every node and displays, what happens. Also, if you shut down a node, it will start showing its state on the same timeline (and what rediscovery operations does it run), plus what errors does it observe. + +To run the tool (depending on your package manager): + +``` +pnpm exec ent-scoreboard +``` + +Parameters: + +* `--pingExecTimeMs`: when it sends a ping query (which is `pg_sleep()`), how long do you want this test query to run (by default, 0 ms, which is "as fast as possible"). +* `--pingParallelism`: how many pings to send in parallel (by default, it is 1). +* `--pingPollMs`: how often to send the ping messages (200 ms). +* `--maxQueries`: how many last pings to show on each line +* `--refreshMs`: how often to repaint the screen + +## Ping Tool + +As opposed to Scoreboard tool that sends test queries to all nodes of the cluster in parallel, the Ping tool only does it to one chosen shard client (master or replica), but it prints more detailed messages about what's happening. + +To run the tool (depending on your package manager): + +``` +pnpm exec ent-ping +``` + +Parameters: + +* `--shard`: the shard number to ping. +* `--pingExecTimeMs`: how long you want each ping query to take (each ping is a call to `pg_sleep()`). +* `--pingPollMs`: delay between pings (default: 500 ms). +* `--pingIsWrite` : if true, the pings will be sent to the master node, not to a random shard replica. + + +# Composite Primary Keys + +In each Ent instance, there is always a property named `id`. + +Ent Framework follows the pattern "convention over configuration" to simplify the most frequent use cases. In the world of database, the approach of having an explicit primary key `id` field (typically, generated based on some sequence) is considered a best practice. + +There are still databases where it's not the case though. You can use Ent Framework for them by utilizing the composite (or custom) primary keys feature. + +{% hint style="info" %} +It is strongly recommended to define an explicit `id` column on your tables though, since it solves many other problems and is just convenient in practice. Do not over-engineer, use the standard approaches. +{% endhint %} + +## Multi-Column Composite Primary Key + +Let's start with an example: + +```sql +CREATE TABLE memberships( + group_id bigint NOT NULL, + member_id bigint NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (group_id, member_id) +); +``` + +And the corresponding Ent class: + +```typescript +const schema = new PgSchema( + "memberships", + { + group_id: { type: ID }, + member_id: { type: ID }, + created_at: { type: Date, autoInsert: "now()" }, + }, + ["group_id", "member_id"], +); + +export class EntMembership extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.member_id, + privacyLoad: [...], + privacyInsert: [...], + }); + } +} +``` + +This Ent schema doesn't have an `id` property, and thus, Ent Framework understands that it should use the Ent's unique key `group_id, user_id` instead. + +So, it's that simple: if you don't define `id` field in the schema, then your schema's unique key becomes the primary key. + +Despite not defining an `id` field in the schema, your Ent instances will have it! + +```typescript +const membership = await EntMembership.insertReturning(vc, { + group_id: "100001001", + member_id: "100001002", +}); +// This prints "(100001001,100001002)" +console.log(membership.id); +// This also works! +const reloaded = EntMembership.loadX(vc, "(100001001,100001002)"); +// All other Ent calls work too. +await membership.deleteOriginal(); +``` + +Basically, if you don't have an `id` field in the schema, Ent Framework will create it for you and put a PostgreSQL unique key tuple in it. Tuples are a standard PostgreSQL syntax, and they look like: `(100001001,100001002)`. + +There is no way to define both a composite primary key and a different unique key in an Ent class. It's also impossible to have more than 1 unique key in a particular Ent schema. But it doesn't mean you can't define more right in your database itself (at SQL table level) and then use them in custom `select()` queries: of course you can. It is just considered an anti-pattern for most of the cases. + +## Single-Column Custom Primary Key + +If your unique key includes only 1 field, and there is no `id` property defined in the schema, that field becomes the value of Ent instance's `id` field. This is what you would naturally expect. + +```sql +CREATE TABLE users( + email varchar(64) NOT NULL PRIMARY KEY, + name varchar(128) NOT NULL, + created_at timestamptz NOT NULL +); +``` + +And the corresponding Ent class: + +```typescript +const schema = new PgSchema( + "users", + { + email: { type: String }, + name: { type: String }, + created_at: { type: Date, autoInsert: "now()" }, + }, + ["email"], +); + +export class EntUser extends BaseEnt(cluster, schema) { + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.email, + privacyLoad: [...], + privacyInsert: [...], + }); + } +} +``` + +Now notice how it's used: + +```typescript +const user = await EntUser.insertReturning(vc.toOmniDangerous(), { + email: "test@example.com", + name: "Alice", +}); +// This prints "test@example.com" (no parentheses). +console.log(user.id); +// VC's principal is also "test@example.com". +console.log(user.vc.principal); +// This also works! +const reloaded = EntMembership.loadX(vc, "test@example.com"); +// All other Ent calls work too. +await membership.deleteOriginal(); +``` + +Still, it's highly discouraged to do such things when you add a new table to your service. Better follow the best practices and add a regular `id` field. You can still use a custom primary key, but then you lose an ability to use other field(s) as a separate unique key in your schema. + + +# Passwords Rotation + +If your company regularly undergoes security audits (like SOC2), you know how challenging it is to rotate database passwords while keeping the service running without downtime. + +The goal of password rotation is to ensure that, at any given time, two login-password pairs exist in the database—"previous" and "current"—both functional. When rotating the password, you assign the new password to the "previous" login and then swap them. On startup, the app always uses the "current" login-password pair. + +Alternatively, you can use a single login while maintaining "previous" and "current" passwords for it. The app must be able to check both passwords and, if one stops working, quickly reconnect using the other. This approach would only work if your connection pooler (like PgBouncer) uses a "pass-through" mode (see [auth\_query](https://www.pgbouncer.org/config.html#auth_query) feature) and doesn't have a separate userlist.txt config with login-password pairs (otherwise, it's impossible to update the password for the same login transactionally and simultaneously in multiple places). + +Ent Framework supports both approaches: + +```typescript +import type { PoolConfig } from "pg"; + +export const cluster = new Cluster({ + islands: () => [ + { + no: 0, + nodes: [ + { + name: "island0-master", + config: { + connectionString: myConfig.DATABASE_URL_1, + ..., + } satisfies PoolConfig, + }, + { + name: "island0-master", // same name! + config: { + connectionString: myConfig.DATABASE_URL_2, + ..., + } satisfies PoolConfig, + }, + ..., + ], + }, + ], + createClient: (node) => new PgClient(node), + ..., +}); +``` + +If the engine sees that there are more than one island in the cluster having the same name, it will probe all connection config for the duplicates sequentially, until it finds the working one. Then, it will remember, which config is best, so next time a reconnect happens, it will start the probing from it (and most likely, it'll immediately hit the success). + +## Config Hot Reloading + +Unlikely your real code will look like the above example though. You'll probably want to iterate over some array in your hot-reloadable config instead of hardcoding `DATABASE_URL_1`, `DATABASE_URL_2` etc. + +Consider that in your app, you have a `config` object looking like this: + +```typescript +config = { + islands: [ + ["postgres://pg-001a/database", "postgres://pg-001b/database"], + ["postgres://pg-002a/database", "postgres://pg-002b/database"], + ["postgres://pg-003a/database", "postgres://pg-003b/database"], + ... + ], + secrets: [ + { login: "app_20380902123218", password: "" }, + { login: "app_20381002121152", password: "" }, + // Can also be the same login, but typically, you rotate + // BOTH login and password, such that the passwords for + // the existing logins are immutable. + }, +} +``` + +Here, we define 3 islands with 2 PostgreSQL nodes on each (one master and one replica; Ent Framework will decide on its own, which is what). We also define several login+password pairs to probe. The tool that you use to rotate the password must guarantee that at any given time, at least 1 login+password pair in this list is working. + +Given the config structure above, let's build an Ent Framework `Cluster` instance: + +```typescript +import type { PoolConfig } from " +import { config } from "./config"; + +export const cluster = new Cluster({ + islands: () => config.islands.map((islandNodes, no) => ({ + no, + nodes: islandNodes.flatMap((connectionString) => + config.secrets.map((secret) => ({ + name: connectionString, + config: { + connectionString, + user: secret.login, + password: secret.password, + ..., + } satisfies PoolConfig, + })) + ) + })), + createClient: (node) => new PgClient(node), + ..., +}); +``` + +And now the main part: the values in `config` object don't have to be constant! Notice that `islands` property in `Cluster` constructor options accepts a callback. This callback is run by Ent Framework from time to time to pull the most up-to-date cluster configuration. + +You can have a background code that reloads the config properties from some service(s) periodically: + +```typescript +setInterval(async () => { + config.islands = await enumerateFromAWSParameterStore(); + config.secrets = await enumerateAndLoadFromAWSSecretsManager(); +}, 10000); +// In real code, you'll likely want some logging and try-catch around. +``` + +This way, you can dynamically change the cluster topology and rotate passwords without downtime or even app reloading. + + +# Abstraction Layers + +
+ +Ent Framework consists of 3 abstraction layers: + +1. **Discovery & Connectivity Layer.** Includes such abstractions as Cluster, Island, Shard. Automatically detects changes in the cluster configuration and applies them (e.g. when a Shard is moved from one Island to another; when a new Node becomes available; when a replica is promoted to master or vice versa). +2. **Low-Level Queries Layer.** At this layer, low-level underlying DB driver features (such as connection pool or being able to send a raw SQL query) are exposed. Also, this layer provides services to the next abstraction, like building batched SQL queries or exposing replication lag tracking APIs. +3. **Ent API Layer.** Allows to define Ent classes with privacy rules, triggers, methods, composite fields. It also exposes an ORM-like query language in TypeScript, plus orchestrates parallel-executing queries batching and query caching. + + +# Ent Framework, Meta’s TAO, entgo + +At this point, it’s time to discuss the origins of Ent Framework, how it came to be, and what it has evolved from. + +The name "Ent Framework" originated at Meta (formerly Facebook), where it was used for an internal storage service. Since then, it has been referenced in numerous public articles across the Internet. + +At Facebook, Ent Framework was primarily a query language layer built on top of another storage service called [TAO](https://engineering.fb.com/2013/06/25/core-infra/tao-the-power-of-the-graph/) which stands for "The Associations and Objects". + +## Meta's TAO + +TAO provides a very low-level API for interacting with a graph. The nodes in this graph are called **Objects**, while the edges are referred to as **Assocs** (associations). + +Each Object in TAO has a globally unique ID and can contain an arbitrary number of named fields. For simplicity, you can think of it like a JSON object with an ID. The fields themselves are opaque to TAO, meaning it always operates on the Object as a whole. For example, if you load an Object by its ID, you retrieve all of its attributes at once. This is also how an Object is stored—in a key-value-like table, where the key is the ID, and the value is a serialized blob containing all the fields. + +As for what an Assoc is, it’s simply a pair of IDs (referred to as id1 and id2, representing the “source” and “destination” Objects). Assocs represent unidirectional edges in the graph, essentially defining an “arrow” from one Object to another. They are stored in a regular table with columns (id1, id2). In reality, this table also includes a timestamp column, and you can define a small number of “custom fields” for each Assoc, stored as a serialized blob. + +There is exactly one compound index defined on this table: (id1, timestamp), which allows for fast selection of all Assocs originating from the same id1, ordered by timestamp. + +When it comes to sharding, Objects are distributed across multiple shards, and the ID of an Object is sufficient to determine which shard it belongs to. Similarly, Assocs are sharded by their id1, meaning that for a given id1, you can quickly load all the id2s that are pointed to by a particular Assoc type. All these Assocs will reside in the same shard as id1. + +These are essentially all the core primitives relevant to the scope of this article. (There are a few additional features, such as Assoc Counters, which allow you to track the number of id2s for a given id1 of a particular Assoc type, and Keys, which enable finding Objects by unique strings instead of IDs, but those details are beyond the scope here.) + +By nature, Assocs are unidirectional. Even though each Assoc has both id1 and id2, you can only fetch them by id1 due to shard colocation based on id1. Now, imagine you wanted to load all Assocs where id2 equals a specific value. How would you identify all the shards that store those Assocs? Since they would be spread across different shards, and the number of involved shards would likely be too large to query efficiently, this becomes impractical. + +This is why, in many cases, instead of just creating one Assoc between a pair of Objects, two are used: one from ObjectA.id to ObjectB.id and another from ObjectB.id to ObjectA.id. This way, establishing a relationship between ObjectA and ObjectB results in two Assoc inserts, in two different shards, allowing traversal of the bi-directional edge in both directions. + +The opposite Assoc for a “forward” Assoc is called an “inverse Assoc” in TAO. There is a large infrastructure dedicated to keeping inverse Assocs in sync with "forward" Assocs and "field edges". This includes things like "assoc fixers", a distributed crawler that ensures data integrity, and a system that subscribes to the Write-Ahead Log (WAL) of databases to replay forward Assoc creations in order to create inverse Assocs, and similarly, handle deletions. Since there are no transactions possible across multiple shards, and forward and inverse Assocs naturally live in different shards, this synchronization is crucial. + +**In fact, the main reason for the existence of inverse Assocs is that related Objects may live in different shards.** Without sharding, we could simply query the Assocs table by id2. Keep this idea in mind. + +## What TypeScript’s Ent Framework Does Differently + +Although the TypeScript Ent Framework discussed in this tutorial shares the same name as Meta’s engine, it handles many aspects quite differently. + +### No Intermediate Layer Like TAO + +The main difference is that the TypeScript Ent Framework doesn’t rely on a lower-level abstraction like TAO; instead, it directly interacts with relational database tables. Ent Framework doesn’t try to obscure the underlying database mechanics (like PostgreSQL internals). It doesn’t generate DDL or manage schema migrations, nor does it create indexes. Instead, it works directly with relational databases (such as PostgreSQL), where an Ent corresponds to a table and fields map to columns, without the need for an intermediate layer like TAO. + +This approach is based on the observation that modern databases are feature-rich enough to eliminate the need for an intermediate layer. They also don’t require object field serialization for storage, since `ALTER TABLE` DDL queries are fast (e.g. adding or removing a column is cheap even on large tables; indexes creation is also cheap and can be done without blocking writes on a table). + +### No Explicit Assocs + +Comparing to Meta’s TAO, each Object (Ent Framework’s Ent) corresponds to a row in a table with the same name, and each Assoc (more precisely, each field edge) is represented as a column (foreign key field) in that table. What’s different it that inverse Assocs are just *indexes* on the relevant fields, automatically managed by the database, plus *something else*. + +This “something else” is an **Inverse**: a record similar to an inverse Assoc in TAO, but instead of storing an (id1, id2) pair, it stores an (id1, shard2) pair. + +To understand it better, consider an example: + +* EntTopic(id, title): conversation topics +* EntComment(id, topic\_id, text): comments of a particular topic +* All EntComment Ents live in different microshards + +In the microshard which holds EntTopic with id=topic\_id, there is an `inverses` table, which effectively stores (topic\_id, shard\_of\_comment) records. Ent Framework automatically keeps this table up to date each time a new EntComment is inserted. + +Now assume that we want to load all EntComment objects for a particular topic ID: + +
await EntComment.select(vc, { topic_id: "123" }, 1000);
+
+ +Ent Framework first loads the list of microshards where EntComment with the particular value of topic\_id reside; to do so, it uses the Inverses table in the microshard of the corresponding EntTopic: + +```sql +SELECT shard FROM inverses WHERE id1='123' AND type='topic2comments'; +``` + +Then, having the list of microshards, Ent Framework queries them all for all EntComment records and merges the results: + +```sql +run_on_each_microshard_in_parallel { + SELECT * FROM comments WHERE topic_id='123' LIMIT 1000; +} +``` + +Those last queries use an index on `topic_id` of course, ensuring the operation is efficient within each microshard. + +In other words, Ent Framework simplifies the concept of Assocs by only storing information about the destination shards, rather than the destination id2s of a relationship. So, Inverses and database indexes work in tandem to help you load the data from multiple microshards. + +The motivation here is that representing edges in a graph using an artificial Assoc concept is often unnecessary and too complex for most people, who are accustomed to edges being represented by standard relational table fields. The only small exception is the use of Inverses, which must be defined explicitly. However, their role is quite different: instead of pointing directly to an Ent on the other side of an edge, Inverses serve as *hints* to the engine about which microshards might contain those Ents. In this sense, Inverses are completely hidden from the user when making `select()` calls to Ent Framework and are used purely to *query only a smaller subset of microshards*, not all of them. In other words, Inverses are treated as a performance optimization, so you don’t need to think much about edges—since, for the most part, edges are just regular table fields. + +### Junction Ents vs. TAO Assocs + +Here’s an important observation regarding “many-to-many” relationships. In classical relational databases, these relationships are represented using “junction tables.” For example, if you have User and Group objects, where users can belong to many groups and groups can have many users, you would define an additional table (e.g., Membership) with user\_id and group\_id columns. This table, along with regular foreign keys and constraints, expresses the relationship. + +This schema is straightforward and easily understood by most people. The junction table typically has a clear noun meaning, and it can also include extra columns (like a timestamp or “friendship type”). So, it makes perfect sense to represent it as an EntMembership in Ent Framework as well. In contrast, in TAO, entgo, or Meta’s Ent Framework, that relationship would be modeled with “an Assoc with extra attributes and an inverse Assoc”, which adds complexity. Moreover, this Assoc isn’t just a regular one—it has additional custom fields, which starts to resemble an Object itself. This can feel like a leaking abstraction, and Ent Framework simplifies that. + +As a result, Ent Framework imposes a constraint on the abstract data graph: there are no direct “many-to-many” edges:
+ +

There is no such thing...

+ +Instead, the only type of edges are “many-to-one”. When a “many-to-many” relationship is needed, an intermediate junction node (like EntMembership) is required: + +

...This is what it we do instead

+ +Notice that "one to one" and "at most one to one" special cases are treated as special cases of "many to one" with unique index. + +### No 9 Kinds of Edges as in entgo, Since There are no Assocs + +If you read [entgo's docs about edges](https://entgo.io/docs/schema-edges/), you probably noticed that **entgo has 9 types of edges**. + +In Ent Framework, edges in the graph are basically just foreign key fields on Ents (with optional Inverses maintained automatically when needed). So, it is much simpler. + +### Ent Framework and entgo + +[Entgo](https://entgo.io) is, as it’s stated on the website, “An entity framework for Go. Simple, yet powerful ORM for modeling and querying data”. It is a library developed and open-sourced by Meta. + +It is not the same as Meta’s Ent Framework though: + +1. entgo is in Go, whilst Meta’s Ent Framework is in Hack +2. entgo does not support sharding, deferring it to the underlying database layer at best +3. there is nothing much about automatic replication lag tracking in entgo +4. no batching for the underlying SQL queries (i.e. no solution for “N+1 Select” problem) +5. despite being open-sourced, entgo is not used actively in Meta, which is very different with Meta’s Ent Framework, backing the entire facebook.com service + +So all in all, entgo is mostly an ORM-like wrapping library around a single database instance (e.g. PostgreSQL), with no attempts to do horizontal or vertical scaling. + +Thus, it is not quite correct to compare TypeScript Ent Framework described here with entgo: the more straight analogy would be ”it’s like a Meta’s Ent Framework, but without TAO and explicit assocs”. + + +# JIT in SQL Queries Batching + +One of the core Ent Framework's features is that it batches multiple concurrently running calls into a single SQL query. It also doesn't use JOIN for good, to enable seamless microsharding support and allow you to write your application code as if there is no "N+1 Selects" problem existing at all. In a typical workload, there are **lots** of concurrent queries running even for a single web request, and the batching factor is high. + +Batching greatly reduces the database connections utilization. Open connections are one of the most expensive resources in the cluster, even when some proxy service (like pgbouncer) sits between the backend and PostgreSQL. + +In fact, even in a small backend cluster, you **must** use something like [pgbouncer](https://www.pgbouncer.org), [pgcat](https://github.com/postgresml/pgcat) or other alternative. + +To do batching of multiple calls efficiently, we need to be able to build the resulting large SQL query as fast as possible, with the minimal Node CPU utilization. + +Notice that PostgreSQL and other relational databases have the concept of "prepared statements": if you run multiple queries of the same shape (e.g. multiple INSERTs to the same table), you can create a "prepared statement" once with `PREPARE` (which will build and cache the execution plan), and then run it multiple times with `EXECUTE`, passing different values for different rows. + +Ent Framework utilizes the same approach, but in Node.JS land. When receiving calls for batching, it recognizes their structure and dynamically builds ("compiles") a JS code for each unique input shape. This JS code is then materialized into a function (with `new Function(...)` which is essentially similar to JS `eval()` call under the hood), and that function is cached in memory. Notice that the function itself knows nothing about the actual data you're putting to the database: it is built based on the metadata only (like Ent field names and their types, DB table name etc.). + +Then, instead of "glueing" the SQL query from pieces and lots of `if` statements on each input row, Ent Framework calls the cached function passing each data row there. + +After several calls, the function becomes "hot", and Node.JS JITs (just-in-time compiles) it into machine code for the fastest execution possible. + +Essentially, it's a "codegen without codegen", or "codegen at runtime with caching", or "JIT-compiling into JS". + +Also, this approach lowers the risk of security vulnerabilities, since the SQL query "skeleton" is always built statically, and the actual values are injected there after the guaranteed escaping. + +If you want to learn more, a good starting point is [PgRunner](https://github.com/clickup/ent-framework/blob/2665ffa319134f35df8e883d8923c4c554b20220/src/pg/PgRunner.ts) class and its: + +* `createAnyBuilder()` method: builds long SQL expressions like `ANY('{aaa,bbb,ccc}')` +* `createInBuilder()` method: builds SQL expressions like `IN('aaa', 'bbb', 'ccc')` +* `createEscapeCode()` method: it knows the types and the explicit list of fields in advance, so it can avoid running multiple `if` statements at runtime and instead make decisions statically + + +# To JOIN or not to JOIN + +Ent Framework design *discourages* people from using SQL JOINs. Instead, it relies on the in-app parallel Promises merging and automatic queries batching, for 2 main reasons: + +1. It allows to work with microshards seamlessly (no JOINs can be run across the database boundaries efficiently). +2. It holistically solves [N+1 Selects problem](/getting-started/n+1-selects-solution). + +## Types of Joins + +In web development, JOINs are often times abused heavily. There are 3 main use cases when people use JOINs traditionaly, and only 2 of them are legit. + +### Type 1: Statistical Queries and OLAP + +When you have a large database, you sometimes need to pull some statistical information out of it. E.g. to answer a question, how many users registered and performed some action within a time frame, or how much money did the service earn, etc. Often times, building an SQL query with JOINs and running it *over a replica database* is the easiest solution. + +This use case is not so much frequent though. Although it's a fully legit use for the JOINs, it is relatively rare. Also, the larger your service becomes, the higher are the chances that you'll need to use some data warehouse solution instead for offline analysis (like Snowflake or a Presto-backed service). + +What distinguishes such a use case is that you run a small number of very heavy queries (OLAP pattern). + +### Type 2: Precise Query Optimization + +Sometimes you just want to squeeze the maximum performance from your database when running an OLTP load (i.e. when running a large number of very fast queries). I.e. you use JOINs for computational performance reasons: instead of transmitting 2 large lists from the database and intersecting them at the client (throwing away the absolute most of the transferred and non-matched rows), you ask the database server to do it internally utilizing indexes. + +But again, although it's a fully legit use case for JOINs, the need for it is relatively rare. + +### Type 3: Parent-Children Loading and N+1 Selects Problem + +And here coms the most frequent use cases when JOINs are traditionally used (actually, abused) in all mainstream ORMs. It is not related to slow queries, and not related to intersecting large lists throwing away non-matching items. The use case is purely about loading some objects and then their parents (or children), i.e. loading a sub-graph from a graph-like structure. + +In fact, such a simple use case composes the absolute most of the queries in real life. + +Let's see how it's done in Prisma: + +```typescript +const commentsWithDetails = await prisma.comment.findMany({ + where: { + id: { + in: commentIDs, + }, + }, + include: { + author: true, // Include the author of the comment. + topic: { + include: { + creator: true, // Include the creator of the topic. + }, + }, + }, +}); +``` + +This query produces a JOIN, and it does it for only one sole purpose: to work-around the [N+1 Selects](/getting-started/n+1-selects-solution) problem. + +Why is it suboptimal? Because such approaches force us to maka an assumption that at this level of abstraction, we have the complete list of comment IDs, and it is almost always not the case. + +Consider that we only know *one* comment ID at a time, but still want to pull the objects related to that comment: + +```typescript +async function loadCommentWithDetails(id: string) { + return prisma.comment.findUnique({ + where: { + id + }, + include: { + author: true, // Include the author of the comment. + topic: { + include: { + creator: true, // Include the creator of the topic. + }, + }, + }, + }); +} +``` + +Such an API has 2 fundamental flaws when using traditional ORMs without built-in query batching: + +1. If you need to load 100 comment—what would you even do? Try using `Promise.all()` with `loadCommentWithDetails()` for 100 IDs, an you'll get 100 database queries with JOINs. +2. "Load comments with details"—with what exact details? In one place of the code you'll need authors and topics, and in another one, you may only need the direct comment creator. Would you build a separate function with boilerplate for that? + +## Painful Boilerplate Analogy + +The above "type 3" of JOINs is not quite what JOINs are designed for: people use it to "duct tape" the real problem in a boilerplatish way. + +This reminds the early days of Web, when people were emitting their HTML as plain text, escaping values in every place explicitly: + +```html +<-- PHP code from 1990x. Beware: your eyes will bleed! --> +

Hellow, +!

+``` + +Thousands of projects were written this way. + +Type 3 JOIN is not much different conceptually. + +Think about the data duplication such JOINs produce over the wire. Consider the following query: + +```sql +SELECT + comments.id, + comments.text, + users.id AS author_id, + users.name AS author_name +FROM comments +JOIN users ON users.id = comments.author_id +``` + +The resulting data that is sends from the database server is: + +| id | text | author\_id | author\_name | +| -- | ------- | ---------- | ------------ | +| 1 | hello | 42 | Alice | +| 2 | my | 42 | Alice | +| 3 | dear | 42 | Alice | +| 4 | friend | 101 | Bob | +| 5 | bye now | 101 | Bon | + +Does it hurt your sense of engineering perfection? Does it smell to you? + +1. The author\_id+author\_name pair of values (42+Alice) is repeated 3 times in the payload for the first 3 comments, and then 101+Bob is repeated 2 times for the last 2 comments. Imagine now that `users` table has way more columns in practice. +2. Another smell is that "author\_" prefix: although being minor, it's clearly a naming boilerplate. You need to introduce some naming mapping convention between the column names in the JOIN result and in your ORM objects (be it glueing parts with "\_" or with "." or whatever). + +Such things are more related not to real resources utilization (the difference is marginal), but to the design and architecture smells. + +## Round Trip Latency Consideration + +To be fair, there is still one benefit in using type 3 JOINs: when you fetch comments, topics and users all at once, you only have 1 round-trip to the database server: + +```sql +-- Traditional ORM's way: 1 round-trip. +SELECT * +FROM comments +JOIN topics ON topics.id = comments.topic_id +JOIN users authors ON authors.id = comments.author_id +JOIN users creators ON creators.id = topics.creator_id +``` + +I.e. you send 1 request and get 1 response (with duplicated data, but anyways). + +If your backend-to-database network connection is slow (like one query takes 50 ms, which happens in commerical and highly vendor-lock-in prone solutions), then such consideration is significant. + +So, in slow networks, JOINs win over the Ent Framework's automatic batching approach: + +```sql +-- Ent Framework's way: 3 round-trips. +SELECT * FROM comments WHERE id IN(...); +SELECT * FROM topics WHERE id IN(...); +SELECT * FROM users WHERE id IN(...); +``` + +What's the catch? + +50 ms for a database query round trip is not a norm. That's the catch. + +In real life (and since 1990x), your network to the database is **not** slow. Quite the opposite, it is very fast, and you have sub-millisecond latency. Otherwise your entire backend becomes just so painfully slow in all other places that you can't manage it. + +Databases are designed to serve queries, do it fast, and with low latency at high concurrency. This is what the databases are for. Let's use the microscope for science and not to hammer nails. + +* False assumption: 50 ms database query is a norm; JOINs are to minimize round trips and solve [N+1 Selects](/getting-started/n+1-selects-solution) problem; take my money dear Vercel & Co. +* Reality: you have troubles with your app design if the query latency is longer than 1-2 ms; round trip time does not affect latency much in case the queries are batched. + +To learn more about batching, "parallel calls", and how event loop works in Node, check out [Loaders and Custom Batching](/advanced/loaders-and-custom-batching) article. + + diff --git a/docs/type-aliases/AfterMutationTrigger.md b/docs/type-aliases/AfterMutationTrigger.md new file mode 100644 index 0000000..4fc73ed --- /dev/null +++ b/docs/type-aliases/AfterMutationTrigger.md @@ -0,0 +1,28 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / AfterMutationTrigger + +# Type Alias: AfterMutationTrigger()\ + +> **AfterMutationTrigger**\<`TTable`\>: (`vc`, `args`) => `Promise`\<`unknown`\> \| `unknown` + +Defined in: [src/ent/Triggers.ts:147](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L147) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `args` | \{ `op`: `"INSERT"`; `newOrOldRow`: `Readonly`\<[`TriggerInsertInput`](TriggerInsertInput.md)\<`TTable`\>\>; \} \| \{ `op`: `"UPDATE"`; `newOrOldRow`: [`TriggerUpdateNewRow`](TriggerUpdateNewRow.md)\<`TTable`\>; \} \| \{ `op`: `"DELETE"`; `newOrOldRow`: [`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\>; \} | + +## Returns + +`Promise`\<`unknown`\> \| `unknown` diff --git a/docs/type-aliases/AfterUpdateTrigger.md b/docs/type-aliases/AfterUpdateTrigger.md new file mode 100644 index 0000000..52c01c8 --- /dev/null +++ b/docs/type-aliases/AfterUpdateTrigger.md @@ -0,0 +1,30 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / AfterUpdateTrigger + +# Type Alias: AfterUpdateTrigger()\ + +> **AfterUpdateTrigger**\<`TTable`\>: (`vc`, `args`) => `Promise`\<`unknown`\> \| `unknown` + +Defined in: [src/ent/Triggers.ts:108](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L108) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `args` | \{ `newRow`: [`TriggerUpdateNewRow`](TriggerUpdateNewRow.md)\<`TTable`\>; `oldRow`: [`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\>; \} | +| `args.newRow` | [`TriggerUpdateNewRow`](TriggerUpdateNewRow.md)\<`TTable`\> | +| `args.oldRow` | [`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\> | + +## Returns + +`Promise`\<`unknown`\> \| `unknown` diff --git a/docs/type-aliases/AnyClass.md b/docs/type-aliases/AnyClass.md new file mode 100644 index 0000000..3861cae --- /dev/null +++ b/docs/type-aliases/AnyClass.md @@ -0,0 +1,21 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / AnyClass + +# Type Alias: AnyClass() + +> **AnyClass**: (...`args`) => `unknown` + +Defined in: [src/ent/QueryCache.ts:17](https://github.com/clickup/ent-framework/blob/master/src/ent/QueryCache.ts#L17) + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| ...`args` | `never`[] | + +## Returns + +`unknown` diff --git a/docs/type-aliases/BeforeMutationTrigger.md b/docs/type-aliases/BeforeMutationTrigger.md new file mode 100644 index 0000000..4b68fea --- /dev/null +++ b/docs/type-aliases/BeforeMutationTrigger.md @@ -0,0 +1,28 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / BeforeMutationTrigger + +# Type Alias: BeforeMutationTrigger()\ + +> **BeforeMutationTrigger**\<`TTable`\>: (`vc`, `args`) => `Promise`\<`unknown`\> \| `unknown` + +Defined in: [src/ent/Triggers.ts:123](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L123) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `args` | \{ `op`: `"INSERT"`; `newOrOldRow`: `Readonly`\<[`TriggerInsertInput`](TriggerInsertInput.md)\<`TTable`\>\>; `input`: [`TriggerInsertInput`](TriggerInsertInput.md)\<`TTable`\>; \} \| \{ `op`: `"UPDATE"`; `newOrOldRow`: [`TriggerUpdateNewRow`](TriggerUpdateNewRow.md)\<`TTable`\>; `input`: [`TriggerUpdateInput`](TriggerUpdateInput.md)\<`TTable`\>; \} \| \{ `op`: `"DELETE"`; `newOrOldRow`: [`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\>; `input`: `Writeable`\<[`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\>\>; \} | + +## Returns + +`Promise`\<`unknown`\> \| `unknown` diff --git a/docs/type-aliases/BeforeUpdateTrigger.md b/docs/type-aliases/BeforeUpdateTrigger.md new file mode 100644 index 0000000..611c5e3 --- /dev/null +++ b/docs/type-aliases/BeforeUpdateTrigger.md @@ -0,0 +1,31 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / BeforeUpdateTrigger + +# Type Alias: BeforeUpdateTrigger()\ + +> **BeforeUpdateTrigger**\<`TTable`\>: (`vc`, `args`) => `Promise`\<`unknown`\> \| `unknown` + +Defined in: [src/ent/Triggers.ts:99](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L99) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `args` | \{ `newRow`: [`TriggerUpdateNewRow`](TriggerUpdateNewRow.md)\<`TTable`\>; `oldRow`: [`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\>; `input`: [`TriggerUpdateInput`](TriggerUpdateInput.md)\<`TTable`\>; \} | +| `args.newRow` | [`TriggerUpdateNewRow`](TriggerUpdateNewRow.md)\<`TTable`\> | +| `args.oldRow` | [`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\> | +| `args.input` | [`TriggerUpdateInput`](TriggerUpdateInput.md)\<`TTable`\> | + +## Returns + +`Promise`\<`unknown`\> \| `unknown` diff --git a/docs/type-aliases/ClientErrorKind.md b/docs/type-aliases/ClientErrorKind.md new file mode 100644 index 0000000..5c1bbef --- /dev/null +++ b/docs/type-aliases/ClientErrorKind.md @@ -0,0 +1,14 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientErrorKind + +# Type Alias: ClientErrorKind + +> **ClientErrorKind**: `"data-on-server-is-unchanged"` \| `"unknown-server-state"` + +Defined in: [src/abstract/ClientError.ts:35](https://github.com/clickup/ent-framework/blob/master/src/abstract/ClientError.ts#L35) + +Sometimes we need to know for sure, is there a chance that the query failed, +but the write was still applied in the database. diff --git a/docs/type-aliases/ClientErrorPostAction.md b/docs/type-aliases/ClientErrorPostAction.md new file mode 100644 index 0000000..7e3d096 --- /dev/null +++ b/docs/type-aliases/ClientErrorPostAction.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientErrorPostAction + +# Type Alias: ClientErrorPostAction + +> **ClientErrorPostAction**: `"rediscover-cluster"` \| `"rediscover-island"` \| `"choose-another-client"` \| `"fail"` + +Defined in: [src/abstract/ClientError.ts:7](https://github.com/clickup/ent-framework/blob/master/src/abstract/ClientError.ts#L7) + +The suggested action, what can we do when facing a ClientError. diff --git a/docs/type-aliases/ClientRole.md b/docs/type-aliases/ClientRole.md new file mode 100644 index 0000000..c400267 --- /dev/null +++ b/docs/type-aliases/ClientRole.md @@ -0,0 +1,16 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClientRole + +# Type Alias: ClientRole + +> **ClientRole**: `"master"` \| `"replica"` \| `"unknown"` + +Defined in: [src/abstract/Client.ts:40](https://github.com/clickup/ent-framework/blob/master/src/abstract/Client.ts#L40) + +Role of the Client as reported after the last successful query. If we know +for sure that the Client is a master or a replica, the role will be "master" +or "replica" correspondingly. If no queries were run by the Client yet (i.e. +we don't know the role for sure), the role is assigned to "unknown". diff --git a/docs/type-aliases/ClusterIslands.md b/docs/type-aliases/ClusterIslands.md new file mode 100644 index 0000000..e5279be --- /dev/null +++ b/docs/type-aliases/ClusterIslands.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ClusterIslands + +# Type Alias: ClusterIslands\ + +> **ClusterIslands**\<`TNode`\>: `ReadonlyArray`\<\{ `no`: `number`; `nodes`: readonly `TNode`[]; \}\> + +Defined in: [src/abstract/Cluster.ts:101](https://github.com/clickup/ent-framework/blob/master/src/abstract/Cluster.ts#L101) + +A type of `ClusterOptions#islands` property. Represents the full list of +Islands and their corresponding Nodes (masters and replicas). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TNode` | diff --git a/docs/type-aliases/CountInput.md b/docs/type-aliases/CountInput.md new file mode 100644 index 0000000..c9d2161 --- /dev/null +++ b/docs/type-aliases/CountInput.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / CountInput + +# Type Alias: CountInput\ + +> **CountInput**\<`TTable`\>: [`Where`](Where.md)\<`TTable`\> + +Defined in: [src/types.ts:327](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L327) + +Table -> { f: 10, [$or]: [ { f2: "a }, { f3: "b""} ], $literal: ["x=?", 1] } + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/DeleteTrigger.md b/docs/type-aliases/DeleteTrigger.md new file mode 100644 index 0000000..b098e8e --- /dev/null +++ b/docs/type-aliases/DeleteTrigger.md @@ -0,0 +1,29 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / DeleteTrigger + +# Type Alias: DeleteTrigger()\ + +> **DeleteTrigger**\<`TTable`\>: (`vc`, `args`) => `Promise`\<`unknown`\> \| `unknown` + +Defined in: [src/ent/Triggers.ts:116](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L116) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `args` | \{ `oldRow`: [`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\>; \} | +| `args.oldRow` | [`TriggerUpdateOrDeleteOldRow`](TriggerUpdateOrDeleteOldRow.md)\<`TTable`\> | + +## Returns + +`Promise`\<`unknown`\> \| `unknown` diff --git a/docs/type-aliases/DeleteWhereInput.md b/docs/type-aliases/DeleteWhereInput.md new file mode 100644 index 0000000..f5f70c1 --- /dev/null +++ b/docs/type-aliases/DeleteWhereInput.md @@ -0,0 +1,25 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / DeleteWhereInput + +# Type Alias: DeleteWhereInput\ + +> **DeleteWhereInput**\<`TTable`\>: `object` & `Omit`\<[`Where`](Where.md)\<`TTable`\>, *typeof* [`ID`](../variables/ID.md)\> + +Defined in: [src/types.ts:337](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L337) + +Table -> { id: ["1", "2", "3"], ... } + +## Type declaration + +### id + +> **id**: `string`[] + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/DepsBuilder.md b/docs/type-aliases/DepsBuilder.md new file mode 100644 index 0000000..1600f81 --- /dev/null +++ b/docs/type-aliases/DepsBuilder.md @@ -0,0 +1,28 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / DepsBuilder + +# Type Alias: DepsBuilder()\ + +> **DepsBuilder**\<`TTable`\>: (`vc`, `row`) => `unknown`[] \| `Promise`\<`unknown`[]\> + +Defined in: [src/ent/Triggers.ts:164](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L164) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `row` | `Flatten`\<`Readonly`\<[`Row`](Row.md)\<`TTable`\>\>\> | + +## Returns + +`unknown`[] \| `Promise`\<`unknown`[]\> diff --git a/docs/type-aliases/ExistsInput.md b/docs/type-aliases/ExistsInput.md new file mode 100644 index 0000000..e8105a0 --- /dev/null +++ b/docs/type-aliases/ExistsInput.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ExistsInput + +# Type Alias: ExistsInput\ + +> **ExistsInput**\<`TTable`\>: [`Where`](Where.md)\<`TTable`\> + +Defined in: [src/types.ts:332](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L332) + +Table -> { f: 10, [$or]: [ { f2: "a }, { f3: "b""} ], $literal: ["x=?", 1] } + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/Field.md b/docs/type-aliases/Field.md new file mode 100644 index 0000000..9209587 --- /dev/null +++ b/docs/type-aliases/Field.md @@ -0,0 +1,26 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Field + +# Type Alias: Field\ + +> **Field**\<`TTable`\>: keyof `TTable` & `string` + +Defined in: [src/types.ts:102](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L102) + +A database table's field (no symbols). In regards to some table structure, +there can be 3 options: +1. Field: only DB-stored attributes, no ephemeral symbols +2. keyof TTable: both real and ephemeral attributes +3. keyof TTable & symbol: only "ephemeral" attributes available to triggers + +By doing `& string`, we ensure that we select only regular (non-symbol) +fields. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/FieldAliased.md b/docs/type-aliases/FieldAliased.md new file mode 100644 index 0000000..d738f03 --- /dev/null +++ b/docs/type-aliases/FieldAliased.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FieldAliased + +# Type Alias: FieldAliased\ + +> **FieldAliased**\<`TTable`\>: [`Field`](Field.md)\<`TTable`\> \| \{ `field`: [`Field`](Field.md)\<`TTable`\>; `alias`: `string`; \} + +Defined in: [src/types.ts:108](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L108) + +Same as Field, but may optionally hold information about of "alias value +source" for a field name (e.g. `{ field: "abc", alias: "$cas.abc" }`). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/FieldIsValidatorPlain.md b/docs/type-aliases/FieldIsValidatorPlain.md new file mode 100644 index 0000000..3509418 --- /dev/null +++ b/docs/type-aliases/FieldIsValidatorPlain.md @@ -0,0 +1,32 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FieldIsValidatorPlain + +# Type Alias: FieldIsValidatorPlain()\ + +> **FieldIsValidatorPlain**\<`TField`, `TRow`\>: (`fieldValue`, `row`, `vc`) => [`ValidatorPlainResult`](ValidatorPlainResult.md) \| `Promise`\<[`ValidatorPlainResult`](ValidatorPlainResult.md)\> + +Defined in: [src/ent/predicates/FieldIs.ts:13](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/FieldIs.ts#L13) + +A field validator function that returns a boolean. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TField` *extends* `string` | +| `TRow` *extends* `Partial`\<`Record`\<`TField`, `unknown`\>\> | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `fieldValue` | `TRow`\[`TField`\] | +| `row` | `TRow` | +| `vc` | [`VC`](../classes/VC.md) | + +## Returns + +[`ValidatorPlainResult`](ValidatorPlainResult.md) \| `Promise`\<[`ValidatorPlainResult`](ValidatorPlainResult.md)\> diff --git a/docs/type-aliases/FieldIsValidatorStandardSchemaV1.md b/docs/type-aliases/FieldIsValidatorStandardSchemaV1.md new file mode 100644 index 0000000..82b63d6 --- /dev/null +++ b/docs/type-aliases/FieldIsValidatorStandardSchemaV1.md @@ -0,0 +1,31 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FieldIsValidatorStandardSchemaV1 + +# Type Alias: FieldIsValidatorStandardSchemaV1()\ + +> **FieldIsValidatorStandardSchemaV1**\<`TRow`\>: (`fieldValue`, `row`, `vc`) => [`ValidatorStandardSchemaResult`](ValidatorStandardSchemaResult.md) \| `Promise`\<[`ValidatorStandardSchemaResult`](ValidatorStandardSchemaResult.md)\> + +Defined in: [src/ent/predicates/FieldIs.ts:34](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/FieldIs.ts#L34) + +A field validator function that returns a Standard Schema validation result. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TRow` | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `fieldValue` | `unknown` | +| `row` | `TRow` | +| `vc` | [`VC`](../classes/VC.md) | + +## Returns + +[`ValidatorStandardSchemaResult`](ValidatorStandardSchemaResult.md) \| `Promise`\<[`ValidatorStandardSchemaResult`](ValidatorStandardSchemaResult.md)\> diff --git a/docs/type-aliases/FieldIsValidatorZodSafeParse.md b/docs/type-aliases/FieldIsValidatorZodSafeParse.md new file mode 100644 index 0000000..9e8b110 --- /dev/null +++ b/docs/type-aliases/FieldIsValidatorZodSafeParse.md @@ -0,0 +1,31 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FieldIsValidatorZodSafeParse + +# Type Alias: FieldIsValidatorZodSafeParse()\ + +> **FieldIsValidatorZodSafeParse**\<`TRow`\>: (`fieldValue`, `row`, `vc`) => [`ValidatorZodSafeParseResult`](ValidatorZodSafeParseResult.md) \| `Promise`\<[`ValidatorZodSafeParseResult`](ValidatorZodSafeParseResult.md)\> + +Defined in: [src/ent/predicates/FieldIs.ts:25](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/FieldIs.ts#L25) + +A field validator function that returns a Zod result. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TRow` | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `fieldValue` | `unknown` | +| `row` | `TRow` | +| `vc` | [`VC`](../classes/VC.md) | + +## Returns + +[`ValidatorZodSafeParseResult`](ValidatorZodSafeParseResult.md) \| `Promise`\<[`ValidatorZodSafeParseResult`](ValidatorZodSafeParseResult.md)\> diff --git a/docs/type-aliases/FieldOfIDType.md b/docs/type-aliases/FieldOfIDType.md new file mode 100644 index 0000000..4ebcc8d --- /dev/null +++ b/docs/type-aliases/FieldOfIDType.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FieldOfIDType + +# Type Alias: FieldOfIDType\ + +> **FieldOfIDType**\<`TTable`\>: `{ [K in Field]: K extends string ? TTable[K] extends { type: typeof ID } ? K : never : never }`\[[`Field`](Field.md)\<`TTable`\>\] + +Defined in: [src/types.ts:134](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L134) + +Table -> "user_id" | "some_id" | ... + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/FieldOfIDTypeRequired.md b/docs/type-aliases/FieldOfIDTypeRequired.md new file mode 100644 index 0000000..6b77e15 --- /dev/null +++ b/docs/type-aliases/FieldOfIDTypeRequired.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FieldOfIDTypeRequired + +# Type Alias: FieldOfIDTypeRequired\ + +> **FieldOfIDTypeRequired**\<`TTable`\>: [`InsertFieldsRequired`](InsertFieldsRequired.md)\<`TTable`\> & [`FieldOfIDType`](FieldOfIDType.md)\<`TTable`\> + +Defined in: [src/types.ts:145](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L145) + +Table -> "user_id" | "some_id" | ... + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/FieldOfPotentialUniqueKey.md b/docs/type-aliases/FieldOfPotentialUniqueKey.md new file mode 100644 index 0000000..00b80e0 --- /dev/null +++ b/docs/type-aliases/FieldOfPotentialUniqueKey.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / FieldOfPotentialUniqueKey + +# Type Alias: FieldOfPotentialUniqueKey\ + +> **FieldOfPotentialUniqueKey**\<`TTable`\>: \{ \[K in Field\\]: TTable\[K\] extends \{ type: typeof Number \| typeof String \| typeof Boolean \| typeof ID \| typeof Date \| \{ dbValueToJs: (dbValue: never) =\> string \| number \} \} ? K : never \}\[[`Field`](Field.md)\<`TTable`\>\] + +Defined in: [src/types.ts:116](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L116) + +(Table) -> "field1" | "field2" | ... where the union contains only fields +which can potentially be used as a part of unique key. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/Hints.md b/docs/type-aliases/Hints.md new file mode 100644 index 0000000..b21a732 --- /dev/null +++ b/docs/type-aliases/Hints.md @@ -0,0 +1,14 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Hints + +# Type Alias: Hints + +> **Hints**: `Record`\<`string`, `string` \| `null` \| `undefined`\> + +Defined in: [src/types.ts:346](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L346) + +Planner hints. Null means "reset to the engine's default", and "undefined" +means the same as "no key mentioned at all". diff --git a/docs/type-aliases/InsertFieldsOptional.md b/docs/type-aliases/InsertFieldsOptional.md new file mode 100644 index 0000000..91bed50 --- /dev/null +++ b/docs/type-aliases/InsertFieldsOptional.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / InsertFieldsOptional + +# Type Alias: InsertFieldsOptional\ + +> **InsertFieldsOptional**\<`TTable`\>: `{ [K in keyof TTable]: TTable[K] extends { autoInsert: unknown } ? K : TTable[K] extends { autoUpdate: unknown } ? K : never }`\[keyof `TTable`\] + +Defined in: [src/types.ts:201](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L201) + +Insert: Table -> "created_at" | "field2" | ... deduction (optional fields). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/InsertFieldsRequired.md b/docs/type-aliases/InsertFieldsRequired.md new file mode 100644 index 0000000..7e52bcf --- /dev/null +++ b/docs/type-aliases/InsertFieldsRequired.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / InsertFieldsRequired + +# Type Alias: InsertFieldsRequired\ + +> **InsertFieldsRequired**\<`TTable`\>: `{ [K in keyof TTable]: TTable[K] extends { autoInsert: unknown } ? never : TTable[K] extends { autoUpdate: unknown } ? never : K }`\[keyof `TTable`\] + +Defined in: [src/types.ts:190](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L190) + +Insert: Table -> "field1" | "field2" | ... deduction (required). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/InsertInput.md b/docs/type-aliases/InsertInput.md new file mode 100644 index 0000000..40796ec --- /dev/null +++ b/docs/type-aliases/InsertInput.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / InsertInput + +# Type Alias: InsertInput\ + +> **InsertInput**\<`TTable`\>: `{ [K in InsertFieldsRequired]: Value }` & `{ [K in InsertFieldsOptional]?: Value }` + +Defined in: [src/types.ts:213](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L213) + +Insert: Table -> { field: string, updated_at?: Date, created_at?: Date... }. +Excludes id Spec entirely and makes autoInsert/autoUpdate Specs optional. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/InsertTrigger.md b/docs/type-aliases/InsertTrigger.md new file mode 100644 index 0000000..ea3acde --- /dev/null +++ b/docs/type-aliases/InsertTrigger.md @@ -0,0 +1,65 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / InsertTrigger + +# Type Alias: InsertTrigger()\ + +> **InsertTrigger**\<`TTable`\>: (`vc`, `args`) => `Promise`\<`unknown`\> \| `unknown` + +Defined in: [src/ent/Triggers.ts:94](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L94) + +Triggers could be used to simulate "transactional best-effort behavior" in a +non-transactional combination of some services. Imagine we have a relational +database and a queue service; each time we change something in the query, we +want to schedule the ID to the queue. Queue service is faulty: if a queueing +operation fails, we don't want the data to be stored to the DB afterwards. +Queries are faulty too, but it's okay for us to have something added to the +queue even if the corresponding query failed after it (a queue worker will +just do a no-op since it anyway rechecks the source of truth in relational +DBs). Queue service is like a write-ahead log for DB which always has +not-less records than the DB. In this case, we have the following set of +triggers: + +1. beforeInsert: schedules ID to the queue (ID is known, see below why) +2. beforeUpdate: schedules ID to the queue +3. afterDelete: optionally schedule ID removal to the queue (notice "after") + +Notice that ID is always known in all cases, even in insertBefore triggers, +because we split an INSERT operation into gen_id+insert parts, and the +triggers are executed in between. + +Triggers are invoked sequentially. Any exception thrown in a before-trigger +is propagated to the caller, and the DB operation is skipped. + +Triggers for beforeInsert and beforeUpdate can change their input parameter, +the change will apply to the database. + +Naming convention for trigger arguments: +1. input: whatever is passed to the operation. Notice that due to us having + autoInsert/autoUpdate fields, the set of fields can be incomplete here! +1. oldRow: the entire row in the DB which was there before the operation. All + the fields will be presented there. +2. newRow: a row in the DB as it will looks like after the operation. Notice + that it can be non precise, because we don't always reload the updated row + from the database! What we do is just field by field application of input + properties to oldRow. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `args` | \{ `input`: [`TriggerInsertInput`](TriggerInsertInput.md)\<`TTable`\>; \} | +| `args.input` | [`TriggerInsertInput`](TriggerInsertInput.md)\<`TTable`\> | + +## Returns + +`Promise`\<`unknown`\> \| `unknown` diff --git a/docs/type-aliases/JSONValue.md b/docs/type-aliases/JSONValue.md new file mode 100644 index 0000000..710e193 --- /dev/null +++ b/docs/type-aliases/JSONValue.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / JSONValue + +# Type Alias: JSONValue + +> **JSONValue**: `null` \| `string` \| `number` \| `boolean` \| [`JSONValue`](JSONValue.md)[] \| `{ [k in string]?: JSONValue }` + +Defined in: [src/types.ts:407](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L407) + +A JSON-serializable value. diff --git a/docs/type-aliases/Literal.md b/docs/type-aliases/Literal.md new file mode 100644 index 0000000..478ece5 --- /dev/null +++ b/docs/type-aliases/Literal.md @@ -0,0 +1,15 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Literal + +# Type Alias: Literal + +> **Literal**: (`string` \| `number` \| `boolean` \| `Date` \| `null` \| (`string` \| `number` \| `boolean` \| `Date` \| `null`)[])[] + +Defined in: [src/types.ts:14](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L14) + +Literal operation with placeholders. We don't use a tuple type here (like +`[string, ...T[]]`), because it would force us to use `as const` everywhere, +which we don't want to do. diff --git a/docs/type-aliases/LoadByInput.md b/docs/type-aliases/LoadByInput.md new file mode 100644 index 0000000..d3c53f3 --- /dev/null +++ b/docs/type-aliases/LoadByInput.md @@ -0,0 +1,21 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / LoadByInput + +# Type Alias: LoadByInput\ + +> **LoadByInput**\<`TTable`, `TUniqueKey`\>: `TUniqueKey` *extends* \[\] ? `never` : `{ [K in TUniqueKey[number]]: Value }` + +Defined in: [src/types.ts:257](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L257) + +(Table, UniqueKey) -> { field1: number, field2: number, field3: number }. +loadBy operation is allowed for exact unique key attributes only. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](UniqueKey.md)\<`TTable`\> | diff --git a/docs/type-aliases/LoadRule.md b/docs/type-aliases/LoadRule.md new file mode 100644 index 0000000..349a05f --- /dev/null +++ b/docs/type-aliases/LoadRule.md @@ -0,0 +1,17 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / LoadRule + +# Type Alias: LoadRule\ + +> **LoadRule**\<`TInput`\>: [`AllowIf`](../classes/AllowIf.md)\<`TInput`\> \| [`DenyIf`](../classes/DenyIf.md)\<`TInput`\> + +Defined in: [src/ent/Validation.ts:24](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L24) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` *extends* `object` | diff --git a/docs/type-aliases/Order.md b/docs/type-aliases/Order.md new file mode 100644 index 0000000..a851668 --- /dev/null +++ b/docs/type-aliases/Order.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Order + +# Type Alias: Order\ + +> **Order**\<`TTable`\>: `ReadonlyArray`\<`{ [K in Field]?: string }` & `object`\> + +Defined in: [src/types.ts:310](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L310) + +Table -> [["f1", "ASC"], ["f2", "DESC"]] or [ [{[$literal]: ["a=?", 10]}, +"ASC"], ["b", "DESC"] ] + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/PrimitiveClass.md b/docs/type-aliases/PrimitiveClass.md new file mode 100644 index 0000000..90be857 --- /dev/null +++ b/docs/type-aliases/PrimitiveClass.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / PrimitiveClass + +# Type Alias: PrimitiveClass\ + +> **PrimitiveClass**\<`TTable`, `TUniqueKey`, `TClient`\>: `OmitNew`\<[`ConfigClass`](../interfaces/ConfigClass.md)\<`TTable`, `TUniqueKey`, `TClient`\>\> & () => [`PrimitiveInstance`](../interfaces/PrimitiveInstance.md)\<`TTable`\> + +Defined in: [src/ent/mixins/PrimitiveMixin.ts:68](https://github.com/clickup/ent-framework/blob/master/src/ent/mixins/PrimitiveMixin.ts#L68) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](UniqueKey.md)\<`TTable`\> | +| `TClient` *extends* [`Client`](../classes/Client.md) | diff --git a/docs/type-aliases/Row.md b/docs/type-aliases/Row.md new file mode 100644 index 0000000..1f34dda --- /dev/null +++ b/docs/type-aliases/Row.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Row + +# Type Alias: Row\ + +> **Row**\<`TTable`\>: [`RowWithID`](RowWithID.md) & `{ [K in Field]: Value }` + +Defined in: [src/types.ts:183](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L183) + +Table -> Row deduction (no symbols). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/RowIsValidatorPlain.md b/docs/type-aliases/RowIsValidatorPlain.md new file mode 100644 index 0000000..209fb4f --- /dev/null +++ b/docs/type-aliases/RowIsValidatorPlain.md @@ -0,0 +1,30 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / RowIsValidatorPlain + +# Type Alias: RowIsValidatorPlain()\ + +> **RowIsValidatorPlain**\<`TRow`\>: (`row`, `vc`) => [`ValidatorPlainResult`](ValidatorPlainResult.md) \| `Promise`\<[`ValidatorPlainResult`](ValidatorPlainResult.md)\> + +Defined in: [src/ent/predicates/RowIs.ts:13](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/RowIs.ts#L13) + +A row validator function that returns a boolean. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TRow` | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `row` | `TRow` | +| `vc` | [`VC`](../classes/VC.md) | + +## Returns + +[`ValidatorPlainResult`](ValidatorPlainResult.md) \| `Promise`\<[`ValidatorPlainResult`](ValidatorPlainResult.md)\> diff --git a/docs/type-aliases/RowIsValidatorStandardSchemaV1.md b/docs/type-aliases/RowIsValidatorStandardSchemaV1.md new file mode 100644 index 0000000..6bfd3a3 --- /dev/null +++ b/docs/type-aliases/RowIsValidatorStandardSchemaV1.md @@ -0,0 +1,30 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / RowIsValidatorStandardSchemaV1 + +# Type Alias: RowIsValidatorStandardSchemaV1()\ + +> **RowIsValidatorStandardSchemaV1**\<`TRow`\>: (`row`, `vc`) => [`ValidatorStandardSchemaResult`](ValidatorStandardSchemaResult.md) \| `Promise`\<[`ValidatorStandardSchemaResult`](ValidatorStandardSchemaResult.md)\> + +Defined in: [src/ent/predicates/RowIs.ts:29](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/RowIs.ts#L29) + +A row validator function that returns a Standard Schema validation result. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TRow` | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `row` | `TRow` | +| `vc` | [`VC`](../classes/VC.md) | + +## Returns + +[`ValidatorStandardSchemaResult`](ValidatorStandardSchemaResult.md) \| `Promise`\<[`ValidatorStandardSchemaResult`](ValidatorStandardSchemaResult.md)\> diff --git a/docs/type-aliases/RowIsValidatorZodSafeParse.md b/docs/type-aliases/RowIsValidatorZodSafeParse.md new file mode 100644 index 0000000..3ace958 --- /dev/null +++ b/docs/type-aliases/RowIsValidatorZodSafeParse.md @@ -0,0 +1,30 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / RowIsValidatorZodSafeParse + +# Type Alias: RowIsValidatorZodSafeParse()\ + +> **RowIsValidatorZodSafeParse**\<`TRow`\>: (`row`, `vc`) => [`ValidatorZodSafeParseResult`](ValidatorZodSafeParseResult.md) \| `Promise`\<[`ValidatorZodSafeParseResult`](ValidatorZodSafeParseResult.md)\> + +Defined in: [src/ent/predicates/RowIs.ts:21](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/RowIs.ts#L21) + +A row validator function that returns a Zod result. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TRow` | + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `row` | `TRow` | +| `vc` | [`VC`](../classes/VC.md) | + +## Returns + +[`ValidatorZodSafeParseResult`](ValidatorZodSafeParseResult.md) \| `Promise`\<[`ValidatorZodSafeParseResult`](ValidatorZodSafeParseResult.md)\> diff --git a/docs/type-aliases/RowWithID.md b/docs/type-aliases/RowWithID.md new file mode 100644 index 0000000..9331804 --- /dev/null +++ b/docs/type-aliases/RowWithID.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / RowWithID + +# Type Alias: RowWithID + +> **RowWithID**: `object` + +Defined in: [src/types.ts:33](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L33) + +{ id: string } + +## Type declaration + +### id + +> **id**: `string` diff --git a/docs/type-aliases/RuleDecision.md b/docs/type-aliases/RuleDecision.md new file mode 100644 index 0000000..c968cfa --- /dev/null +++ b/docs/type-aliases/RuleDecision.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / RuleDecision + +# Type Alias: RuleDecision + +> **RuleDecision**: `"ALLOW"` \| `"TOLERATE"` \| `"SKIP"` \| `"DENY"` + +Defined in: [src/ent/rules/Rule.ts:9](https://github.com/clickup/ent-framework/blob/master/src/ent/rules/Rule.ts#L9) + +Each Rule evaluates to some Decision (or throws). diff --git a/docs/type-aliases/SelectByInput.md b/docs/type-aliases/SelectByInput.md new file mode 100644 index 0000000..2d57b8b --- /dev/null +++ b/docs/type-aliases/SelectByInput.md @@ -0,0 +1,21 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / SelectByInput + +# Type Alias: SelectByInput\ + +> **SelectByInput**\<`TTable`, `TUniqueKey`\>: [`LoadByInput`](LoadByInput.md)\<`TTable`, `TuplePrefixes`\<`TUniqueKey`\>\> + +Defined in: [src/types.ts:268](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L268) + +(Table, UniqueKey) -> { field1: number [, field2: number [, ...] ] }. +selectBy operation is allowed for unique key PREFIX attributes only. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | +| `TUniqueKey` *extends* [`UniqueKey`](UniqueKey.md)\<`TTable`\> | diff --git a/docs/type-aliases/SelectInput.md b/docs/type-aliases/SelectInput.md new file mode 100644 index 0000000..bf0779c --- /dev/null +++ b/docs/type-aliases/SelectInput.md @@ -0,0 +1,37 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / SelectInput + +# Type Alias: SelectInput\ + +> **SelectInput**\<`TTable`\>: `object` + +Defined in: [src/types.ts:317](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L317) + +Table -> { where: ..., order?: ..., ... } + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Type declaration + +### where + +> **where**: [`Where`](Where.md)\<`TTable`\> + +### order? + +> `optional` **order**: [`Order`](Order.md)\<`TTable`\> + +### custom? + +> `optional` **custom**: `object` + +### limit + +> **limit**: `number` diff --git a/docs/type-aliases/SelectInputCustom.md b/docs/type-aliases/SelectInputCustom.md new file mode 100644 index 0000000..230fd28 --- /dev/null +++ b/docs/type-aliases/SelectInputCustom.md @@ -0,0 +1,14 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / SelectInputCustom + +# Type Alias: SelectInputCustom + +> **SelectInputCustom**: \{ `ctes`: [`Literal`](Literal.md)[]; `joins`: [`Literal`](Literal.md)[]; `from`: [`Literal`](Literal.md); `hints`: [`Hints`](Hints.md); \} \| `undefined` + +Defined in: [src/pg/PgQuerySelect.ts:17](https://github.com/clickup/ent-framework/blob/master/src/pg/PgQuerySelect.ts#L17) + +This is mostly to do hacks in PostgreSQL queries. Not even exposed by Ent +framework, but can be used by PG-dependent code. diff --git a/docs/type-aliases/ShardAffinity.md b/docs/type-aliases/ShardAffinity.md new file mode 100644 index 0000000..6a9404d --- /dev/null +++ b/docs/type-aliases/ShardAffinity.md @@ -0,0 +1,28 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ShardAffinity + +# Type Alias: ShardAffinity\ + +> **ShardAffinity**\<`TField`, `TF`\>: *typeof* [`GLOBAL_SHARD`](../variables/GLOBAL_SHARD.md) \| `TField` *extends* *typeof* [`ID`](../variables/ID.md) ? readonly `TF`[] : readonly \[`TF`, `...TF[]`\] + +Defined in: [src/ent/ShardAffinity.ts:19](https://github.com/clickup/ent-framework/blob/master/src/ent/ShardAffinity.ts#L19) + +Defines Ent Shard collocation to some Ent's field when this Ent is inserted. +- The Shard can always be Shard 0 ("global Shard"), be inferred based on the + value in other Ent field during the insertion ("colocation"), or, in case + colocation inference didn't succeed, be chosen pseudo-randomly at insertion + time ("random Shard"). +- E.g. a random Shard can also be chosen in case an empty array is passed to + Shard affinity (like "always fallback"), or when a field's value points to + a global Shard. +- Passing ID to ShardAffinity is prohibited by TS. + +## Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `TField` *extends* `string` | - | +| `TF` | `Exclude`\<`TField`, *typeof* [`ID`](../variables/ID.md)\> | diff --git a/docs/type-aliases/Spec.md b/docs/type-aliases/Spec.md new file mode 100644 index 0000000..9e72921 --- /dev/null +++ b/docs/type-aliases/Spec.md @@ -0,0 +1,31 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Spec + +# Type Alias: Spec + +> **Spec**: `object` + +Defined in: [src/types.ts:78](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L78) + +{ type: ..., ... } - one attribute spec. + +## Type declaration + +### type + +> **type**: [`SpecType`](SpecType.md) + +### allowNull? + +> `optional` **allowNull**: `true` + +### autoInsert? + +> `optional` **autoInsert**: `string` + +### autoUpdate? + +> `optional` **autoUpdate**: `string` diff --git a/docs/type-aliases/SpecType.md b/docs/type-aliases/SpecType.md new file mode 100644 index 0000000..e2786af --- /dev/null +++ b/docs/type-aliases/SpecType.md @@ -0,0 +1,95 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / SpecType + +# Type Alias: SpecType + +> **SpecType**: *typeof* `Boolean` \| *typeof* `Date` \| *typeof* [`ID`](../variables/ID.md) \| *typeof* `Number` \| *typeof* `String` \| \{ `dbValueToJs`: (`dbValue`) => `unknown`; `stringify`: (`jsValue`) => `string`; `parse`: (`str`) => `unknown`; \} + +Defined in: [src/types.ts:40](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L40) + +Spec (metadata) of some field. + +## Type declaration + +*typeof* `Boolean` + +*typeof* `Date` + +*typeof* [`ID`](../variables/ID.md) + +*typeof* `Number` + +*typeof* `String` + +\{ `dbValueToJs`: (`dbValue`) => `unknown`; `stringify`: (`jsValue`) => `string`; `parse`: (`str`) => `unknown`; \} + +### dbValueToJs() + +> **dbValueToJs**: (`dbValue`) => `unknown` + +Converts a value of some field returned by the low-level DB engine to +its Client representation, which can be reacher (e.g. support +encryption/decryption). Notice that some DB engines already do some +conversions internally: e.g. for node-postgres and an array field, +dbValue returned by the engine is already an array of things, so +dbValueToJs for it will likely do nothing. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `dbValue` | `DesperateAny` | + +#### Returns + +`unknown` + +### stringify() + +> **stringify**: (`jsValue`) => `string` + +Converts a Client value to the internal stringified representation of +the low-level DB engine, which is suitable for injecting it into a +plaintext query (with e.g. ?-placeholders). +- Notice that this is intentionally NOT the opposite to dbValueToJs, + because it always needs to convert the value to a string, not to the + DB engine's row field type. +- Example: node-postgres natively understands json/jsonb PG types and + can unescape them (called "PG type parsers" and mainly lives in + pg-types module; notice that there are no "PG type stringifiers + though"). The problem is that the low-level library's facilities for + escaping data is poor or doesn't exist (we do escaping by ourselves + for various reasons, like batching queries and better logging). So we + trust the library on the dbValueToJs path, but must manually + serialize on stringify path. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `jsValue` | `DesperateAny` | + +#### Returns + +`string` + +### parse() + +> **parse**: (`str`) => `unknown` + +The opposite to stringify function. Generally, it is not used on the +read path (because the low level engine returns the rows suitable for +dbValueToJs), but it's still here for completeness of the interface. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `str` | `string` | + +#### Returns + +`unknown` diff --git a/docs/type-aliases/Table.md b/docs/type-aliases/Table.md new file mode 100644 index 0000000..861ade9 --- /dev/null +++ b/docs/type-aliases/Table.md @@ -0,0 +1,17 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Table + +# Type Alias: Table + +> **Table**: `object` + +Defined in: [src/types.ts:88](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L88) + +{ id: Spec, name: Spec, ... } - table columns. + +## Index Signature + +\[`K`: `string` \| `symbol`\]: [`Spec`](Spec.md) diff --git a/docs/type-aliases/TimelineCaughtUpReason.md b/docs/type-aliases/TimelineCaughtUpReason.md new file mode 100644 index 0000000..273768b --- /dev/null +++ b/docs/type-aliases/TimelineCaughtUpReason.md @@ -0,0 +1,14 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / TimelineCaughtUpReason + +# Type Alias: TimelineCaughtUpReason + +> **TimelineCaughtUpReason**: `false` \| `"replica-bc-master-state-unknown"` \| `"replica-bc-caught-up"` \| `"replica-bc-pos-expired"` + +Defined in: [src/abstract/Timeline.ts:7](https://github.com/clickup/ent-framework/blob/master/src/abstract/Timeline.ts#L7) + +The reason why the decision that this replica timeline is "good enough" has +been made. diff --git a/docs/type-aliases/TriggerInsertInput.md b/docs/type-aliases/TriggerInsertInput.md new file mode 100644 index 0000000..fe30472 --- /dev/null +++ b/docs/type-aliases/TriggerInsertInput.md @@ -0,0 +1,24 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / TriggerInsertInput + +# Type Alias: TriggerInsertInput\ + +> **TriggerInsertInput**\<`TTable`\>: `Flatten`\<[`InsertInput`](InsertInput.md)\<`TTable`\> & [`RowWithID`](RowWithID.md)\> + +Defined in: [src/ent/Triggers.ts:22](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L22) + +Table -> trigger's before- and after-insert input. Below, we use InsertInput +and not Row, because before and even after some INSERT, we may still not know +some values of the row (they can be filled by the DB in e.g. autoInsert +clause). InsertInput is almost a subset of Row, but it has stricter symbol +keys: e.g. if some symbol key is non-optional in INSERT (aka doesn't have +autoInsert), it will always be required in InsertInput too. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/TriggerUpdateInput.md b/docs/type-aliases/TriggerUpdateInput.md new file mode 100644 index 0000000..6a89daf --- /dev/null +++ b/docs/type-aliases/TriggerUpdateInput.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / TriggerUpdateInput + +# Type Alias: TriggerUpdateInput\ + +> **TriggerUpdateInput**\<`TTable`\>: `Flatten`\<[`UpdateInput`](UpdateInput.md)\<`TTable`\>\> + +Defined in: [src/ent/Triggers.ts:29](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L29) + +Table -> trigger's before-update input. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/TriggerUpdateNewRow.md b/docs/type-aliases/TriggerUpdateNewRow.md new file mode 100644 index 0000000..f3d9d9e --- /dev/null +++ b/docs/type-aliases/TriggerUpdateNewRow.md @@ -0,0 +1,21 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / TriggerUpdateNewRow + +# Type Alias: TriggerUpdateNewRow\ + +> **TriggerUpdateNewRow**\<`TTable`\>: `Flatten`\<`Readonly`\<[`Row`](Row.md)\<`TTable`\> & `{ [K in keyof TTable & symbol]?: Value }`\>\> + +Defined in: [src/ent/Triggers.ts:38](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L38) + +Table -> trigger's before- and after-update NEW row. Ephemeral (symbol) +fields may or may not be passed depending on what the user passes to the +update method. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/TriggerUpdateOrDeleteOldRow.md b/docs/type-aliases/TriggerUpdateOrDeleteOldRow.md new file mode 100644 index 0000000..6ba5bcd --- /dev/null +++ b/docs/type-aliases/TriggerUpdateOrDeleteOldRow.md @@ -0,0 +1,22 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / TriggerUpdateOrDeleteOldRow + +# Type Alias: TriggerUpdateOrDeleteOldRow\ + +> **TriggerUpdateOrDeleteOldRow**\<`TTable`\>: `Flatten`\<`Readonly`\<[`Row`](Row.md)\<`TTable`\> & `Record`\\>\> + +Defined in: [src/ent/Triggers.ts:52](https://github.com/clickup/ent-framework/blob/master/src/ent/Triggers.ts#L52) + +Table -> trigger's before- and after-update (or delete) OLD row. Ephemeral +(symbol) fields are marked as always presented, but "never" typed, so they +will be available for dereferencing in newOrOldRow of before/after mutation +triggers without guard-checking of op value. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/UniqueKey.md b/docs/type-aliases/UniqueKey.md new file mode 100644 index 0000000..cc0e534 --- /dev/null +++ b/docs/type-aliases/UniqueKey.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / UniqueKey + +# Type Alias: UniqueKey\ + +> **UniqueKey**\<`TTable`\>: \[\] \| \[[`FieldOfPotentialUniqueKey`](FieldOfPotentialUniqueKey.md)\<`TTable`\>, `...FieldOfPotentialUniqueKey[]`\] + +Defined in: [src/types.ts:246](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L246) + +Table -> ["field1", "field2", ...], list of fields allowed to compose an +unique key on the table; fields must be allowed in insert/upsert. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/UpdateField.md b/docs/type-aliases/UpdateField.md new file mode 100644 index 0000000..600d5da --- /dev/null +++ b/docs/type-aliases/UpdateField.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / UpdateField + +# Type Alias: UpdateField\ + +> **UpdateField**\<`TTable`\>: `Exclude`\ + +Defined in: [src/types.ts:222](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L222) + +Update: Table -> "field1" | "created_at" | "updated_at" | ... deduction. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/UpdateInput.md b/docs/type-aliases/UpdateInput.md new file mode 100644 index 0000000..5fbf89a --- /dev/null +++ b/docs/type-aliases/UpdateInput.md @@ -0,0 +1,34 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / UpdateInput + +# Type Alias: UpdateInput\ + +> **UpdateInput**\<`TTable`\>: `{ [K in UpdateField]?: Value }` & `object` + +Defined in: [src/types.ts:235](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L235) + +Update: Table -> { field?: string, created_at?: Date, updated_at?: Date }. +- Excludes id Spec entirely and makes all fields optional. +- If $literal is passed, it will be appended to the list of updating fields + (engine specific). +- If $cas is passed, only the rows whose fields match the exact values in + $cas will be updated; the non-matching rows will be skipped. + +## Type declaration + +### $literal? + +> `optional` **$literal**: [`Literal`](Literal.md) + +### $cas? + +> `optional` **$cas**: `{ [K in UpdateField]?: Value }` + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/UpdateOriginalInput.md b/docs/type-aliases/UpdateOriginalInput.md new file mode 100644 index 0000000..233fb9d --- /dev/null +++ b/docs/type-aliases/UpdateOriginalInput.md @@ -0,0 +1,31 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / UpdateOriginalInput + +# Type Alias: UpdateOriginalInput\ + +> **UpdateOriginalInput**\<`TTable`\>: `{ [K in UpdateField]?: Value }` & `object` + +Defined in: [src/ent/types.ts:81](https://github.com/clickup/ent-framework/blob/master/src/ent/types.ts#L81) + +The input of updateOriginal() method. It supports some additional syntax +sugar for $cas property, so to work-around TS weakness of Omit<> & type +inference, we redefine this type from scratch. + +## Type declaration + +### $literal? + +> `optional` **$literal**: [`Literal`](Literal.md) + +### $cas? + +> `optional` **$cas**: `"skip-if-someone-else-changed-updating-ent-props"` \| `ReadonlyArray`\<[`UpdateField`](UpdateField.md)\<`TTable`\>\> \| [`UpdateInput`](UpdateInput.md)\<`TTable`\>\[`"$cas"`\] + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/ValidationRules.md b/docs/type-aliases/ValidationRules.md new file mode 100644 index 0000000..908eb54 --- /dev/null +++ b/docs/type-aliases/ValidationRules.md @@ -0,0 +1,58 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ValidationRules + +# Type Alias: ValidationRules\ + +> **ValidationRules**\<`TTable`\>: `object` + +Defined in: [src/ent/Validation.ts:59](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L59) + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | + +## Type declaration + +### tenantPrincipalField? + +> `readonly` `optional` **tenantPrincipalField**: [`InsertFieldsRequired`](InsertFieldsRequired.md)\<`TTable`\> & `string` + +### inferPrincipal() + +> `readonly` **inferPrincipal**: (`vc`, `row`) => `Promise`\<[`VC`](../classes/VC.md)\> + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `vc` | [`VC`](../classes/VC.md) | +| `row` | [`Row`](Row.md)\<`TTable`\> | + +#### Returns + +`Promise`\<[`VC`](../classes/VC.md)\> + +### load + +> `readonly` **load**: [`Validation`](../classes/Validation.md)\<`TTable`\>\[`"load"`\] + +### insert + +> `readonly` **insert**: [`Validation`](../classes/Validation.md)\<`TTable`\>\[`"insert"`\] + +### update? + +> `readonly` `optional` **update**: [`Validation`](../classes/Validation.md)\<`TTable`\>\[`"update"`\] + +### delete? + +> `readonly` `optional` **delete**: [`Validation`](../classes/Validation.md)\<`TTable`\>\[`"delete"`\] + +### validate? + +> `readonly` `optional` **validate**: [`AbstractIs`](../interfaces/AbstractIs.md)\<[`InsertInput`](InsertInput.md)\<`TTable`\>\>[] diff --git a/docs/type-aliases/ValidatorPlainResult.md b/docs/type-aliases/ValidatorPlainResult.md new file mode 100644 index 0000000..16365c4 --- /dev/null +++ b/docs/type-aliases/ValidatorPlainResult.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ValidatorPlainResult + +# Type Alias: ValidatorPlainResult + +> **ValidatorPlainResult**: `boolean` + +Defined in: [src/ent/predicates/AbstractIs.ts:20](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/AbstractIs.ts#L20) + +Result of plain validators. diff --git a/docs/type-aliases/ValidatorStandardSchemaResult.md b/docs/type-aliases/ValidatorStandardSchemaResult.md new file mode 100644 index 0000000..45ea899 --- /dev/null +++ b/docs/type-aliases/ValidatorStandardSchemaResult.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ValidatorStandardSchemaResult + +# Type Alias: ValidatorStandardSchemaResult + +> **ValidatorStandardSchemaResult**: \{ `value`: `unknown`; `issues`: `undefined`; \} \| \{ `issues`: `object`[]; \} + +Defined in: [src/ent/predicates/AbstractIs.ts:43](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/AbstractIs.ts#L43) + +Result of a Standard Schema validators: https://standardschema.dev/ diff --git a/docs/type-aliases/ValidatorZodSafeParseResult.md b/docs/type-aliases/ValidatorZodSafeParseResult.md new file mode 100644 index 0000000..c0d14f9 --- /dev/null +++ b/docs/type-aliases/ValidatorZodSafeParseResult.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ValidatorZodSafeParseResult + +# Type Alias: ValidatorZodSafeParseResult + +> **ValidatorZodSafeParseResult**: \{ `success`: `true`; `error`: `undefined`; \} \| \{ `success`: `false`; `error`: \{ `issues`: `ReadonlyArray`\<\{ `message`: `string`; `path`: readonly `PropertyKey`[]; \}\>; \}; \} + +Defined in: [src/ent/predicates/AbstractIs.ts:25](https://github.com/clickup/ent-framework/blob/master/src/ent/predicates/AbstractIs.ts#L25) + +Result of Zod safeParse() calls. diff --git a/docs/type-aliases/Value.md b/docs/type-aliases/Value.md new file mode 100644 index 0000000..580f4e9 --- /dev/null +++ b/docs/type-aliases/Value.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Value + +# Type Alias: Value\ + +> **Value**\<`TSpec`\>: `TSpec` *extends* `object` ? [`ValueRequired`](ValueRequired.md)\<`TSpec`\> \| `null` : [`ValueRequired`](ValueRequired.md)\<`TSpec`\> + +Defined in: [src/types.ts:176](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L176) + +Spec -> nullable Value or non-nullable Value. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TSpec` *extends* [`Spec`](Spec.md) | diff --git a/docs/type-aliases/ValueRequired.md b/docs/type-aliases/ValueRequired.md new file mode 100644 index 0000000..e8bd471 --- /dev/null +++ b/docs/type-aliases/ValueRequired.md @@ -0,0 +1,19 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ValueRequired + +# Type Alias: ValueRequired\ + +> **ValueRequired**\<`TSpec`\>: `TSpec`\[`"type"`\] *extends* *typeof* `Number` ? `number` : `TSpec`\[`"type"`\] *extends* *typeof* `String` ? `string` : `TSpec`\[`"type"`\] *extends* *typeof* `Boolean` ? `boolean` : `TSpec`\[`"type"`\] *extends* *typeof* [`ID`](../variables/ID.md) ? `string` : `TSpec`\[`"type"`\] *extends* *typeof* `Date` ? `Date` : `TSpec`\[`"type"`\] *extends* `object` ? `TSpec`\[`"type"`\] *extends* `object` ? `TJSValue` : `never` : `never` + +Defined in: [src/types.ts:151](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L151) + +SpecType -> Value deduction (always deduces non-nullable type). + +## Type Parameters + +| Type Parameter | +| ------ | +| `TSpec` *extends* [`Spec`](Spec.md) | diff --git a/docs/type-aliases/Where.md b/docs/type-aliases/Where.md new file mode 100644 index 0000000..f61aa15 --- /dev/null +++ b/docs/type-aliases/Where.md @@ -0,0 +1,47 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / Where + +# Type Alias: Where\ + +> **Where**\<`TTable`\>: `object` & `object` & \{ \[K in Field\\]?: Value\ \| ReadonlyArray\\> \| \{ $lte: NonNullable\\> \} \| \{ $lt: NonNullable\\> \} \| \{ $gte: NonNullable\\> \} \| \{ $gt: NonNullable\\> \} \| \{ $overlap: NonNullable\\> \} \| \{ $ne: Value\ \| ReadonlyArray\\> \} \| \{ $isDistinctFrom: Value\ \} \} + +Defined in: [src/types.ts:276](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L276) + +Table -> { f: 10, [$or]: [ { f2: "a }, { f3: "b""} ], $literal: ["x=?", 1] } + +## Type declaration + +### $and? + +> `optional` **$and**: `ReadonlyArray`\<[`Where`](Where.md)\<`TTable`\>\> + +### $or? + +> `optional` **$or**: `ReadonlyArray`\<[`Where`](Where.md)\<`TTable`\>\> + +### $not? + +> `optional` **$not**: [`Where`](Where.md)\<`TTable`\> + +### $literal? + +> `optional` **$literal**: [`Literal`](Literal.md) + +### $shardOfID? + +> `optional` **$shardOfID**: `string` + +## Type declaration + +### id? + +> `optional` **id**: `TTable` *extends* `object` ? `unknown` : `string` \| `string`[] + +## Type Parameters + +| Type Parameter | +| ------ | +| `TTable` *extends* [`Table`](Table.md) | diff --git a/docs/type-aliases/WhyClient.md b/docs/type-aliases/WhyClient.md new file mode 100644 index 0000000..f4d6025 --- /dev/null +++ b/docs/type-aliases/WhyClient.md @@ -0,0 +1,20 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / WhyClient + +# Type Alias: WhyClient + +> **WhyClient**: `Exclude`\<[`TimelineCaughtUpReason`](TimelineCaughtUpReason.md), `false`\> \| `"replica-bc-stale-replica-freshness"` \| `"master-bc-is-write"` \| `"master-bc-master-freshness"` \| `"master-bc-no-replicas"` \| `"master-bc-replica-not-caught-up"` + +Defined in: [src/abstract/QueryAnnotation.ts:13](https://github.com/clickup/ent-framework/blob/master/src/abstract/QueryAnnotation.ts#L13) + +A reason why master or replica was chosen to send the query to. The most +noticeable ones are: +- "replica-bc-master-state-unknown": 99% of cases (since writes are rare) +- "master-bc-replica-not-caught-up": happens immediately after each write, + until the write is propagated to replica +- "replica-bc-caught-up": must happen eventually (in 0.1-2s) after each write +- "replica-bc-pos-expired": signals that the replication lag is huge, we + should carefully monitor this case and make sure it never happens diff --git a/docs/type-aliases/WriteRules.md b/docs/type-aliases/WriteRules.md new file mode 100644 index 0000000..8231d72 --- /dev/null +++ b/docs/type-aliases/WriteRules.md @@ -0,0 +1,23 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / WriteRules + +# Type Alias: WriteRules\ + +> **WriteRules**\<`TInput`\>: \[\] \| \[[`Require`](../classes/Require.md)\<`TInput`\>, `...Require[]`\] \| \[[`LoadRule`](LoadRule.md)\<`TInput`\>, [`Require`](../classes/Require.md)\<`TInput`\>, `...Require[]`\] \| \[[`LoadRule`](LoadRule.md)\<`TInput`\>, [`LoadRule`](LoadRule.md)\<`TInput`\>, [`Require`](../classes/Require.md)\<`TInput`\>, `...Require[]`\] \| \[[`LoadRule`](LoadRule.md)\<`TInput`\>, [`LoadRule`](LoadRule.md)\<`TInput`\>, [`LoadRule`](LoadRule.md)\<`TInput`\>, [`Require`](../classes/Require.md)\<`TInput`\>, `...Require[]`\] \| \[[`LoadRule`](LoadRule.md)\<`TInput`\>, [`LoadRule`](LoadRule.md)\<`TInput`\>, [`LoadRule`](LoadRule.md)\<`TInput`\>, [`LoadRule`](LoadRule.md)\<`TInput`\>, [`Require`](../classes/Require.md)\<`TInput`\>, `...Require[]`\] + +Defined in: [src/ent/Validation.ts:33](https://github.com/clickup/ent-framework/blob/master/src/ent/Validation.ts#L33) + +For safety, we enforce all Require rules to be in the end of the +insert/update/delete privacy list, and have at least one of them. In +TypeScript, it's not possible to create [...L[], R, ...R[]] type +(double-variadic) when both L[] and R[] are open-ended (i.e. tuples with +unknown length), so we have to brute-force. + +## Type Parameters + +| Type Parameter | +| ------ | +| `TInput` *extends* `object` | diff --git a/docs/variables/GLOBAL_SHARD.md b/docs/variables/GLOBAL_SHARD.md new file mode 100644 index 0000000..1d4a756 --- /dev/null +++ b/docs/variables/GLOBAL_SHARD.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / GLOBAL\_SHARD + +# Variable: GLOBAL\_SHARD + +> `const` **GLOBAL\_SHARD**: `"global_shard"` = `"global_shard"` + +Defined in: [src/ent/ShardAffinity.ts:6](https://github.com/clickup/ent-framework/blob/master/src/ent/ShardAffinity.ts#L6) + +The table is located in the global Shard (0). diff --git a/docs/variables/GUEST_ID.md b/docs/variables/GUEST_ID.md new file mode 100644 index 0000000..c808ad2 --- /dev/null +++ b/docs/variables/GUEST_ID.md @@ -0,0 +1,14 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / GUEST\_ID + +# Variable: GUEST\_ID + +> `const` **GUEST\_ID**: `"guest"` = `"guest"` + +Defined in: [src/ent/VC.ts:23](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L23) + +Guest VC: has minimum permissions. Typically if the user is not logged in, +this VC is used. diff --git a/docs/variables/ID.md b/docs/variables/ID.md new file mode 100644 index 0000000..47b411e --- /dev/null +++ b/docs/variables/ID.md @@ -0,0 +1,14 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / ID + +# Variable: ID + +> `const` **ID**: `"id"` = `"id"` + +Defined in: [src/types.ts:7](https://github.com/clickup/ent-framework/blob/master/src/types.ts#L7) + +Primary key field's name is currently hardcoded for simplicity. It's a +convention to have it named as "id". diff --git a/docs/variables/MASTER.md b/docs/variables/MASTER.md new file mode 100644 index 0000000..a4bec0e --- /dev/null +++ b/docs/variables/MASTER.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / MASTER + +# Variable: MASTER + +> `const` **MASTER**: *typeof* [`MASTER`](MASTER.md) + +Defined in: [src/abstract/Shard.ts:11](https://github.com/clickup/ent-framework/blob/master/src/abstract/Shard.ts#L11) + +Master freshness: reads always go to master. diff --git a/docs/variables/OMNI_ID.md b/docs/variables/OMNI_ID.md new file mode 100644 index 0000000..4358893 --- /dev/null +++ b/docs/variables/OMNI_ID.md @@ -0,0 +1,16 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / OMNI\_ID + +# Variable: OMNI\_ID + +> `const` **OMNI\_ID**: `"omni"` = `"omni"` + +Defined in: [src/ent/VC.ts:31](https://github.com/clickup/ent-framework/blob/master/src/ent/VC.ts#L31) + +Temporary "omniscient" VC. Any Ent can be loaded with it, but this VC is +replaced with lower-pri VC as soon as possible. E.g. when some Ent is loaded +with omni VC, its ent.vc is assigned to either this Ent's "owner" VC +(accessible via VC pointing field) or, if not detected, to guest VC. diff --git a/docs/variables/STALE_REPLICA.md b/docs/variables/STALE_REPLICA.md new file mode 100644 index 0000000..32000f2 --- /dev/null +++ b/docs/variables/STALE_REPLICA.md @@ -0,0 +1,13 @@ +[**ent-framework**](../README.md) + +*** + +[ent-framework](../globals.md) / STALE\_REPLICA + +# Variable: STALE\_REPLICA + +> `const` **STALE\_REPLICA**: *typeof* [`STALE_REPLICA`](STALE_REPLICA.md) + +Defined in: [src/abstract/Shard.ts:16](https://github.com/clickup/ent-framework/blob/master/src/abstract/Shard.ts#L16) + +Stale replica freshness: reads always go to a replica, even if it's stale. diff --git a/eslint.base.config.mjs b/eslint.base.config.mjs new file mode 100644 index 0000000..7799df9 --- /dev/null +++ b/eslint.base.config.mjs @@ -0,0 +1,675 @@ +"use strict"; + +import eslint from "@eslint/js"; +import tseslint from "@typescript-eslint/eslint-plugin"; +import eslintParser from "@typescript-eslint/parser"; +import eslintPluginImport from "eslint-plugin-import"; +import eslintPluginLodash from "eslint-plugin-lodash"; +import eslintPluginNoOnlyTests from "eslint-plugin-no-only-tests"; +import eslintPluginNode from "eslint-plugin-n"; +import eslintPluginTypescriptEnum from "eslint-plugin-typescript-enum"; +import eslintPluginUnusedImports from "eslint-plugin-unused-imports"; +import globals from "globals"; +import stylistic from "@stylistic/eslint-plugin"; + +// Custom local rules for Slapdash codebase +const localRules = { + rules: { + "zod-prefer-safe-optional": { + meta: { + type: "problem", + docs: { + description: + "Prefer .safeOptional() over .optional(), .nullable(), or .nullish() for LLM response Zod schemas", + category: "Best Practices", + recommended: true, + }, + messages: { + preferSafeOptional: + "Use .safeOptional() instead of .{{method}}() for LLM response schemas. " + + ".safeOptional() handles both undefined and null values, and it converts null to undefined " + + "to avoid the need for null checks. LLMs sometimes generate null instead of undefined " + + "and vice versa, so .safeOptional() is more robust.", + }, + schema: [], + }, + create(context) { + return { + MemberExpression(node) { + // Check if this is a method call like .optional(), .nullable(), or .nullish() + if ( + node.property && + node.property.type === "Identifier" && + ["optional", "nullable", "nullish"].includes(node.property.name) + ) { + // Check if this is being called (e.g., .optional() not just .optional) + const parent = node.parent; + if ( + parent && + parent.type === "CallExpression" && + parent.callee === node + ) { + // Try to determine if this is a Zod schema + // This is a heuristic: we look for z. in the call chain + let current = node.object; + let isZodSchema = false; + + // Walk up the chain looking for z.something + while (current) { + if ( + current.type === "MemberExpression" && + current.object && + current.object.type === "Identifier" && + current.object.name === "z" + ) { + isZodSchema = true; + break; + } + + if (current.type === "CallExpression") { + current = current.callee; + } else if (current.type === "MemberExpression") { + current = current.object; + } else { + break; + } + } + + if (isZodSchema) { + context.report({ + node: node.property, + messageId: "preferSafeOptional", + data: { + method: node.property.name, + }, + }); + } + } + } + }, + }; + }, + }, + }, +}; + +// Fixing a bug in the eslint-plugin-react library. +const browserGlobals = { + ...globals.browser, + AudioWorkletGlobalScope: false, // this is the default, +}; +delete browserGlobals["AudioWorkletGlobalScope "]; +delete tseslint.configs.recommended["extends"]; + +function toFlatConfig(plugin, name) { + delete plugin.configs.recommended["extends"]; + const parserOptions = plugin.configs.recommended.parserOptions; + delete plugin.configs.recommended["parserOptions"]; + return { + ...plugin.configs.recommended, + plugins: { [name]: plugin }, + ...(parserOptions ? { languageOptions: { parserOptions } } : {}), + }; +} + +export default function createConfig({ + projectRoot, + eslintTsConfig, + extraRules, + extraIgnorePatterns, +}) { + return [ + { + ignores: [ + "**/node_modules/**/*", + "dist/**", + "**/webpack.config.ts", + "**/bin/**", + "**/*.d.ts", + "**/*.sh", + "*.js", + "src/scripts/code_templates/*", + "examples/**", + ...(extraIgnorePatterns ?? []), + ], + }, + eslint.configs.recommended, + toFlatConfig(tseslint, "@typescript-eslint"), + eslintPluginImport.flatConfigs.recommended, + toFlatConfig(eslintPluginLodash, "lodash"), + eslintPluginNode.configs["flat/recommended"], + { + files: ["**/*.ts", "**/*.tsx", "**/*.jsx"], + languageOptions: { + globals: { + ...globals.node, + ...browserGlobals, + ...globals.es6, + ...globals.jest, + Atomics: "readonly", + SharedArrayBuffer: "readonly", + NodeJS: "readonly", // For NodeJS.Timeout, NodeJS.Process, etc. + }, + parser: eslintParser, + parserOptions: { + ecmaFeatures: { + jsx: true, + }, + ecmaVersion: 2018, + sourceType: "module", + tsconfigRootDir: projectRoot, + project: eslintTsConfig, + warnOnUnsupportedTypeScriptVersion: false, + // Reuse editor TS service and slim project to reduce memory and speed up. + EXPERIMENTAL_useProjectService: true, + }, + }, + + plugins: { + eslint, + "@typescript-eslint": tseslint, + lodash: eslintPluginLodash, + "no-only-tests": eslintPluginNoOnlyTests, + node: eslintPluginNode, + "typescript-enum": eslintPluginTypescriptEnum, + "unused-imports": eslintPluginUnusedImports, + stylistic, + local: localRules, + }, + settings: { + react: { + version: "detect", + }, + "import/parsers": { + "@typescript-eslint/parser": [".ts", ".tsx"], + }, + "import/resolver": { + typescript: { + project: eslintTsConfig, + alwaysTryTypes: false, + }, + node: { + extensions: [".js", ".jsx", ".ts", ".tsx"], + }, + }, + }, + rules: { + // TODO: slowly enable no-extraneous-dependencies rule below. For now, it's + // enforced only for some packages. + // + // In an ideal world, the root package.json should have 0 dependencies, and + // all packages/* should define their own dependencies by themselves, + // independently and locally. The rule below is exactly for that: it ensures + // that all package's dependencies are explicitly mentioned in its + // package.json, and no dependencies are borrowed implicitly from the root + // node_modules. + // + // In real life though, enforcing packages independency is dangerous: we may + // e.g. start accidentally bundle 2 React or 2 Redux versions if we forget + // to sync their versions in different monorepo packages' package.json + // files. (There must be some other lint rule for this hopefully.) + // + // In all cases, we should treat node_modules folders content as something + // secondary and transient. (It's true even now with the new "yarn + // Plug-n-Play" technology which we don't use yet.) The source of truth is + // always package.json (enforced by lint) and yarn.lock (defines the exact + // contents of all node_modules folders, bit by bit). In this schema, it + // doesn't matter at all, does yarn use hoisting or not. + // + // "import/no-extraneous-dependencies": "error"; + "arrow-body-style": ["error", "as-needed"], + + "lodash/collection-ordering": "off", + "lodash/preferred-alias": "off", + "lodash/chaining": "off", + // Lodash matches property syntax is not typesafe. + "lodash/matches-prop-shorthand": ["warn", "never"], + "lodash/prefer-constant": "off", + "lodash/prefer-flat-map": "off", + "lodash/prefer-immutable-method": "off", + "lodash/prefer-includes": "warn", + "lodash/prefer-reject": "off", + "lodash/identity-shorthand": "off", + "lodash/prefer-is-nil": "off", + "lodash/prefer-lodash-chain": "off", + "lodash/matches-shorthand": "warn", + "lodash/prefer-matches": "off", + "lodash/prefer-noop": "off", + "lodash/prefer-thru": "warn", + "lodash/prefer-startswith": "off", + "lodash/prefer-lodash-method": "off", + "lodash/prefer-lodash-typecheck": "off", + "lodash/prop-shorthand": "off", + "lodash/import-scope": "off", // Doesn't matter much in node.js. + "lodash/unwrap": "off", + + "node/prefer-global/process": "error", + "node/prefer-global/console": "error", + "node/prefer-global/buffer": "error", + "node/prefer-global/url-search-params": "error", + "node/prefer-global/url": "error", + + // Disable node plugin import checking for TypeScript files since TypeScript handles this. + "n/no-deprecated-api": "warn", + "n/no-missing-import": "off", + "n/no-missing-require": "off", + "n/no-extraneous-import": "off", + "n/no-extraneous-require": "off", + "n/no-process-exit": "off", + "n/hashbang": "warn", + "n/no-unsupported-features/es-syntax": "off", + "n/no-unsupported-features/node-builtins": "off", + "n/no-unpublished-import": "off", + + "require-atomic-updates": "off", + "no-prototype-builtins": "off", + + "@typescript-eslint/no-misused-promises": "error", + "@typescript-eslint/promise-function-async": "error", + "@typescript-eslint/await-thenable": "error", + "@typescript-eslint/no-floating-promises": [ + "error", + { ignoreVoid: false }, + ], + "@typescript-eslint/no-namespace": "off", + "@typescript-eslint/no-this-alias": "off", + "@typescript-eslint/no-unsafe-assignment": "off", + "@typescript-eslint/unbound-method": ["error", { ignoreStatic: true }], + "@typescript-eslint/array-type": ["error", { default: "array-simple" }], + "@typescript-eslint/ban-ts-comment": ["error"], + "@typescript-eslint/no-non-null-asserted-optional-chain": "off", + "@typescript-eslint/no-useless-constructor": ["error"], + "@typescript-eslint/no-empty-object-type": "off", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-require-imports": "off", + "@typescript-eslint/no-unused-expressions": "off", + "@typescript-eslint/no-unsafe-function-type": "off", + "@typescript-eslint/prefer-optional-chain": ["warn"], + "@typescript-eslint/consistent-type-imports": ["error"], + "@typescript-eslint/return-await": "warn", + "@typescript-eslint/require-array-sort-compare": [ + "error", + { ignoreStringArrays: true }, + ], + "@typescript-eslint/use-unknown-in-catch-callback-variable": "warn", + + eqeqeq: ["error"], + "object-shorthand": ["error", "always"], + + "typescript-enum/no-const-enum": ["error"], // not supported in SWC + "typescript-enum/no-enum": "off", + + "@typescript-eslint/naming-convention": [ + "error", + { + selector: "variable", + format: ["camelCase", "PascalCase", "UPPER_CASE"], + leadingUnderscore: "allow", + trailingUnderscore: "allow", + filter: { + regex: "^__webpack", + match: false, + }, + }, + ], + + // Disable in favour of @typescript-eslint/no-unused-vars. + "no-unused-vars": "off", + "@typescript-eslint/no-unused-vars": [ + "error", + { + args: "all", + vars: "local", // Only check local variables, ignore imports used only as types. + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + caughtErrorsIgnorePattern: "^_", + destructuredArrayIgnorePattern: "^_", + ignoreRestSiblings: true, + }, + ], + + // Disable in favour of @typescript-eslint/no-redeclare which understands TS namespaces + "no-redeclare": "off", + "@typescript-eslint/no-redeclare": "off", + "@typescript-eslint/member-ordering": [ + "error", + { + // + // ATTENTION: the rules here are not simple, mainly because of this: + // https://github.com/typescript-eslint/typescript-eslint/issues/6133 + // + // Besides that, we also want contradictory things, like: + // + // 1. Having constructor close to fields definition (because people + // often define fields in the constructor arguments), although it + // logically should've been below static methods. + // 2. Having all abstract things in the class grouped, irregardless on + // their public/protected/private modifiers. + // + default: [ + "signature", + "call-signature", + + // Typically, class constants (that's why they're on top). + "public-static-field", + "public-static-get", + "public-static-set", + "protected-static-field", + "protected-static-get", + "protected-static-set", + + // All concrete fields. What's interesting is that the order we + // emotionally want here for properties is private-protected-public, + // which is the opposite to the order of methods (which is + // public-protected-private). This is likely because the methods are + // bulky, and properties are lean. + "private-static-field", + "private-instance-field", + "public-instance-field", + "public-abstract-field", + "public-abstract-get", + "public-abstract-set", + + // Protected fields and methods are grouped, because eslint currently + // doesn't distinguish fields assigned with a lambda FROM methods, and + // we often times expose abstract protected overridable lambdas: + // https://github.com/typescript-eslint/typescript-eslint/issues/6133 + "protected-abstract-field", + "protected-abstract-get", + "protected-abstract-set", + "protected-abstract-method", + "public-abstract-method", // the only exception; it's to group all abstract things too + "protected-instance-field", + "protected-constructor", + "protected-static-method", + "protected-instance-get", + "protected-instance-set", + "protected-instance-method", + + // Public constructor, instance methods, static methods. + "public-constructor", // often defines more public/protected/private properties, so should be close to fields + "public-static-method", + "public-instance-get", + "public-instance-set", + "public-instance-method", + + // Private constructor, instance methods, static methods. + "private-constructor", + "private-static-method", + "private-instance-get", + "private-instance-set", + "private-instance-method", + "private-static-get", + "private-static-set", + ], + }, + ], + + "no-constant-condition": ["error", { checkLoops: false }], + "no-buffer-constructor": ["error"], + "no-console": ["error"], + curly: ["error", "all"], + "no-case-declarations": "off", + + "padding-line-between-statements": "off", + "stylistic/padding-line-between-statements": [ + "error", + // Force empty lines. + { + blankLine: "always", + prev: [ + "block", + "block-like", + "function", + "class", + "interface", + "type", + ], + next: "*", + }, + { + blankLine: "always", + prev: "import", + next: [ + "const", + "if", + "let", + "var", + "export", + "function", + "class", + "interface", + "type", + ], + }, + { + blankLine: "always", + prev: "*", + next: ["function", "class", "interface", "type"], + }, + // Allow one-liner functions without extra spacing (hacky): + { blankLine: "any", prev: "singleline-const", next: "*" }, + { blankLine: "any", prev: "singleline-var", next: "*" }, + { blankLine: "any", prev: "singleline-let", next: "*" }, + ], + + "no-restricted-properties": [ + "error", + { + object: "window", + property: "location", + message: + "We use React Router and History to control the location of our web or desktop app. Prefer `useLocation` in React components and `historyFromContext` in Redux Saga.", + }, + ...(projectRoot.endsWith("client") + ? [ + { + object: "window", + property: "document", + message: "Please use `useDocument` from `useDocument.tsx`.", + }, + ...[ + "addEventListener", + "removeEventListener", + "getElementById", + "documentElement", + "activeElement", + "querySelectorAll", + ].map((property) => ({ + object: "document", + property, + message: "Please use `useDocument` from `useDocument.tsx`.", + })), + ] + : []), + ], + + "no-restricted-globals": [ + "warn", + { + name: "location", + message: + "We use React Router and History to control the location of our web or desktop app. Prefer `useLocation` in React components and `historyFromContext` in Redux Saga.", + }, + ], + + "no-restricted-syntax": [ + "error", + { + selector: (() => { + const RE_BAD = "/([a-z0-9_ ]|^)E[Ii][Dd]|(^|[-_: ])eid/"; + return [ + `Identifier[name=${RE_BAD}]`, + `Literal[value=${RE_BAD}]`, + `TemplateElement[value.raw=${RE_BAD}]`, + `TSInterfaceDeclaration[id.name=${RE_BAD}]`, + ].join(","); + })(), + message: + 'Do not use "eid" or "EID" as a part of a name/field/type. Instead, prefer externalID or external_id.', + }, + ], + + "prefer-const": [ + "error", + { + destructuring: "all", + }, + ], + + "no-var": "error", + "no-void": "error", + + "no-sequences": ["error"], + "no-undef": "off", + // Too noisy about `react` and other node_modules + "import/default": 0, + // This complains about React.forwardRef, ReactDOM.render, etc. + "import/no-named-as-default-member": 0, + // This complains about "apollo" exporting ApolloClient as a default and as a + // named import at the same time. + "import/no-named-as-default": 0, + // Does not seem to work well with node_modules + "import/named": 0, + "import/newline-after-import": "error", + "import/order": [ + "error", + { + groups: ["builtin", "external", "index", "parent", "sibling"], + pathGroups: [ + { + pattern: "./**.module.css", + group: "sibling", + position: "after", + }, + { + pattern: "./**.module.scss", + group: "sibling", + position: "after", + }, + ], + alphabetize: { + order: "asc", + caseInsensitive: true, + }, + }, + ], + "import/no-useless-path-segments": ["error", { noUselessIndex: true }], + "unused-imports/no-unused-imports": "error", + "no-restricted-imports": [ + "error", + { + patterns: [ + { + group: ["react-router"], + message: + "Please use react-router-dom instead, since react-router's useLocation() doesn't work properly with StaticRouter on server side.", + }, + { + group: ["braintrust/loadPrompt"], + message: + "You should use DynamicPromptsRepository to load prompts from braintrust.", + }, + ], + }, + ], + // Fixes a common mistake: `a ?? b < c` which feels like `(a ?? b) < c`, but + // actually is `a ?? (b < c)` + "no-mixed-operators": [ + "error", + { + allowSamePrecedence: false, + groups: [ + ["??", "+"], + ["??", "-"], + ["??", "*"], + ["??", "/"], + ["??", "%"], + ["??", "**"], + ["??", "&"], + ["??", "|"], + ["??", "^"], + ["??", "~"], + ["??", "<<"], + ["??", ">>"], + ["??", ">>>"], + ["??", "=="], + ["??", "!="], + ["??", "==="], + ["??", "!=="], + ["??", ">"], + ["??", ">="], + ["??", "<"], + ["??", "<="], + ["??", "&&"], + ["??", "||"], + ["??", "in"], + ["??", "instanceof"], + ], + }, + ], + + quotes: ["error", "double", { avoidEscape: true }], + + "no-only-tests/no-only-tests": "error", + + "local/zod-prefer-safe-optional": "warn", + + ...extraRules, + }, + }, + { + files: ["**/*.js"], + languageOptions: { + ecmaVersion: 2022, + }, + rules: { + "n/hashbang": "off", + "n/no-unsupported-features/node-builtins": "off", + "n/no-extraneous-import": "off", + "@typescript-eslint/no-require-imports": "off", + "lodash/prefer-lodash-method": "off", + "n/no-process-exit": "off", + "n/no-missing-require": "off", + }, + }, + { + files: ["**/*.mjs"], + languageOptions: { + sourceType: "module", + ecmaVersion: 2020, + }, + settings: { + "import/core-modules": ["@stylistic/eslint-plugin", "@eslint/js"], + }, + rules: { + "n/hashbang": "off", + "n/no-extraneous-import": "off", + "n/no-unsupported-features/es-syntax": "off", + "n/no-unpublished-import": "off", + "n/no-process-exit": "off", + "lodash/prefer-lodash-method": "off", + // Disable import plugin checks for config files to avoid CJS/ESM export parsing issues + "import/default": "off", + "import/namespace": "off", + "import/no-unresolved": "off", + "import/no-named-as-default": "off", + "import/no-named-as-default-member": "off", + }, + }, + // Disable zod-prefer-safe-optional for braintrust/eval files since those + // schemas are for validating test configuration, not for guiding LLM output + { + files: [ + "**/scripts/evals/**/*.ts", + "**/scripts/evals/**/*.tsx", + "**/braintrust/**/*.ts", + "**/braintrust/**/*.tsx", + ], + rules: { + "local/zod-prefer-safe-optional": "off", + "local/comment-punctuation": "off", // Otherwise, it adds a period after every multiline // commit line. + }, + }, + ]; +} diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..4bd1102 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,20 @@ +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import createConfig from "./eslint.base.config.mjs"; + +const filename = fileURLToPath(import.meta.url); +const foldername = dirname(filename); + +export default createConfig({ + projectRoot: foldername, + eslintTsConfig: join(foldername, "tsconfig.json"), + extraRules: { + "import/no-extraneous-dependencies": "error", + "@typescript-eslint/explicit-function-return-type": [ + "error", + { allowExpressions: true, allowedNames: ["configure"] }, + ], + "lodash/import-scope": ["error", "method"], + "@typescript-eslint/no-explicit-any": "error", + }, +}); diff --git a/gitbook/README.md b/gitbook/README.md index d71b097..2ef1417 100644 --- a/gitbook/README.md +++ b/gitbook/README.md @@ -5,8 +5,8 @@ The TypeScript library for working with microsharded PostgreSQL databases. * [Getting Started and Tutorials](https://ent-framework.net) -* [API documentation](https://github.com/clickup/ent-framework/blob/main/docs/globals.md) -* [Source code](https://github.com/clickup/ent-framework) +* [API documentation](https://github.com/dimikot/ent-framework/blob/main/docs/globals.md) +* [Source code](https://github.com/dimikot/ent-framework/tree/main/src) * [Ent Framework's Discord](https://discord.gg/QXvN6VTCKS) #### Core Features diff --git a/index.d.ts b/index.d.ts index 5c32411..2002852 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1 +1,5 @@ -export * from "@clickup/ent-framework"; +export * from "./dist/abstract"; +export * from "./dist/ent"; +export * from "./dist/helpers"; +export * from "./dist/tools"; +export * from "./dist/types"; diff --git a/index.js b/index.js index fdf9119..ae23037 100644 --- a/index.js +++ b/index.js @@ -1 +1,8 @@ -module.exports = require("@clickup/ent-framework"); +"use strict"; +module.exports = { + ...require("./dist/abstract"), + ...require("./dist/ent"), + ...require("./dist/helpers"), + ...require("./dist/tools"), + ...require("./dist/types"), +}; diff --git a/internal/build.sh b/internal/build.sh new file mode 100755 index 0000000..a8e4474 --- /dev/null +++ b/internal/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +cat gitbook/README.md | sed 's/\.gitbook/gitbook\/.gitbook/' > README.md +internal/fix-svg-fonts.js +internal/docs.sh +internal/llms.sh + +cat tsconfig.json | sed 's/"references"/"references-off"/'> tsconfig.json.tmp +trap 'rm -f tsconfig.json.tmp' EXIT +tsgo -p tsconfig.json.tmp diff --git a/internal/clean.sh b/internal/clean.sh index dfef516..c50cecc 100755 --- a/internal/clean.sh +++ b/internal/clean.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -e -x - -rm -rf node_modules package-lock.json yarn.lock pnpm-lock.yaml +set -e +rm -rf dist dist.* yarn.lock package-lock.json pnpm-lock.yaml node_modules ./*.log diff --git a/internal/deploy.sh b/internal/deploy.sh index 23e9071..5f80508 100755 --- a/internal/deploy.sh +++ b/internal/deploy.sh @@ -1,10 +1,8 @@ #!/bin/bash set -e -x -git pull --rebase - -cat gitbook/README.md | sed '1,/# / d' | sed 's/\.gitbook/gitbook\/.gitbook/' > README.md -git add README.md - -internal/update-package-json.js -internal/fix-svg-fonts.js +npm run clean +npm install +npm run test +npm run lint +npm publish --access=public diff --git a/internal/dev.sh b/internal/dev.sh new file mode 100755 index 0000000..a0d2495 --- /dev/null +++ b/internal/dev.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# Inject current directory name to the 1st argument of command line (useful for +# "yarn pstree" output in a monorepo). +if [[ "${1:0:1}" == "{" ]]; then + shift +else + exec ${BASH_SOURCE[0]} "{$(basename "$PWD")}" "$@" +fi + +if [[ "$TS_BUILD_MODE_IS_WATCHING_AT_MONOREPO_LEVEL" != "1" ]]; then + tsgo --watch --preserveWatchOutput +fi diff --git a/internal/docs.sh b/internal/docs.sh new file mode 100755 index 0000000..947532a --- /dev/null +++ b/internal/docs.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +rm -rf docs +typedoc --plugin typedoc-plugin-markdown --plugin typedoc-plugin-merge-modules diff --git a/internal/init-db.sh b/internal/init-db.sh new file mode 100755 index 0000000..953aac8 --- /dev/null +++ b/internal/init-db.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +LOG="db-init.log" + +rm -f $LOG +trap "rm -f $LOG" EXIT + +for schema in sh000{0..4}; do + PG_ID_ENV_NO=1 psql -v ON_ERROR_STOP=on \ + -c "SET client_min_messages TO WARNING; CREATE SCHEMA IF NOT EXISTS $schema; SET search_path TO $schema" \ + -f "$(npm root)/@clickup/pg-id/pg-id-up.sql" >> $LOG 2>&1 \ + || { cat $LOG; exit 1; } +done diff --git a/internal/lint.sh b/internal/lint.sh new file mode 100755 index 0000000..8815fd6 --- /dev/null +++ b/internal/lint.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +eslint . --ext .ts --cache --cache-location dist/.eslintcache diff --git a/internal/llms.sh b/internal/llms.sh new file mode 100755 index 0000000..61338d2 --- /dev/null +++ b/internal/llms.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Fetches the GitBook-generated llms-full.txt and stores it in docs/llms-full.md +# so it ships alongside the typedoc output. +set -euo pipefail + +SOURCE_URL="https://docs.ent-framework.net/llms-full.txt" +OUTPUT_FILE="docs/llms-full.md" +TEMP_FILE="$OUTPUT_FILE.tmp" + +mkdir -p "$(dirname "$OUTPUT_FILE")" + +echo "Fetching $SOURCE_URL ..." +curl -fsSL "$SOURCE_URL" -o "$TEMP_FILE" + +if [ ! -s "$TEMP_FILE" ]; then + rm -f "$TEMP_FILE" + echo "ERROR: Fetched content is empty. Aborting." >&2 + exit 1 +fi + +mv "$TEMP_FILE" "$OUTPUT_FILE" +echo "Written to $OUTPUT_FILE ($(wc -l < "$OUTPUT_FILE" | tr -d ' ') lines)" diff --git a/internal/test.sh b/internal/test.sh new file mode 100755 index 0000000..4bd96a0 --- /dev/null +++ b/internal/test.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +internal/build.sh +jest "$@" diff --git a/internal/update-package-json.js b/internal/update-package-json.js deleted file mode 100755 index 4eee2b2..0000000 --- a/internal/update-package-json.js +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env node -const { execSync } = require("child_process"); -const { writeFileSync } = require("fs"); - -function run(command) { - console.log(`$ ${command}`); - execSync(command, { stdio: "inherit" }); -} - -async function updatePackageJson(org, name) { - const packageJsonPath = `${__dirname}/../package.json`; - const npmVersion = execSync(`npm view ${name} version`).toString().trim(); - - const orgPackageJson = await fetch( - `https://raw.githubusercontent.com/${org}/${name}/refs/heads/main/package.json` - ).then((res) => res.json()); - - const packageJson = require(packageJsonPath); - packageJson.name = name; - packageJson.description = orgPackageJson.description; - packageJson.version = orgPackageJson.version; - packageJson.keywords = orgPackageJson.keywords; - packageJson.dependencies = { - [`@${org}/${name}`]: orgPackageJson.version, - }; - writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2) + "\n"); - - run("git add package.json"); - - let commitError = null; - try { - run(`git commit -m "v${packageJson.version}"`); - } catch (e) { - commitError = e; - } - - if (!commitError) { - run("git push"); - } - - if (packageJson.version !== npmVersion) { - run("npm publish"); - } -} - -updatePackageJson("clickup", "ent-framework").catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/internal/with-docker-compose-up.postgres.sh b/internal/with-docker-compose-up.postgres.sh new file mode 100755 index 0000000..cdfb8fe --- /dev/null +++ b/internal/with-docker-compose-up.postgres.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e + +docker compose -f docker-compose.postgres.yml up --quiet-pull -d + +for _i in $(seq 1 20); do + docker compose -f docker-compose.postgres.yml ps | grep -q healthy && break + sleep 1 +done + +export PGHOST=127.0.0.1 +export PGDATABASE=postgres +export PGUSER=postgres +export PGPASSWORD=postgres + +# Run "$1" as a bash command (possibly with && in it) and pass $2, $3, ... as +# properly quoted arguments to it. +cmd="$1" +shift +if [[ "$*" == "" ]]; then + eval "$cmd" +else + eval "$cmd \"\$@\"" +fi diff --git a/jest.config.base.js b/jest.config.base.js new file mode 100644 index 0000000..90408ec --- /dev/null +++ b/jest.config.base.js @@ -0,0 +1,13 @@ +"use strict"; +module.exports = { + roots: ["/src"], + testMatch: ["**/*.test.ts"], + clearMocks: true, + restoreMocks: true, + ...(process.env.IN_JEST_PROJECT + ? {} + : { forceExit: true, testTimeout: 30000 }), + transform: { + "\\.ts$": ["ts-jest", { diagnostics: { ignoreCodes: [151002] } }], + }, +}; diff --git a/jest.config.js b/jest.config.js new file mode 100644 index 0000000..208e748 --- /dev/null +++ b/jest.config.js @@ -0,0 +1,2 @@ +"use strict"; +module.exports = { ...require("./jest.config.base") }; diff --git a/package.json b/package.json index 1f2cbb4..6f16523 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "ent-framework", "description": "A PostgreSQL graph-database-alike library with microsharding and row-level security", - "version": "2.26.1", + "version": "3.0.1", "license": "MIT", "keywords": [ "postgresql", @@ -22,15 +22,65 @@ "./pg": "./pg.js" }, "scripts": { + "build": "barrelsby -c barrelsby.json && internal/build.sh", + "dev": "barrelsby -c barrelsby.json && internal/dev.sh", + "lint": "internal/lint.sh", + "test": "PGPORT=54832 internal/with-docker-compose-up.postgres.sh 'bash ./internal/init-db.sh && internal/test.sh'", + "psql": "PGPORT=54832 internal/with-docker-compose-up.postgres.sh psql", "clean": "internal/clean.sh", + "copy-package-to-public-dir": "copy-package-to-public-dir.sh", + "backport-package-from-public-dir": "backport-package-from-public-dir.sh", "deploy": "internal/deploy.sh" }, "dependencies": { - "@clickup/ent-framework": "2.26.1" + "chalk": "^4.1.2", + "delay": "^4.4.1", + "fast-typescript-memoize": "^1.2.1", + "lodash": "^4.17.21", + "object-hash": "^2.2.0", + "p-defer": "^3.0.0", + "p-timeout": "^3.2.0", + "pg": "^8.7.1", + "quick-lru": "^5.1.1", + "table": "^6.8.0" + }, + "devDependencies": { + "@clickup/pg-id": "^2.16.1", + "@eslint/js": "^9.35.0", + "@stylistic/eslint-plugin": "^5.3.1", + "@types/jest": "^29.5.5", + "@types/lodash": "^4.14.175", + "@types/node": "^20.4.1", + "@types/object-hash": "^2.2.1", + "@types/pg": "^8.11.11", + "@types/uniqid": "^4.1.3", + "@typescript-eslint/eslint-plugin": "^8.43.0", + "@typescript-eslint/parser": "^8.43.0", + "@typescript/native-preview": "^7.0.0-dev.20260305.1", + "barrelsby": "^2.8.1", + "eslint-import-resolver-typescript": "^4.4.4", + "eslint-plugin-import": "^2.32.0", + "eslint-plugin-lodash": "^8.0.0", + "eslint-plugin-n": "^17.21.3", + "eslint-plugin-no-only-tests": "^3.3.0", + "eslint-plugin-react-hooks": "^5.2.0", + "eslint-plugin-react": "^7.37.5", + "eslint-plugin-typescript-enum": "^2.1.0", + "eslint-plugin-unused-imports": "^4.2.0", + "eslint": "^9.35.0", + "globals": "^17.0.0", + "jest": "^29.7.0", + "prettier": "3.8.1", + "streaming-iterables": "^7.1.0", + "ts-jest": "^29.1.1", + "typedoc-plugin-markdown": "4.4.2", + "typedoc-plugin-merge-modules": "6.1.0", + "typedoc": "0.27.5", + "typescript": "^5.2.2", + "wait-for-expect": "^3.0.2" }, "repository": { "type": "git", "url": "git://github.com/dimikot/ent-framework.git" - }, - "packageManager": "pnpm@7.25.1+sha1.1ef0ac6acb8a8d4f726f8321f900a151111f7387" + } } diff --git a/pg-id.config.sql b/pg-id.config.sql new file mode 100644 index 0000000..8e00a4c --- /dev/null +++ b/pg-id.config.sql @@ -0,0 +1,7 @@ +-- Example constants to run Ent Framework unit tests. +\set CONST_ENV_MUL 10 +\set CONST_SHARD_MUL 10000 +\set CONST_RND_TS_START 1577836800 +\set CONST_MUL 17141763000000 +\set CONST_SUM 13795571000000 +\set CONST_MOD 19458232000000 diff --git a/pg.d.ts b/pg.d.ts index 8414577..b0dd9d6 100644 --- a/pg.d.ts +++ b/pg.d.ts @@ -1 +1 @@ -export * from "@clickup/ent-framework/pg"; +export * from "./dist/pg"; diff --git a/pg.js b/pg.js index 1cf0a44..16a3e37 100644 --- a/pg.js +++ b/pg.js @@ -1 +1,2 @@ -module.exports = require("@clickup/ent-framework/pg"); +"use strict"; +module.exports = require("./dist/pg"); diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..30d1c6d --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + unrs-resolver: false diff --git a/src/__tests__/types.test.ts b/src/__tests__/types.test.ts new file mode 100644 index 0000000..c651796 --- /dev/null +++ b/src/__tests__/types.test.ts @@ -0,0 +1,42 @@ +import { testSpecTypeIntegrity } from "../helpers/testSpecTypeIntegrity"; +import { Base64BufferType, EnumType, JSONType } from "../types"; + +enum SizeStr { + ONE = "one", +} + +enum SizeNum { + ONE = 1, +} + +test("EnumType", () => { + expect(testSpecTypeIntegrity(EnumType<"a">(), "a")).toEqual({ + jsValueDecoded: "a", + stringifiedBack: "a", + }); + expect(testSpecTypeIntegrity(EnumType(), SizeStr.ONE)).toEqual({ + jsValueDecoded: SizeStr.ONE, + stringifiedBack: "one", + }); + expect(testSpecTypeIntegrity(EnumType(), SizeNum.ONE)).toEqual({ + // The value comes as a number from the DB. + jsValueDecoded: SizeNum.ONE, + // This is a quirk: for numeric enums, stringify() returns the numeric value + // and not a string, since this is what we expect to be stored in the DB. + stringifiedBack: 1, + }); +}); + +test("Base64BufferType", () => { + expect(testSpecTypeIntegrity(Base64BufferType(), "3q2+7w==")).toEqual({ + jsValueDecoded: Buffer.from([222, 173, 190, 239]), + stringifiedBack: "3q2+7w==", + }); +}); + +test("JSONType", () => { + expect(testSpecTypeIntegrity(JSONType<{ a: string }>(), { a: "b" })).toEqual({ + jsValueDecoded: { a: "b" }, + stringifiedBack: '{"a":"b"}', + }); +}); diff --git a/src/abstract/Batcher.ts b/src/abstract/Batcher.ts new file mode 100644 index 0000000..c707b4d --- /dev/null +++ b/src/abstract/Batcher.ts @@ -0,0 +1,185 @@ +import delay from "delay"; +import type { DeferredPromise } from "p-defer"; +import pDefer from "p-defer"; +import { DefaultMap } from "../internal/DefaultMap"; +import type { MaybeCallable } from "../internal/misc"; +import { appendCaller, maybeCall, runInVoid } from "../internal/misc"; +import type { QueryAnnotation } from "./QueryAnnotation"; +import type { Runner } from "./Runner"; + +/** + * Batcher is similar to DataLoader, but with a few important differences: + * 1. It's strongly typed not only for the output, but for input too. And input + * can be arbitrary, not only strings (e.g. rows). + * 2. It does requests dedupping for all queries (including selects). + * 3. It's not limited by read-only requests like DataLoader, and thus it + * doesn't to any caching. Caching is delegated to some other layer (either + * above Batcher or in Runner). + */ +export class Batcher { + // key -> input + private queuedInputs = new Map(); + + // key -> DeferredPromise[] + private queuedDefers = new DefaultMap< + string, + Array> + >(); + + // Dedupped annotations; each annotation identifies a caller of the query. + private queuedAnnotations = new Map(); + + protected flushQueue = async (): Promise => { + if (!this.queuedInputs.size) { + return; + } + + const inputs = this.queuedInputs; + const defers = this.queuedDefers; + const annotations = [...this.queuedAnnotations.values()]; + this.queuedInputs = new Map(); + this.queuedDefers = new DefaultMap(); + this.queuedAnnotations = new Map(); + + let outputs = new Map(); + const errors = new Map(); + + if (inputs.size === 1 || !this.runner.runBatch || this.disableBatching) { + // Relatively rare since most of the requests come batched. + await this.runSingleForEach(inputs, annotations, outputs, errors); + } else { + // Called most of the times. + try { + outputs = await this.runner.runBatch(inputs, annotations); + } catch (e: unknown) { + // Relatively rare under heavy load (since errors are rare). + if (this.runner.shouldDebatchOnError(e)) { + await this.runSingleForEach( + inputs, + incrementAttempt(annotations), + outputs, + errors, + ); + } else { + for (const key of defers.keys()) { + errors.set(key, e); + } + } + } + } + + for (const [key, defersOfKey] of defers.entries()) { + const error = errors.get(key); + const output = outputs.get(key); + if (error === undefined) { + const outputOrDefault = + output === undefined ? this.runner.default : output; + for (const defer of defersOfKey) { + // There are typically multiple callers waiting for the query results + // (due to e.g. same-ID queries coalescing). + defer.resolve(outputOrDefault); + } + } else { + for (const defer of defersOfKey) { + defer.reject(error); + } + } + } + }; + + constructor( + private runner: Runner, + private batchDelayMs: MaybeCallable, + private disableBatching: boolean, + ) {} + + async run(input: TInput, annotation: QueryAnnotation): Promise { + const key = this.runner.key(input); + const delay = maybeCall(this.batchDelayMs); + + // Queue return promise of this method. + const defer = pDefer(); + this.queuedDefers.getOrAdd(key, Array).push(defer); + + // In case of dedupping by key, prefer the last value. E.g. if 2 UPDATEs + // for the same ID have different values, then the last one will win, not + // the 1st one. + this.queuedInputs.set(key, input); + + // Annotations are dedupped by their content. + this.queuedAnnotations.set( + annotation.trace + + annotation.vc + + annotation.debugStack + + annotation.whyClient, + annotation, + ); + + if ( + this.queuedInputs.size >= this.runner.maxBatchSize || + !this.runner.runBatch || + this.disableBatching + ) { + runInVoid(this.flushQueue); + } else if (this.queuedInputs.size === 1) { + // Defer calling of flushQueue() to the "end of the event loop's spin", to + // have a chance to collect more run() calls for it to execute. We + // actually defer twice (to the end of microtasks sub-loop and then once + // again), just in case: the original DataLoader library wraps the + // nextTick() call into a "global resolved Promise" object, so we do the + // same here blindly. See some of details here: + // https://github.com/graphql/dataloader/blob/fae38f14702e925d1e59051d7e5cb3a9a78bfde8/src/index.js#L234-L241 + // https://stackoverflow.com/a/27648394 + runInVoid( + Promise.resolve().then(() => + delay > 0 + ? setTimeout(() => runInVoid(this.flushQueue()), delay) + : process.nextTick(this.flushQueue), + ), + ); + } + + return defer.promise.catch((e: unknown) => { + throw appendCaller(e); + }) as Promise; + } + + private async runSingleForEach( + inputs: Map, + annotations: QueryAnnotation[], + outOutputs: Map, + outErrors: Map, + ): Promise { + const promises: Array> = []; + for (const [key, input] of inputs) { + promises.push( + this.runner + .runSingle(input, annotations) + .catch(async (error: unknown) => { + const retryMs = this.runner.delayForSingleQueryRetryOnError(error); + + if (typeof retryMs === "number") { + await delay(retryMs); + } + + if (retryMs !== "no_retry") { + return this.runner.runSingle( + input, + incrementAttempt(annotations), + ); + } + + throw error; + }) + .then((output) => outOutputs.set(key, output)) + .catch((error: unknown) => outErrors.set(key, error)), + ); + } + + await Promise["all"](promises); + } +} + +function incrementAttempt(annotations: QueryAnnotation[]): QueryAnnotation[] { + return annotations.map((a) => ({ ...a, attempt: a.attempt + 1 })); +} diff --git a/src/abstract/Client.ts b/src/abstract/Client.ts new file mode 100644 index 0000000..4e324fa --- /dev/null +++ b/src/abstract/Client.ts @@ -0,0 +1,215 @@ +import { Memoize } from "fast-typescript-memoize"; +import defaults from "lodash/defaults"; +import type { MaybeCallable, PickPartial } from "../internal/misc"; +import { objectId } from "../internal/objectId"; +import type { Table } from "../types"; +import { Batcher } from "./Batcher"; +import type { ClientErrorKind, ClientErrorPostAction } from "./ClientError"; +import type { Loggers, SwallowedErrorLoggerProps } from "./Loggers"; +import type { QueryAnnotation } from "./QueryAnnotation"; +import type { Runner } from "./Runner"; +import type { Schema } from "./Schema"; +import type { ShardNamer } from "./ShardNamer"; +import type { TimelineManager } from "./TimelineManager"; + +/** + * Options for Client constructor. + */ +export interface ClientOptions { + /** Name of the Client; used for logging. */ + name: string; + /** Info on how to build/parse Shard names. If not set, then Cluster injects + * its own ShardNamer here right after creating a Client instance. */ + shardNamer?: ShardNamer | null; + /** Loggers to be called at different stages. Client code calls into them. + * Also, Cluster injects its own loggers here, in addition to the provided + * ones (if any). */ + loggers?: Loggers | null; + /** If passed, there will be an artificial queries accumulation delay while + * batching the requests. Default is 0 (turned off). Passed to + * Batcher#batchDelayMs. */ + batchDelayMs?: MaybeCallable; +} + +/** + * Role of the Client as reported after the last successful query. If we know + * for sure that the Client is a master or a replica, the role will be "master" + * or "replica" correspondingly. If no queries were run by the Client yet (i.e. + * we don't know the role for sure), the role is assigned to "unknown". + */ +export type ClientRole = "master" | "replica" | "unknown"; + +/** + * An information about Client's connection related issue. + */ +export interface ClientConnectionIssue { + timestamp: Date; + cause: unknown; + postAction: ClientErrorPostAction; + kind: ClientErrorKind; + comment: string; +} + +/** + * Input for Client#ping(). + */ +export interface ClientPingInput { + execTimeMs: number; + isWrite: boolean; + annotation: QueryAnnotation; +} + +/** + * Client is a Shard name aware abstraction which sends an actual query and + * tracks the master/replica timeline. The concrete query sending implementation + * (including required arguments) is up to the derived classes. + */ +export abstract class Client { + /** Default values for the constructor options. */ + static readonly DEFAULT_OPTIONS: Required> = { + shardNamer: null, + loggers: null, + batchDelayMs: 0, + }; + + /** Client configuration options. */ + readonly options: Required; + + /** Date when this Client instance was constructed. */ + readonly createdAt: Date; + + /** Each Client may be bound to some Shard, so the queries executed via it + * will be namespaced to this Shard. E.g. in relational databases, Shard name + * may be a namespace (or schema) name (or "public" if the Client wasn't + * created by withShard() method). */ + abstract readonly shardName: string; + + /** Tracks the master/replica replication timeline position. Shared across all + * the Clients within the same Island. */ + abstract readonly timelineManager: TimelineManager; + + /** + * Represents the full destination address this Client is working with. + * Depending on the implementation, it may include hostname, port number, + * database name, shard name etc. It is required that the address is stable + * enough to be able to cache some destination database related metadata (e.g. + * shardNos) based on that address. + */ + abstract address(): string; + + /** + * Gracefully closes the connections to let the caller destroy the Client. The + * pending queries are awaited to finish before returning. The Client becomes + * unusable after calling this method: you should not send queries to it. + */ + abstract end(): Promise; + + /** + * Returns true if the Client is ended and can't be used anymore. + */ + abstract isEnded(): boolean; + + /** + * Returns all Shard numbers discoverable via the connection to the Client's + * database. + */ + abstract shardNos(): Promise; + + /** + * Sends a read or write test query to the server. Tells the server to sit and + * wait for at least the provided number of milliseconds. + */ + abstract ping(input: ClientPingInput): Promise; + + /** + * Creates a new Client which is namespaced to the provided Shard number. The + * new Client will share the same connection pool with the parent's Client. + */ + abstract withShard(no: number): this; + + /** + * Returns the Client's role reported after the last successful query. Master + * and replica roles may switch online unpredictably, without reconnecting, so + * we only know the role after a query. + */ + abstract role(): ClientRole; + + /** + * Returns a non-nullable value if the Client couldn't connect to the server + * (or it could, but the load balancer reported the remote server as not + * working), so it should ideally be removed from the list of active replicas + * until e.g. the next discovery query to it (or any query) succeeds. + */ + abstract connectionIssue(): ClientConnectionIssue | null; + + /** + * Calls swallowedErrorLogger() doing some preliminary amendment. + */ + protected logSwallowedError(props: SwallowedErrorLoggerProps): void { + this.options.loggers?.swallowedErrorLogger({ + ...props, + where: `${this.constructor.name}(${this.options.name}): ${props.where}`, + }); + } + + /** + * Initializes an instance of Client. + */ + constructor(options: ClientOptions) { + this.options = defaults({}, options, Client.DEFAULT_OPTIONS); + this.createdAt = new Date(); + } + + /** + * Batcher is per-Client per-query-type + * per-table-name-and-shape-and-disableBatching: + * + * - Per-Client means that batchers are removed as soon as the Client is + * removed, i.e. the Client owns all the batchers for all tables. + * - Per-query-type means that the batcher for a SELECT query is different + * from the batcher for an INSERT query (obviously). + * - Per-table-name-and-shape-and-disableBatching means that each table has + * its own set of batchers (obviously). Also, some queries may be complex + * (like UPDATE), so the batcher also depends on the "shape" - the list of + * fields we're updating. Plus, for some inputs, we want to disable batching + * at all - that produces a separate Batcher instance. + * + * Also, for every Batcher, there is exactly one Runner (which knows how to + * build the actual query in the context of the current Client). Batchers are + * generic (like DataLoader, but more general), and Runners are very custom to + * the query (and are private to these queries). + * + * All that means that in a 1000-Shard 20-table Cluster we'll eventually have + * 1000x20x8 Batchers/Runners (assuming we have 8 different operations). + */ + @Memoize( + (QueryClass, schema, additionalShape, _, disableBatching) => + `${objectId(QueryClass)}:${objectId(schema)}:${additionalShape}:${disableBatching}`, + ) + batcher( + _QueryClass: Function, + _schema: Schema, + _additionalShape: string, + disableBatching: boolean, + runnerCreator: () => Runner, + ): Batcher { + // At the moment, Runner doesn't depend on the Client. So theoretically we + // could share the same Runner across multiple Batchers (and multiple + // Clients) to save memory (and inject the Client via Runner.run*() + // methods). But we don't do all that right now. + const runner = runnerCreator(); + return new Batcher( + runner, + this.options.batchDelayMs, + disableBatching, + ); + } + + /** + * A convenience method to put connections prewarming logic to. The idea is to + * keep the needed number of open connections and also, in each connection, + * minimize the time which the very 1st query will take (e.g. pre-cache + * full-text dictionaries). + */ + prewarm(): void {} +} diff --git a/src/abstract/ClientError.ts b/src/abstract/ClientError.ts new file mode 100644 index 0000000..66e98bd --- /dev/null +++ b/src/abstract/ClientError.ts @@ -0,0 +1,75 @@ +import type { MaybeError } from "../internal/misc"; +import { addSentenceSuffixes, appendCause } from "../internal/misc"; + +/** + * The suggested action, what can we do when facing a ClientError. + */ +export type ClientErrorPostAction = + /** E.g. a Shard is relocated to another Island. We need to update the + * Island-to-Shards mapping. Run Shards discovery on the entire Cluster before + * retrying. */ + | "rediscover-cluster" + /** E.g. a master node suddenly appears as replica (a switchover happened). We + * don't want to rediscover ALL Islands, we only want to ping Clients of the + * current Island (otherwise, we'd have a combinatorial explosion of + * rediscovery requests everywhere if e.g. an Island with global shards + * experiences an issue). Run shardNos() on all Clients of the current Island + * before retrying. */ + | "rediscover-island" + /** E.g. an attempt to use Client which is end()'ed: trigger a retry which + * will choose another Client. This may happen when e.g. a Client instance is + * returned to the Shards logic, and immediately after that it's been end()'ed + * due to a rediscovery succeeding and recycling the old Clients. We can't + * control the lifetime of Client instances returned to the caller (i.e. there + * is always a chance that the caller will try to use the Client after it's + * been end()'ed), but at least for Shards logic, we are able to retry. Choose + * another (healthy) Client, but don't run rediscovery. */ + | "choose-another-client" + /** Giving up on retries. Do not retry, fail immediately. */ + | "fail"; + +/** + * Sometimes we need to know for sure, is there a chance that the query failed, + * but the write was still applied in the database. + */ +export type ClientErrorKind = + | "data-on-server-is-unchanged" + | "unknown-server-state"; + +/** + * Encapsulates the error thrown when running a Client query. The object also + * carries suggestions, what to do next. + */ +export class ClientError extends Error { + constructor( + public readonly cause: MaybeError, + where: string, + public readonly postAction: ClientErrorPostAction, + public readonly kind: ClientErrorKind, + public readonly abbreviation: string, + public readonly comment?: string, + ) { + super( + addSentenceSuffixes( + typeof cause === "string" ? cause : `${cause?.message}`, + ` (${abbreviation})`, + comment ? `\n${comment}` : undefined, + ), + ); + + Object.defineProperty(this, "name", { + value: this.constructor.name, + writable: true, + enumerable: false, + }); + + if (typeof cause === "string") { + this.cause = Error(cause); + } else { + appendCause(this, cause); + } + + this.stack += `\n on ${where}`; + delete cause?.stack; + } +} diff --git a/src/abstract/Cluster.ts b/src/abstract/Cluster.ts new file mode 100644 index 0000000..44be104 --- /dev/null +++ b/src/abstract/Cluster.ts @@ -0,0 +1,660 @@ +import { types } from "util"; +import delayMod from "delay"; +import { Memoize } from "fast-typescript-memoize"; +import defaults from "lodash/defaults"; +import random from "lodash/random"; +import pTimeout from "p-timeout"; +import { CachedRefreshedValue } from "../internal/CachedRefreshedValue"; +import type { + Writeable, + DesperateAny, + MaybeCallable, + MaybeError, + PickPartial, + MaybeAsyncCallable, +} from "../internal/misc"; +import { + nullthrows, + mapJoin, + runInVoid, + objectHash, + maybeCall, + jsonHash, + jitter, + maybeAsyncCall, +} from "../internal/misc"; +import { Registry } from "../internal/Registry"; +import type { Client } from "./Client"; +import { ClientError } from "./ClientError"; +import { Island } from "./Island"; +import type { LocalCache } from "./LocalCache"; +import type { Loggers, SwallowedErrorLoggerProps } from "./Loggers"; +import { Shard } from "./Shard"; +import { ShardIsNotDiscoverableError } from "./ShardIsNotDiscoverableError"; +import type { ShardNamer } from "./ShardNamer"; + +/** Same as vanilla delay(), but with unref()ed timers. */ +const delay = delayMod.createWithTimers({ + setTimeout: (...args) => setTimeout(...args).unref(), + clearTimeout: (...args) => clearTimeout(...args), +}); + +/** + * Options for Cluster constructor. + */ +export interface ClusterOptions { + /** Islands configuration of the Cluster. */ + islands: MaybeAsyncCallable>; + /** Given a node of some Island, instantiates a Client for this node. Called + * when a new node appears in the Cluster statically or dynamically. */ + createClient: (node: TNode) => TClient; + /** Loggers to be injected into all Clients returned by createClient(). */ + loggers: Loggers; + /** An instance of LocalCache which may be used for auxiliary purposes when + * discovering Shards/Clients. */ + localCache?: LocalCache | null; + /** How often to recheck for changes in `options.islands`. If it is SYNC, then + * by default - often, like every 500 ms (since it's assumed that + * `options.islands` calculation is cheap). If it is ASYNC, then by default - + * not so often, every `shardsDiscoverIntervalMs` (we assume that getting the + * list of Island nodes may be expensive, e.g. fetching from AWS API or so). + * If the Islands list here changes, then we trigger Shards rediscovery and + * Clients recreation ASAP. */ + reloadIslandsIntervalMs?: MaybeCallable; + /** Info on how to build/parse Shard names. */ + shardNamer?: ShardNamer | null; + /** How often to run Shards rediscovery in normal circumstances. */ + shardsDiscoverIntervalMs?: MaybeCallable; + /** Jitter for shardsDiscoverIntervalMs and reloadIslandsIntervalMs. */ + shardsDiscoverIntervalJitter?: MaybeCallable; + /** Used in the following situations: + * 1. If we think that we know the Island of a particular Shard, but an + * attempt to access it fails. This means that maybe the Shard is migrating + * to another Island. So, we wait a bit and retry that many times. We + * should not do it too many times though, because all DB requests will be + * blocked waiting for the resolution. + * 2. If we sent a WRITE request to a Client, but it appeared that this Client + * is a replica, and the master moved to some other Client. In this case, + * we wait a bit and ping all Clients of the Island to refresh, who is + * master and who is replica. */ + runOnShardErrorRetryCount?: MaybeCallable; + /** How much time to wait before we retry rediscovering the entire Cluster + * after a Shard-to-Island resolution error. The time here should be just + * enough to wait for switching the Shard from one Island to another + * (typically quick). */ + runOnShardErrorRediscoverClusterDelayMs?: MaybeCallable; + /** How much time to wait before sending discover requests to all Clients of + * the Island trying to find the new master (or to reconnect). The time here + * may reach several seconds, since some DBs shut down the old master and + * promote some replica to it not simultaneously. */ + runOnShardErrorRediscoverIslandDelayMs?: MaybeCallable; + /** Delay the Client ending when this Client got removed from the Cluster. + * This allows to lower the number of client_is_ended errors in case a Client + * instance gets captured somewhere in the application code. */ + clientEndDelayMs?: MaybeCallable; +} + +/** + * A type of `ClusterOptions#islands` property. Represents the full list of + * Islands and their corresponding Nodes (masters and replicas). + */ +export type ClusterIslands = ReadonlyArray<{ + no: number; + nodes: readonly TNode[]; +}>; + +/** + * Holds the complete auto-discovered and non-contradictory snapshot of Islands + * and a map of Shards to figure out, which Island each Shard is located on. + * Also, includes all errors which caused some Islands to be completely + * undiscoverable (i.e. if we could not locate Shards on master and all + * replicas, so we gave up for that Island till the next rediscovery). + */ +interface ShardsDiscovered { + islandNoToIsland: Map>; + shardNoToIslandNo: ReadonlyMap; + nonGlobalShardNos: readonly number[]; + errors: SwallowedErrorLoggerProps[]; +} + +/** + * Cluster is a collection of Islands and an orchestration of shardNo -> Island + * resolution. + * + * It's unknown beforehand, which Island some particular Shard belongs to; the + * resolution is done asynchronously and lazily. + * + * Shard 0 is a special "global" Shard. + */ +export class Cluster { + /** Default values for the constructor options. */ + static readonly DEFAULT_OPTIONS: Required< + PickPartial> + > = { + localCache: null, + shardNamer: null, + shardsDiscoverIntervalMs: 10000, + shardsDiscoverIntervalJitter: 0.2, + reloadIslandsIntervalMs: NaN, + runOnShardErrorRetryCount: 2, + runOnShardErrorRediscoverClusterDelayMs: 1000, + runOnShardErrorRediscoverIslandDelayMs: 5000, + clientEndDelayMs: 10000, + }; + + /** The complete registry of all initialized Clients. Cluster nodes may change + * at runtime, so once a new node appears, its Client is added to the + * registry. Also, the Clients of disappeared nodes are eventually removed + * from the registry on the next Shards discovery. */ + private clientRegistry: Registry; + /** The complete registry of all Islands ever created. If some Island changes + * configuration, its old version is eventually removed from the registry + * during the next Shards discovery. */ + private islandRegistry: Registry< + { no: number; nodes: readonly TNode[]; clients: readonly Client[] }, + Island + >; + /** Represents the result of the recent successful call to + * `options.islands()`. */ + private islandsCache: CachedRefreshedValue>; + /** Represents the result of the recent successful Shards discovery. */ + private shardsDiscoverCache: CachedRefreshedValue>; + /** Once set to true, Clients for newly appearing nodes will be pre-warmed. */ + private prewarmEnabled = false; + + /** Cluster configuration options. */ + readonly options: Required>; + + /** + * Initializes the Cluster, but doesn't send any queries yet, even discovery + * queries (also, no implicit prewarming). + */ + constructor(options: ClusterOptions) { + this.options = defaults({}, options, Cluster.DEFAULT_OPTIONS); + + if ( + typeof this.options.reloadIslandsIntervalMs === "number" && + isNaN(this.options.reloadIslandsIntervalMs) + ) { + this.options.reloadIslandsIntervalMs = types.isAsyncFunction( + this.options.islands, + ) + ? maybeCall(this.options.shardsDiscoverIntervalMs) + : 500; + } + + this.clientRegistry = new Registry({ + key: (node) => jsonHash(node), + create: (node) => { + const client = this.options.createClient(node); + client.options.shardNamer ??= this.options.shardNamer; + const loggers = { ...client.options.loggers }; + client.options.loggers = { + swallowedErrorLogger: (props) => { + this.options.loggers.swallowedErrorLogger(props); + loggers.swallowedErrorLogger?.(props); + }, + clientQueryLogger: (props) => { + this.options.loggers.clientQueryLogger?.(props); + loggers.clientQueryLogger?.(props); + }, + clientConnectedLogger: (props) => { + this.options.loggers.clientConnectedLogger?.(props); + loggers.clientConnectedLogger?.(props); + }, + }; + return client; + }, + end: async (client, key, node) => { + await delay(maybeCall(this.options.clientEndDelayMs)); + const startTime = performance.now(); + this.options.loggers.clientEndLogger?.({ client, key, node }); + await client.end().catch((error: unknown) => + this.options.loggers.swallowedErrorLogger({ + where: `${this.constructor.name}.clientRegistry`, + error, + elapsed: Math.round(performance.now() - startTime), + importance: "normal", + }), + ); + }, + }); + + this.islandRegistry = new Registry({ + key: ({ no, nodes }) => jsonHash({ no, nodes }), + create: ({ no, clients }) => + new Island({ + no, + clients, + createShard: (no) => this.shardByNo(no), + localCache: this.options.localCache ?? undefined, + }), + }); + + this.islandsCache = new CachedRefreshedValue({ + delayMs: () => + Math.round( + maybeCall(this.options.reloadIslandsIntervalMs) * + jitter(maybeCall(this.options.shardsDiscoverIntervalJitter)), + ), + warningTimeoutMs: () => maybeCall(this.options.reloadIslandsIntervalMs), + deps: { + // If `options.islands` is reassigned externally (e.g. in a unit test), + // then it will be reflected in `await this.islandsCache.cached()` + // within `deps.delayMs` (not immediately). To expedite this (and Shards + // map) recheck, call Cluster#rediscover(). + delayMs: 50, + // We use the value of `options.islands` itself as a dependency, so if + // `options.islands` is reassigned externally, then we'll catch the + // change quickly, within `deps.delayMs`. We do NOT call + // `options.islands()` intentionally, we use its value - since we just + // want to check for reassignment (e.g. in unit tests). + handler: () => this.options.islands, + }, + resolverName: "Cluster#options.islands", + resolverFn: async () => maybeAsyncCall(this.options.islands), + delay, + onError: (error, elapsed) => + this.options.loggers.swallowedErrorLogger({ + where: `${this.constructor.name}.islandsCache`, + error, + elapsed, + importance: "normal", + }), + }); + + this.shardsDiscoverCache = new CachedRefreshedValue({ + delayMs: () => + Math.round( + maybeCall(this.options.shardsDiscoverIntervalMs) * + jitter(maybeCall(this.options.shardsDiscoverIntervalJitter)), + ), + warningTimeoutMs: () => maybeCall(this.options.shardsDiscoverIntervalMs), + deps: { + delayMs: () => maybeCall(this.options.reloadIslandsIntervalMs), + handler: async () => + jsonHash(await maybeAsyncCall(this.islandsCache.cached())), + }, + resolverName: "Cluster#shardsDiscoverExpensive", + resolverFn: async () => this.shardsDiscoverExpensive(), + delay, + onError: (error, elapsed) => + this.options.loggers.swallowedErrorLogger({ + where: `${this.constructor.name}.shardsDiscoverCache`, + error, + elapsed, + importance: "normal", + }), + }); + } + + /** + * Signals the Cluster to keep the Clients pre-warmed, e.g. open. (It's up to + * the particular Client's implementation, what does a "pre-warmed Client" + * mean; typically, it's keeping some minimal number of pooled connections.) + * + * Except when `randomizedDelayMs` is passed as 0, the actual prewarm (and + * Islands discovery) queries will run with a randomized delay between N/2 and + * N ms. It is better to operate in such mode: if multiple Node processes + * start simultaneously in the cluster, then the randomization helps to avoid + * new connections burst (new connections establishment is expensive for e.g. + * pgbouncer or when DB is accessed over SSL). + */ + prewarm( + randomizedDelayMs: number = 5000, + onInitialPrewarm?: (delayMs: number) => void, + ): void { + if (this.prewarmEnabled) { + return; + } + + this.prewarmEnabled = true; + const initialDelayMs = randomizedDelayMs + ? Math.round(random(randomizedDelayMs / 2, randomizedDelayMs)) + : 0; + setTimeout( + () => + runInVoid(async () => { + onInitialPrewarm?.(initialDelayMs); + for (const island of await this.islands()) { + for (const client of island.clients) { + client.prewarm(); + } + } + }), + initialDelayMs, + ).unref(); + } + + /** + * Returns a global Shard of the Cluster. This method is made synchronous + * intentionally, to defer the I/O and possible errors to the moment of the + * actual query. + */ + globalShard(): Shard { + return this.shardByNo(0); + } + + /** + * Returns all currently known (discovered) non-global Shards in the Cluster. + */ + async nonGlobalShards(): Promise>> { + const { nonGlobalShardNos } = await this.shardsDiscoverCache.cached(); + return nonGlobalShardNos.map((shardNo) => this.shardByNo(shardNo)); + } + + /** + * Returns Shard of a particular id. This method is made synchronous + * intentionally, to defer the I/O and possible errors to the moment of the + * actual query. + * + * Why is it important? Because Shards may go up and down temporarily at + * random moments of time. Imagine we made this method async and asserted that + * the Shard is actually available at the moment when the method is called. + * What would happen if the Shard object was stored somewhere as "successful" + * by the caller, then the Island went down, and then a query is sent to the + * Shard in, say, 20 seconds? We'd get an absolutely different exception, at + * the moment of the query. We don't want this to happen: we want all of the + * exceptions to be thrown with a consistent call stack (e.g. at the moment of + * the query), no matter whether it was an immediate call or a deferred one. + */ + shard(id: string): Shard { + return this.shardByNo( + this.options.shardNamer ? this.options.shardNamer.shardNoByID(id) : 0, + ); + } + + /** + * Returns a Shard if we know its number. The idea: for each Shard number + * (even for non-discovered yet Shards), we keep the corresponding Shard + * object in a Memoize cache, so Shards with the same number always resolve + * into the same Shard object. Then, an actual Island locating process happens + * when the caller wants to get a Client of that Shard (and it throws if such + * Shard hasn't been discovered actually). + */ + @Memoize() + shardByNo(shardNo: number): Shard { + return new Shard(shardNo, this.runOnShard.bind(this)); + } + + /** + * Returns a random Shard among the ones which are currently known + * (discovered) in the Cluster. + */ + async randomShard(seed?: object): Promise> { + const { nonGlobalShardNos } = await this.shardsDiscoverCache.cached(); + + let index; + if (seed !== undefined) { + const numHash = objectHash(seed).readUInt32BE(); + index = numHash % nonGlobalShardNos.length; + } else { + // TODO: implement power-of-two algorithm to pick the Shard which is + // smallest in size. + index = random(0, nonGlobalShardNos.length - 1); + } + + return this.shardByNo( + nullthrows( + nonGlobalShardNos[index], + () => "There are no non-global Shards in the Cluster", + ), + ); + } + + /** + * Returns an Island by its number. + */ + async island(islandNo: number): Promise> { + const { islandNoToIsland } = await this.shardsDiscoverCache.cached(); + return nullthrows( + islandNoToIsland.get(islandNo), + () => `No such Island: ${islandNo}`, + ); + } + + /** + * Returns all Islands in the Cluster. + */ + async islands(): Promise>> { + const { islandNoToIsland } = await this.shardsDiscoverCache.cached(); + return [...islandNoToIsland.values()]; + } + + /** + * Triggers shards rediscovery and finishes as soon as it's done. To be used + * in unit tests mostly, because in real life, it's enough to just modify the + * cluster configuration. + */ + async rediscover(what?: "islands" | "shards"): Promise { + if (!what || what === "islands") { + await this.islandsCache.refreshAndWait(); + } + + if (!what || what === "shards") { + await this.shardsDiscoverCache.refreshAndWait(); + } + } + + /** + * Runs the body function with retries. The Island injected into the body + * function is located automatically by the Shard number. In case of an error + * after any run attempt, calls onAttemptError(). + */ + private async runOnShard( + shardNo: number, + body: (island: Island, attempt: number) => Promise, + onAttemptError?: (error: unknown, attempt: number) => void, + ): Promise { + for (let attempt = 0; ; attempt++) { + let island: Island; + try { + // Re-read Islands map on every retry, because it might change. + const startTime = performance.now(); + const { shardNoToIslandNo, islandNoToIsland, errors } = + await this.shardsDiscoverCache.cached(); + const islandNo = shardNoToIslandNo.get(shardNo); + if (islandNo === undefined) { + // Notice that we don't retry ShardIsNotDiscoverableError below (it's + // not a ClientError) to avoid DoS, since it could be e.g. a fake ID + // passed to us in some URL or something else. We still want to log it + // through runOnShardErrorLogger() though. + throw new ShardIsNotDiscoverableError( + shardNo, + errors, + [...islandNoToIsland.values()], + Math.round(performance.now() - startTime), + ); + } + + // Retry the entire call to body(), to let it re-elect Client if needed. + island = nullthrows(islandNoToIsland.get(islandNo)); + return await body(island, attempt); + } catch (cause: unknown) { + const error = cause as MaybeError | ClientError; + + if (typeof error?.stack === "string") { + const suffix = `\n after ${attempt + 1} attempt${attempt > 0 ? "s" : ""}`; + if (!error.stack.endsWith(suffix)) { + error.stack = error.stack.trimEnd() + suffix; + } + } + + onAttemptError?.(error, attempt); + this.options.loggers.runOnShardErrorLogger?.({ error, attempt }); + + if ( + !(error instanceof ClientError) || + attempt >= maybeCall(this.options.runOnShardErrorRetryCount) + ) { + throw error; + } + + switch (error.postAction) { + case "rediscover-cluster": + await this.rediscoverCluster(); + continue; + case "rediscover-island": + await this.rediscoverIsland(island!); + continue; + case "choose-another-client": + continue; + case "fail": + throw error; + } + } + } + } + + /** + * Runs the whole-cluster rediscover after a delay, hoping that we'll load the + * new Shards-to-Island mapping. + * + * Multiple concurrent calls to this method will be coalesced into one + * (including the delay period): + * 1. This protects against the burst of rediscover requests caused by + * multiple failing concurrent queries. + * 2. It also allows to keep the queries batched when they are retried (i.e. + * the whole batch will be retried, not individual queries). + */ + @Memoize({ clearOnResolve: true }) + private async rediscoverCluster(): Promise { + await delay( + maybeCall(this.options.runOnShardErrorRediscoverClusterDelayMs), + ); + // We don't want to wait forever if some Island is completely down. + const startTime = performance.now(); + await pTimeout( + // Notice that we intentionally DO NOT call `islandsCache#refreshAndWait() + // here: changes in the list of Islands never reveal moved Shards. + this.shardsDiscoverCache.refreshAndWait(), + Math.round( + maybeCall(this.options.shardsDiscoverIntervalMs) * + jitter(maybeCall(this.options.shardsDiscoverIntervalJitter)) * + 2, + ), + "Timed out while waiting for whole-Cluster Shards discovery.", + ).catch((error: unknown) => + this.options.loggers.swallowedErrorLogger({ + where: `${this.constructor.name}.rediscoverCluster`, + error, + elapsed: Math.round(performance.now() - startTime), + importance: "normal", + }), + ); + } + + /** + * Runs Island#rediscover() after a delay. + * + * Multiple concurrent calls to this method will be coalesced into one + * (including the delay period): + * 1. This protects against the burst of rediscover requests caused by + * multiple failing concurrent queries. + * 2. It also allows to keep the queries batched when they are retried (i.e. + * the whole batch will be retried, not individual queries). + */ + @Memoize((island) => island.no, { clearOnResolve: true }) + private async rediscoverIsland(island: Island): Promise { + await delay(maybeCall(this.options.runOnShardErrorRediscoverIslandDelayMs)); + // We don't want to wait forever if the Island is completely down. + const startTime = performance.now(); + await pTimeout( + island.rediscover(), + Math.round( + maybeCall(this.options.shardsDiscoverIntervalMs) * + jitter(maybeCall(this.options.shardsDiscoverIntervalJitter)) * + 2, + ), + `Timed out while waiting for Island ${island.no} Shards discovery.`, + ).catch((error: unknown) => + this.options.loggers.swallowedErrorLogger({ + where: `${this.constructor.name}.rediscoverIsland(${island.no})`, + error, + elapsed: Math.round(performance.now() - startTime), + importance: "normal", + }), + ); + } + + /** + * Runs the actual Shards discovery queries over all Islands and updates the + * mapping from each Shard number to an Island where it lives. These queries + * may be expensive, so it's expected that the returned Promise is heavily + * cached by the caller code. + */ + private async shardsDiscoverExpensive(): Promise> { + const islands = await this.islandsCache.cached(); + const seenKeys = new Set(); + const islandNoToIsland = new Map>( + islands.map(({ no, nodes }) => { + const clients = nodes.map((node) => { + const [client, key] = this.clientRegistry.getOrCreate(node); + seenKeys.add(key); + this.prewarmEnabled && client.prewarm(); + return client; + }); + const [island, key] = this.islandRegistry.getOrCreate({ + no, + nodes, + clients, + }); + seenKeys.add(key); + return [no, island as Island]; + }), + ); + + const shardNoToIslandNo = new Map(); + const nonGlobalShardNos: number[] = []; + const errors: SwallowedErrorLoggerProps[] = []; + const shards: Array> = []; + await mapJoin( + [...islandNoToIsland.entries()], + async ([islandNo, island]) => { + errors.push(...(await island.rediscover())); + for (const shard of island.shards()) { + shards.push(shard); + const otherIslandNo = shardNoToIslandNo.get(shard.no); + if (otherIslandNo !== undefined) { + throw Error( + `Shard #${shard.no} exists on more than one Island: ` + + islandNoToIsland.get(otherIslandNo)?.master().options.name + + `(${otherIslandNo})` + + " and " + + island.master().options.name + + `(${islandNo})`, + ); + } + + shardNoToIslandNo.set(shard.no, islandNo); + if (shard.no !== 0) { + nonGlobalShardNos.push(shard.no); + } + } + }, + ); + + // Assign the last known Island number to all Shards synchronously. + for (const shard of shards) { + (shard as Writeable>).lastKnownIslandNo = + shardNoToIslandNo.get(shard.no) ?? null; + } + + // Gracefully delete and disconnect the Clients which didn't correspond to + // the list of nodes mentioned in this.options.islands, and also, delete + // leftover Islands which are not used anymore. In case we don't reach this + // point and threw earlier, it will eventually be reached on the next Shards + // discovery iterations. + for (const registry of [this.clientRegistry, this.islandRegistry]) { + runInVoid(registry.deleteExcept(seenKeys)); + } + + // Return the updated ENTIRE snapshot. + return { + islandNoToIsland, + shardNoToIslandNo, + nonGlobalShardNos: nonGlobalShardNos.sort((a, b) => a - b), + errors, + }; + } +} diff --git a/src/abstract/Island.ts b/src/abstract/Island.ts new file mode 100644 index 0000000..a0d430c --- /dev/null +++ b/src/abstract/Island.ts @@ -0,0 +1,342 @@ +import compact from "lodash/compact"; +import defaults from "lodash/defaults"; +import first from "lodash/first"; +import sample from "lodash/sample"; +import sortBy from "lodash/sortBy"; +import pTimeout from "p-timeout"; +import type { PickPartial } from "../internal/misc"; +import { mapJoin, nullthrows } from "../internal/misc"; +import type { Client, ClientRole } from "./Client"; +import type { LocalCache } from "./LocalCache"; +import type { SwallowedErrorLoggerProps } from "./Loggers"; +import type { Shard } from "./Shard"; + +/** + * The list of Clients grouped into master, replica and unknown groups. In each + * group, there are healthy and unhealthy Clients. + */ +type ClassifiedClients = Record< + ClientRole, + { healthy: TClient[]; unhealthy: TClient[] } +>; + +/** + * Options for Island constructor. + */ +export interface IslandOptions { + /** Island number. */ + no: number; + /** Clients of that Island (the order is arbitrary). */ + clients: readonly TClient[]; + /** Should return a Memoize'd Shards object by its number. */ + createShard: (no: number) => Shard; + /** An auxillary LocalCache used to fallback-infer master/replica role in case + * some Client is unavailable right now. */ + localCache?: LocalCache<{ + address: string; + role: ClientRole; + // We must not put non-deterministic fields (like timestamps) here, + // otherwise the cache file will be overwritten over and over even when + // writing the same data. + }> | null; + /** If nonzero, runs the second shardNos() call attempt on a Client if the 1st + * call on that Client gets stuck for longer than the provided number of ms. + * + * This option is used to detect the unhealthy DB connection quicker, and + * thus, exit from rediscover() faster (the Shards map can likely be loaded + * from a replica still, so the down DB is not the end of the world). The idea + * is that the 1st shardNos() could get stuck due to the load balancer trying + * to wait until the DB goes back up again (e.g. for PgBouncer, that is + * query_wait_timeout situation; "pause_client" is printed to PgBouncer debug + * logs, and then the Client gets frozen for up to query_wait_timeout; other + * engines may have similar behavior). But for the NEW connections/queries, + * after a small delay, the load balancer may realize that the DB is really + * down (the load balancer typically can get "connection refused" while + * connecting to the DB server really quickly), and the 2nd shardNos() call + * will reject almost immediately ("fast fail" workflow), way before the 1st + * call rejects (e.g. for PgBouncer and query_wait_timeout=15s, the 1st call + * may get stuck for up to 15 seconds!). So, we will not wait that long to + * figure out that the DB is down, and will detect that situation quicker. + * + * Typically, the connection attempt from load balancer to an unhealthy DB + * server ends up quickly with "connection refused" TCP error (e.g. when the + * load balancer and the DB server run on the same host), so the value in this + * option can be small. But not always. Sometimes, the new connection from + * load balancer to the DB server gets stuck in "connecting..." state (e.g. + * this happens when the load balancer runs in a Docker container, and the DB + * container gets killed; the connection attempt will eventually fail, but in + * 1+ minutes and with "no route to host" error). In this case, the value in + * the option must be greater than e.g. server_connect_timeout (example for + * PgBouncer; basically, server_connect_timeout is PgBouncer's tool to detect + * "stuck" connection attempts (the connections which don't get "connection + * refused" quickly). */ + shardNosConcurrentRetryDelayMs?: number; +} + +/** + * Island is a moderately short-lived collection of DB connections (represented + * as Clients) that contains a single master Client and any number of replicas. + * + * - In normal situations, you don't likely need to work with Islands directly, + * you can rely on higher level abstractions which support automatic + * rediscovery and retries: Ent (or lower level Shard and Schema). + * - Islands are helpful mostly when working with cross-Shards logic. + * - Island is somewhat temporary: if the Cluster is reconfigured in real-time, + * then its Island objects may be recycled and re-created, and the + * corresponding Clients may be ended. This also applies to any given Client + * instance. Don't retain and reuse those objects for too long. The reliable + * abstractions (resilient to disconnects, shards migration, failover etc.) + * start from Shard level. + * - There is no guarantee that the data returned by shards(), master() or + * replica() will be up to date. Shards may be just migrated to another + * Island. Master may become a replica, or vice versa. + */ +export class Island { + /** Default values for the constructor options. */ + static readonly DEFAULT_OPTIONS: Required< + PickPartial> + > = { + localCache: null, + // https://willbryant.net/overriding_the_default_linux_kernel_20_second_tcp_socket_connect_timeout + // Convenient when tcp_syn_retries=1 is set in docker-compose.yml file: + // "Linux ... sends 5 SYNs ... the retries are after 3s, 6s, 12s, 24s" + shardNosConcurrentRetryDelayMs: 3500, + }; + + /** Clients grouped based on their roles and health. */ + private classifiedClients!: ClassifiedClients; + /** In case shardNos discovery for some Client hasn't succeeded yet, and thus, + * we are not sure about the role of that Client, then we try to load the role + * from fallback cache in this map and use further instead of "unknown". */ + private fallbackRoles = new WeakMap(); + /** Recently discovered Shard numbers. */ + private shardNos: number[] | null = null; + + /** Island configuration options. */ + readonly options: Required>; + + /** + * Initializes the Island by copying the Client references into it. + */ + constructor(options: IslandOptions) { + this.options = defaults({}, options, Island.DEFAULT_OPTIONS); + if (options.clients.length === 0) { + throw Error("Island does not have nodes"); + } + + this.reclassifyClients(); + } + + /** + * Island number. + */ + get no(): number { + return this.options.no; + } + + /** + * The list of Clients in this Island. No assumptions about the order. + */ + get clients(): readonly TClient[] { + return this.options.clients; + } + + /** + * Queries for Shards on the best available Client (preferably master, then + * replicas) and stores the result internally, available for the further + * shards() call. + * - If some Clients are unavailable, tries its best to infer the data from + * other Clients. + * - The method queries ALL clients in parallel, because the caller logic + * anyways needs to know, who's master and who's replica, as a side effect + * of the very 1st query after the Client creation. We infer that as a piggy + * back after calling Client#shardNos(). + * - In case we could not discover shards, returns the list of errors happened + * during the discovery. + */ + async rediscover(): Promise { + // Load fallback roles as early as possible (since shardNo() queries below + // may take a lot of time in case they time out). + await mapJoin(this.clients, async (client) => { + if (!this.fallbackRoles.has(client)) { + const fallback = await this.options.localCache?.get(client.address()); + if (fallback) { + this.fallbackRoles.set(client, fallback.role); + } + } + }); + + // We don't use Promise.race() here! We really want to wait until ALL + // clients either respond or reject, which is what mapJoin() is doing. If we + // used Promise.race(), then timing out Clients could've been requested by + // the caller logic concurrently over and over, so the number of pending + // requests to them would grow. We want to control that parallelism. + const errors: SwallowedErrorLoggerProps[] = []; + const res = sortBy( + compact( + await mapJoin(this.clients, async (client) => { + const startTime = performance.now(); + try { + const shardNos = await this.clientShardNos(client); + const address = client.address(); + const role = client.role(); + await this.options.localCache?.set(address, { address, role }); + this.fallbackRoles.set(client, role); + return { role, shardNos }; + } catch (error: unknown) { + errors.push({ + where: `${client.constructor.name}(${client.options.name}).shardNos`, + error, + elapsed: Math.round(performance.now() - startTime), + importance: "low", + }); + client.options.loggers?.swallowedErrorLogger( + errors[errors.length - 1], + ); + return null; + } + }), + ), + ({ role }) => (role === "master" ? 0 : role === "replica" ? 1 : 2), + ({ shardNos }) => -1 * shardNos.length, + ); + this.reclassifyClients(); + + if (res.length > 0) { + this.shardNos = [...res[0].shardNos].sort((a, b) => a - b); + return []; + } else { + // Being unable to access all DB Clients is not a critical error here, + // we'll just miss some Shards (and other Shards will work). DO NOT throw + // through here yet! This needs to be addressed holistically and with + // careful retries. Also, we have Shards rediscovery every N seconds, so a + // missing Island will self-heal eventually. + this.shardNos = []; + return errors; + } + } + + /** + * Returns the currently best-known Shards on this Island. This method is + * needed only when working with cross-Shards logic; in normal situations, it + * is not called much. + */ + shards(): Array> { + const shardNos = nullthrows( + this.shardNos, + "Before shards() can be used, rediscover() must finish", + ); + return shardNos.map((no) => this.options.createShard(no)); + } + + /** + * Returns the currently best-known master Client among the Clients of this + * Island. + * + * - If all masters are unhealthy, we still return one of them and prefer not + * to fall back on a replica, because otherwise, we'll see non-obvious + * errors in logs ("can't write in a read-only Client" or so) and suspect + * that there is a bug in logic, although there is really no bug, it's just + * the master node went down. It's way better to throw a straightforward + * error like "Client is down". + * - If we can't find a master, but there is a list of Clients with unknown + * roles, prefer returning one of them vs. any known replica, since there is + * a chance that among those unknown Clients, there will be a master. + * - In case all Clients are read-only (replicas), still returns the 1st of + * them, assuming that it's better to throw at the caller side on a failed + * write (at worst) rather than here. It is not common to have an Island + * without a master Client, that happens only temporarily during + * failover/switchover, so the caller will likely rediscover and find a new + * master on a next retry. + */ + master(): TClient { + if (this.clients.length === 1) { + return first(this.clients)!; + } + + const master = sample(this.classifiedClients.master.healthy); + if (master && master.role() === "master" && !master.connectionIssue()) { + // Fast and most frequent path. + return master; + } + + this.reclassifyClients(); + return ( + sample(this.classifiedClients.master.healthy) ?? + sample(this.classifiedClients.master.unhealthy) ?? // prefer unhealthy master + sample(this.classifiedClients.unknown.healthy) ?? + sample(this.classifiedClients.unknown.unhealthy) ?? + sample(this.classifiedClients.replica.healthy) ?? + sample(this.classifiedClients.replica.unhealthy) ?? + first(this.clients)! // should never reach here + ); + } + + /** + * Returns a currently best-known random replica Client. In case there are no + * replicas, returns the master Client. + */ + replica(): TClient { + if (this.clients.length === 1) { + return first(this.clients)!; + } + + const replica = sample(this.classifiedClients.replica.healthy); + if (replica && replica.role() === "replica" && !replica.connectionIssue()) { + // Fast and most frequent path. + return replica; + } + + this.reclassifyClients(); + return ( + sample(this.classifiedClients.replica.healthy) ?? + sample(this.classifiedClients.unknown.healthy) ?? + sample(this.classifiedClients.master.healthy) ?? + sample(this.classifiedClients.replica.unhealthy) ?? + sample(this.classifiedClients.unknown.unhealthy) ?? + sample(this.classifiedClients.master.unhealthy) ?? + first(this.clients)! // should never reach here + ); + } + + /** + * Updates the list of classified Clients. We try hard to not put Clients in + * "unknown" group by falling back to fallbackRoles. + */ + private reclassifyClients(): void { + const classifiedClients: ClassifiedClients = { + master: { healthy: [], unhealthy: [] }, + unknown: { healthy: [], unhealthy: [] }, + replica: { healthy: [], unhealthy: [] }, + }; + for (const client of this.clients) { + const health = client.connectionIssue() ? "unhealthy" : "healthy"; + const role = client.role(); + classifiedClients[ + role === "unknown" ? (this.fallbackRoles.get(client) ?? role) : role + ][health].push(client); + } + + this.classifiedClients = classifiedClients; + } + + /** + * Tries to pull shardNos() out of the Client and fail fast if the DB is down. + * See details in shardNosConcurrentRetryDelayMs option description. + */ + private async clientShardNos(client: TClient): Promise { + if (this.options.shardNosConcurrentRetryDelayMs === 0) { + return client.shardNos(); + } + + const promise = client.shardNos(); + const maybeShardNos = await pTimeout( + promise, + this.options.shardNosConcurrentRetryDelayMs, + () => "TIMEOUT" as const, + ); + return maybeShardNos instanceof Array + ? maybeShardNos + : Promise.race([promise, client.shardNos()]); // try again once + } +} diff --git a/src/abstract/Loader.ts b/src/abstract/Loader.ts new file mode 100644 index 0000000..b79d2cf --- /dev/null +++ b/src/abstract/Loader.ts @@ -0,0 +1,85 @@ +import type { DeferredPromise } from "p-defer"; +import pDefer from "p-defer"; + +export interface Handler { + onCollect: (...args: TLoadArgs) => void | "flush" | "wait"; + onWait?: () => Promise; + onFlush: (collected: number) => Promise; + onReturn: (...args: TLoadArgs) => TReturn; +} + +interface LoaderSession { + handler: Handler; + flush: Promise; + abortWait: DeferredPromise; + collected: number; +} + +/** + * Loader allows to batch single-item requests into batches. It uses a different + * architecture than Facebook's DataLoader: + * + * - it's more developers-friendly: multi-parameter loadings, you may implement + * automatic deduplication of requests at onCollect stage, no requirement to + * serialize/deserialize requests into string keys; + * - strong-typed load() and handler arguments. + * + * To create your own specific loader: + * 1. Define a handler class with onCollect/onReturn/onFlush methods. + * 2. In onCollect, accumulate the incoming requests in the handler object's + * private property. + * 3. In onFlush, process what you accumulated so far and save to another + * handler object's private property (and by adding, say, delay(50) in the + * beginning of onFlush, you may group the requests coming within the 1st 50 + * ms). + * 3. In onReturn, extract the result corresponding to the request and return + * it, so the caller will receive it seamlessly as a load() return value. + * + * In the future, Batcher may be refactored to use Loader as the underlying + * engine, but for now they're separate (Batcher is much more domain logic + * specific and Loader is completely abstract). + */ +export class Loader { + private session: LoaderSession | null = null; + + constructor(private handlerCreator: () => Handler) {} + + async load(...args: TLoadArgs): Promise { + const session = (this.session ??= { + collected: 0, + handler: this.handlerCreator(), + abortWait: pDefer(), + flush: new Promise((resolve) => + process.nextTick(async () => { + const waitPromise = session.handler.onWait?.(); + if (waitPromise) { + // If we have onWait() handler, we should wait on it, but interrupt + // this wait in case session.abortWait resolves. + await Promise.race([waitPromise, session.abortWait.promise]); + // Often times, wait() returns a ClearablePromise (e.g. from delay + // module), so we can utilize it here to cancel the lingering timer. + if ( + "clear" in waitPromise && + typeof waitPromise.clear === "function" && + waitPromise.clear.length === 0 + ) { + waitPromise.clear(); + } + } + + this.session = this.session === session ? null : this.session; + resolve(session.handler.onFlush(session.collected)); + }), + ), + }); + + session.collected++; + if (session.handler.onCollect(...args) === "flush") { + this.session = this.session === session ? null : this.session; + session.abortWait.resolve(); + } + + await session.flush; + return session.handler.onReturn(...args); + } +} diff --git a/src/abstract/LocalCache.ts b/src/abstract/LocalCache.ts new file mode 100644 index 0000000..6da4427 --- /dev/null +++ b/src/abstract/LocalCache.ts @@ -0,0 +1,200 @@ +import fs from "fs"; +import { dirname } from "path"; +import { threadId } from "worker_threads"; +import defaults from "lodash/defaults"; +import { jitter, type MaybeError, type PickPartial } from "../internal/misc"; +import type { Loggers } from "./Loggers"; + +export interface LocalCacheOptions { + /** Directory to store the cache in (auto-created). */ + dir: string; + /** Loggers for e.g. swallowed errors. */ + loggers: Pick; + /** Max time (approximately) for an unread key to exist. */ + expirationMs?: number; + /** Extension of cache files (without dot). */ + ext?: string; + /** Jitter for cleanup runs. */ + cleanupJitter?: number; + /** How much time to wait till the very 1st cleanup run. The idea is that Node + * process may be short-lived, so the next cleanup run configured via + * cleanupRoundsPerExpiration may never happen, and we also need to cleanup in + * the very beginning of the object lifetime. */ + cleanupFirstRunDelayMs?: number; + /** How many times per expirationMs interval should we run the cleanup. */ + cleanupRoundsPerExpiration?: number; + /** How often to update mtime on read operations. E.g. if this value is 10, + * then mtime will be updated not more than ~10 times within the expiration + * period (optimizing filesystem writes). */ + mtimeUpdatesOnReadPerExpiration?: number; +} + +/** + * A simple key-value cache stored on the local machine. + * + * - The expectation is that there will be not too many keys stored, since the + * background cleanup process running time to time is O(numKeysStored). + * - Guarantees corruption-free writes to the keys from multiple processes + * running concurrently. + * - The values which are not requested longer than approximately `expirationMs` + * are auto-removed. + * - Each key is stored in an individual file under `dir`. Some temporary files + * may also appear in that directory, but eventually, they will be cleaned up, + * even if they get stuck for some time. + */ +export class LocalCache { + /** Default values for the constructor options. */ + static readonly DEFAULT_OPTIONS: Required> = { + expirationMs: 1000 * 3600 * 24, + ext: "json", + cleanupJitter: 0.2, + cleanupFirstRunDelayMs: 30000, + cleanupRoundsPerExpiration: 2, + mtimeUpdatesOnReadPerExpiration: 10, + }; + + private cleanupTimeout?: NodeJS.Timeout; + + /** LocalCache configuration options. */ + readonly options: Required; + + /** + * Initializes the instance. + */ + constructor(options: LocalCacheOptions) { + this.options = defaults({}, options, LocalCache.DEFAULT_OPTIONS); + this.cleanupTimeout = setTimeout( + () => this.onCleanupTimer(), + Math.round( + this.options.cleanupFirstRunDelayMs * + jitter(this.options.cleanupJitter), + ), + ).unref(); + } + + /** + * Ends the instance lifecycle (e.g. garbage recheck interval). + */ + end(): void { + clearTimeout(this.cleanupTimeout); + this.cleanupTimeout = undefined; + } + + /** + * Returns the value for the given key, or null if the key does not exist. + */ + async get(key: string): Promise { + const path = this.buildPath(key); + + try { + if (!fs.existsSync(path)) { + return null; + } + + const data = await fs.promises.readFile(path, { encoding: "utf-8" }); + return JSON.parse(data); + } catch (e: unknown) { + rethrowExceptFileNotExistsError(e); + return null; + } + } + + /** + * Sets the value for the given key. + */ + async set(key: string, value: TValue): Promise { + const path = this.buildPath(key); + + const data = JSON.stringify(value, undefined, 2); + try { + const stat = await fs.promises.stat(path); + const oldData = await fs.promises.readFile(path, { encoding: "utf-8" }); + if (oldData === data) { + // Don't write if the data in the file is the same, just update mtime, + // but not too frequently too. + const now = Date.now(); + if ( + stat.mtimeMs < + now - + this.options.expirationMs / + this.options.mtimeUpdatesOnReadPerExpiration + ) { + await fs.promises.utimes(path, now, now); + } + + return; + } + } catch (e: unknown) { + rethrowExceptFileNotExistsError(e); + } + + await fs.promises.mkdir(dirname(path), { recursive: true }); + const tmpPath = this.buildPath(`${key}.${process.pid}-${threadId}.tmp`); + await fs.promises.writeFile(tmpPath, data); + await fs.promises.rename(tmpPath, path); + } + + /** + * Deletes the values for keys which were not accessed for a long time. + */ + private async cleanup(): Promise { + await fs.promises.mkdir(this.options.dir, { recursive: true }); + const files = await fs.promises.readdir(this.options.dir); + + const now = Date.now(); + const dotExt = `.${this.options.ext}`; + for (const file of files) { + if (file.endsWith(dotExt)) { + const path = `${this.options.dir}/${file}`; + const stats = await fs.promises.stat(path); + if (now - stats.mtimeMs > this.options.expirationMs) { + try { + await fs.promises.unlink(path); + } catch (e: unknown) { + rethrowExceptFileNotExistsError(e); + } + } + } + } + } + + /** + * Runs then the instance creates (with initial small jitter) and also + * periodically. + */ + private onCleanupTimer(): void { + this.cleanup().catch((error: unknown) => + this.options.loggers.swallowedErrorLogger?.({ + where: `${this.constructor.name}.cleanup`, + error, + elapsed: null, + importance: "normal", + }), + ); + this.cleanupTimeout = setTimeout( + () => this.onCleanupTimer(), + Math.round( + (this.options.expirationMs / this.options.cleanupRoundsPerExpiration) * + jitter(this.options.cleanupJitter), + ), + ).unref(); + } + + /** + * Builds the full path to a file for a given key. + */ + private buildPath(key: string): string { + const filename = key.replace(/[^a-z0-9.]+/gi, "-"); + return `${this.options.dir}/${filename}.${this.options.ext}`; + } +} + +/** + * Throws the passed exception if it's a "file not found" error from Node fs + * module. Otherwise, does nothing. + */ +function rethrowExceptFileNotExistsError(e: unknown): void { + if ((e as MaybeError)?.code !== "ENOENT") { + throw e; + } +} diff --git a/src/abstract/Loggers.ts b/src/abstract/Loggers.ts new file mode 100644 index 0000000..6618b0a --- /dev/null +++ b/src/abstract/Loggers.ts @@ -0,0 +1,80 @@ +import type { DesperateAny } from "../internal/misc"; +import type { Client, ClientRole } from "./Client"; +import type { QueryAnnotation } from "./QueryAnnotation"; + +/** + * Loggers are called at different stages of the query lifecycle. We do not use + * EventEmitter for several reasons: + * 1. It is not friendly to mocking in Jest. + * 2. The built-in EventEmitter is not strongly typed. + */ +export interface Loggers { + /** Logs actual queries to the database (after batching). */ + clientQueryLogger?: (props: ClientQueryLoggerProps) => void; + /** Logs errors which did not throw through (typically recoverable). */ + swallowedErrorLogger: (props: SwallowedErrorLoggerProps) => void; + /** Called when Island-from-Shard location fails (e.g. no such Shard), or when + * a query on a particular Shard fails due to any reason (like transport + * error). Mostly used in unit tests, since it's called for every retry. */ + runOnShardErrorLogger?: (props: RunOnShardErrorLoggerProps) => void; + /** Called when a Client gets ended due to dynamic Islands reconfiguration. + * Allows to debug flaky Island reconfiguration. */ + clientEndLogger?: (props: ClientEndLoggerProps) => void; + /** Called when a Client connects to the database. */ + clientConnectedLogger?: (props: ClientConnectedLoggerProps) => void; +} + +export interface ClientQueryLoggerProps { + annotations: QueryAnnotation[]; + op: string; + shard: string; + table: string; + batchFactor: number; + msg: string; + output: unknown; + elapsed: { + total: number; + acquire: number; + }; + connStats: { + /** The stats related to the used connection. */ + id: string; + /** The number of queries sent to the connection. */ + queriesSent: number; + }; + poolStats: { + /** Total number of connections in the pool. */ + totalConns: number; + /** Connections not busy running a query. */ + idleConns: number; + /** Once all idle connections are over, requests are queued waiting for a + * new available connection. This is the number of such queued requests. */ + queuedReqs: number; + }; + error: string | undefined; + role: ClientRole; + backend: string; + address: string; +} + +export interface SwallowedErrorLoggerProps { + where: string; + error: unknown; + elapsed: number | null; + importance: "low" | "normal"; +} + +export interface RunOnShardErrorLoggerProps { + error: unknown; + attempt: number; +} + +export interface ClientEndLoggerProps { + client: Client; + key: string; + node: TNode; +} + +export interface ClientConnectedLoggerProps { + node: TNode; +} diff --git a/src/abstract/Query.ts b/src/abstract/Query.ts new file mode 100644 index 0000000..b21c316 --- /dev/null +++ b/src/abstract/Query.ts @@ -0,0 +1,11 @@ +import type { Client } from "./Client"; +import type { QueryAnnotation } from "./QueryAnnotation"; + +/** + * A very lean interface for a Query. In practice each query is so different + * that this interface is the only common part of them all. + */ +export interface Query { + readonly IS_WRITE: boolean; + run(client: Client, annotation: QueryAnnotation): Promise; +} diff --git a/src/abstract/QueryAnnotation.ts b/src/abstract/QueryAnnotation.ts new file mode 100644 index 0000000..c90f3ae --- /dev/null +++ b/src/abstract/QueryAnnotation.ts @@ -0,0 +1,40 @@ +import type { TimelineCaughtUpReason } from "./Timeline"; + +/** + * A reason why master or replica was chosen to send the query to. The most + * noticeable ones are: + * - "replica-bc-master-state-unknown": 99% of cases (since writes are rare) + * - "master-bc-replica-not-caught-up": happens immediately after each write, + * until the write is propagated to replica + * - "replica-bc-caught-up": must happen eventually (in 0.1-2s) after each write + * - "replica-bc-pos-expired": signals that the replication lag is huge, we + * should carefully monitor this case and make sure it never happens + */ +export type WhyClient = + | Exclude + | "replica-bc-stale-replica-freshness" + | "master-bc-is-write" + | "master-bc-master-freshness" + | "master-bc-no-replicas" + | "master-bc-replica-not-caught-up"; + +/** + * A debug annotation from each individual place which initiated the query. When + * multiple queries are grouped into one large query by Ent Framework (even + * cross-async-trace and cross-VC), the resulting large query is accompanied + * with all those annotations. + */ +export interface QueryAnnotation { + /** Trace ID of the series of the queries. */ + readonly trace: string; + /** Something which identifies the acting user; it's named `vc` after Ent's VC + * for simplicity, but at this layer of abstractions, there are no Ents. */ + readonly vc: string; + /** Sometimes a query may be annotated by the source stack trace. It's + * typically expensive, so it's likely "" in production. */ + readonly debugStack: string; + /** Answers, why exactly this Client was selected to send the query to. */ + readonly whyClient: WhyClient | undefined; + /** In case it's a retry, the attempt number will be greater than 0. */ + readonly attempt: number; +} diff --git a/src/abstract/QueryBase.ts b/src/abstract/QueryBase.ts new file mode 100644 index 0000000..7330216 --- /dev/null +++ b/src/abstract/QueryBase.ts @@ -0,0 +1,47 @@ +import type { Table } from "../types"; +import type { Client } from "./Client"; +import type { Query } from "./Query"; +import type { QueryAnnotation } from "./QueryAnnotation"; +import type { Runner } from "./Runner"; +import type { Schema } from "./Schema"; + +/** + * A convenient base class for most (but not all) of the queries, where the + * Runner instance is the same for different query input shapes. If the query + * doesn't fit the QueryBase framework (like PgQueryUpdate for instance where we + * have separate Runner instances for separate set of updated fields), a Query + * is used directly instead. + */ +export abstract class QueryBase< + TTable extends Table, + TInput, + TOutput, + TClient extends Client, +> implements Query { + /** @ignore */ + abstract readonly RUNNER_CLASS: { + readonly IS_WRITE: boolean; + new (schema: Schema, client: TClient): Runner; + }; + + constructor( + public readonly schema: Schema, + public readonly input: TInput, + ) {} + + get IS_WRITE(): boolean { + return this.RUNNER_CLASS.IS_WRITE; + } + + async run(client: TClient, annotation: QueryAnnotation): Promise { + return client + .batcher( + this.constructor, + this.schema, + "", + false, + () => new this.RUNNER_CLASS(this.schema, client), + ) + .run(this.input, annotation); + } +} diff --git a/src/abstract/QueryPing.ts b/src/abstract/QueryPing.ts new file mode 100644 index 0000000..6beacae --- /dev/null +++ b/src/abstract/QueryPing.ts @@ -0,0 +1,18 @@ +import type { Client, ClientPingInput } from "./Client"; +import type { Query } from "./Query"; +import type { QueryAnnotation } from "./QueryAnnotation"; + +/** + * A helper Query which delegates to Client#ping(execTimeMs). + */ +export class QueryPing implements Query { + constructor(public readonly input: Omit) {} + + get IS_WRITE(): boolean { + return this.input.isWrite; + } + + async run(client: Client, annotation: QueryAnnotation): Promise { + return client.ping({ ...this.input, annotation }); + } +} diff --git a/src/abstract/Runner.ts b/src/abstract/Runner.ts new file mode 100644 index 0000000..a73af2e --- /dev/null +++ b/src/abstract/Runner.ts @@ -0,0 +1,86 @@ +import type { QueryAnnotation } from "./QueryAnnotation"; + +const INIT_SEQUENCE = 3; // small prime, doesn't matter + +/** + * Knows how to translate individual strongly typed requests into DB language + * and how to parse the result back. + */ +export abstract class Runner { + /** If true, it's a write operation. */ + static readonly IS_WRITE: boolean; + + /** Used to build an unique default operation key. */ + private sequence = INIT_SEQUENCE; + + /** Operation name for logging purposes. */ + abstract readonly op: string; + + /** Maximum batch size for this type of operations. */ + abstract readonly maxBatchSize: number; + + /** In case undefined is returned from batching, this value will be returned + * instead. */ + abstract readonly default: TOutput; + + /** + * Method runSingle is to e.g. produce simple DB requests when we have only + * one input to process, not many. + */ + abstract runSingle( + input: TInput, + annotations: QueryAnnotation[], + ): Promise; + + /** + * Typically issues complex queries with magic. If the method is not defined, + * then the runner doesn't support batching, so only runSingle() will be used. + */ + abstract runBatch?( + inputs: Map, + annotations: QueryAnnotation[], + ): Promise>; + + /** + * If the single query's error needs to be retried (e.g. it's a deadlock + * error), returns the number of milliseconds to wait before retrying. + */ + abstract delayForSingleQueryRetryOnError( + error: unknown, + ): number | "immediate_retry" | "no_retry"; + + /** + * If this method returns true for an error object, the batch is split back + * into sub-queries, they are executed individually, and then the response of + * each query is delivered to each caller individually. Used mostly for e.g. + * batch-deadlock errors or for FK constraint errors when it makes sense to + * retry other members of the batch and not fail it entirely hurting other + * innocent queries. + * + * We can do this, because we know that if some transaction is aborted, it's + * always safe to retry it. (If we're not sure about the transaction, e.g. the + * Client doesn't support transactions at all, then the method should return + * false.) + */ + abstract shouldDebatchOnError(error: unknown): boolean; + + /** + * Parameter `name` is typically a table name. + */ + constructor(public readonly name: string) {} + + /** + * Returns a batch-dedupping key for the input. By default, no dedupping is + * performed (i.e. all inputs are processed individually and not collapsed + * into one input; e.g. this is needed for inserts). + */ + key(_input: TInput): string { + const key = "k" + this.sequence; + this.sequence += INIT_SEQUENCE; + if (this.sequence > INIT_SEQUENCE * 10000000) { + this.sequence = INIT_SEQUENCE; + } + + return key; + } +} diff --git a/src/abstract/Schema.ts b/src/abstract/Schema.ts new file mode 100644 index 0000000..d741501 --- /dev/null +++ b/src/abstract/Schema.ts @@ -0,0 +1,148 @@ +import type { + CountInput, + ExistsInput, + InsertInput, + LoadByInput, + Row, + SelectByInput, + SelectInput, + Table, + UniqueKey, + UpdateInput, +} from "../types"; +import type { Query } from "./Query"; + +export interface SchemaClass { + new < + TTable extends Table, + TUniqueKey extends UniqueKey = UniqueKey, + >( + name: string, + table: TTable, + uniqueKey?: TUniqueKey, + ): Schema; +} + +/** + * Schema is like a "table" in some database (sharded, but it's beyond the scope + * of Schema). It is also a factory of Query: it knows how to build runnable + * Query objects. This 2nd role is database engine specific (e.g. there might be + * PgSchema, RedisSchema etc.): such composition simplifies the code and lowers + * the number of abstractions. + * + * The set of supported Queries is opinionated and is crafted carefully to + * support the minimal possible list of primitives, but at the same time, be not + * too limited in the queries the DB engine can execute. + */ +export abstract class Schema< + TTable extends Table, + TUniqueKey extends UniqueKey = UniqueKey, +> { + /** + * Used in e.g. inverses. This casts this.constructor to SchemaClass with all + * static methods and `new` semantic (TS doesn't do it by default; for TS, + * x.constructor is Function). + */ + declare ["constructor"]: SchemaClass; + + // Below is the API to be used by BaseEnt (which is engine-agnostic). + + /** + * Generates a new ID for the row. Used when e.g. there is a beforeInsert + * trigger on the Ent which needs to know the ID beforehand. + */ + abstract idGen(): Query; + + /** + * Creates a new row. Returns null if the row violates some unique key + * constraint, otherwise returns the row ID. + */ + abstract insert(input: InsertInput): Query; + + /** + * Upserts a row. Always returns the row ID. + */ + abstract upsert(input: InsertInput): Query; + + /** + * Updates one single row by its ID. Returns true if it actually existed. + */ + abstract update(id: string, input: UpdateInput): Query; + + /** + * Deletes a row by id. Returns true if it actually existed. + */ + abstract delete(id: string): Query; + + /** + * "Load" family of methods means that we load exactly one row. This one + * returns a row by its ID or null if it's not found. + */ + abstract load(id: string): Query | null>; + + /** + * Loads one single row by its unique key ("by" denotes that it's based on an + * unique key, not on an ID). Returns null if it's not found. + */ + abstract loadBy( + input: LoadByInput, + ): Query | null>; + + /** + * "Select" family of methods means that we load multiple rows ("by" denotes + * that it's based on an unique key, not on an arbitrary query). This one + * returns all rows whose unique key prefix matches the input. + */ + abstract selectBy( + input: SelectByInput, + ): Query>>; + + /** + * Returns all rows matching an arbitrary query. + */ + abstract select(input: SelectInput): Query>>; + + /** + * Returns the number of rows matching an arbitrary query. + */ + abstract count(input: CountInput): Query; + + /** + * An optimized version of count() for the cases where we only need to know + * whether at least one row exists, and don't need a precise count. + */ + abstract exists(input: ExistsInput): Query; + + constructor( + /** For relational databases, it's likely a table name. */ + public readonly name: string, + /** Structure of the table. */ + public readonly table: TTable, + /** Fields which the native unique key consists of (if any). */ + public readonly uniqueKey: TUniqueKey, + ) { + if (!Object.keys(this.table).length) { + throw Error("Must have at least one field"); + } + + for (const field of this.uniqueKey) { + if (this.table[field].autoUpdate) { + throw Error( + "All fields in upsert unique key list must be non-auto-updatable", + ); + } + } + + // For perf reasons, simplicity and easier CAS support, we don't + // identifier-escape field names in query builder code, so here, we enforce + // the field names to be simple and not require any escaping. (Notice that + // we DO escape table names though.) + for (const field of Object.keys(this.table)) { + if (!field.match(/^[_a-z][_a-z0-9]*$/)) { + throw Error( + `Field name must be a simple identifier, but '${field}' passed`, + ); + } + } + } +} diff --git a/src/abstract/Shard.ts b/src/abstract/Shard.ts new file mode 100644 index 0000000..3318666 --- /dev/null +++ b/src/abstract/Shard.ts @@ -0,0 +1,154 @@ +import { maybeCall } from "../internal/misc"; +import { type Client } from "./Client"; +import type { Island } from "./Island"; +import type { Query } from "./Query"; +import type { QueryAnnotation, WhyClient } from "./QueryAnnotation"; +import type { Timeline } from "./Timeline"; + +/** + * Master freshness: reads always go to master. + */ +export const MASTER = Symbol("MASTER"); + +/** + * Stale replica freshness: reads always go to a replica, even if it's stale. + */ +export const STALE_REPLICA = Symbol("STALE_REPLICA"); + +/** + * Shard lives within an Island with one master and N replicas. + */ +export class Shard { + private shardClients = new WeakMap(); + + /** The last known Island number where this Shard was discovered. It may be + * out of date after the Shard is moved, and also it may be null in case there + * was no discovery happened yet. */ + public readonly lastKnownIslandNo: number | null = null; + + constructor( + /** Shard number. */ + public readonly no: number, + /** A middleware to wrap queries with. It's responsible for locating the + * right Island and retrying the call to body() (i.e. failed queries) in + * case e.g. a shard is moved to another Island. */ + public readonly runOnShard: ( + shardNo: number, + body: (island: Island, attempt: number) => Promise, + onAttemptError?: (error: unknown, attempt: number) => void, + ) => Promise, + ) {} + + /** + * Chooses the right Client to be used for this Shard. We don't memoize, + * because the Shard may relocate to another Island during re-discovery. + */ + async client( + timeline: Timeline | typeof MASTER | typeof STALE_REPLICA, + ): Promise { + const [client] = await this.runOnShard(this.no, async (island) => + this.clientImpl(island, timeline, undefined), + ); + return client; + } + + /** + * Runs a query after choosing the right Client (destination connection, + * Shard, annotation etc.) + */ + async run( + query: Query, + annotation: QueryAnnotation, + timeline: Timeline, + freshness: null | typeof MASTER | typeof STALE_REPLICA, + onAttemptError?: (error: unknown, attempt: number) => void, + ): Promise { + return this.runOnShard( + this.no, + async (island, attempt) => { + const [client, whyClient] = await this.clientImpl( + island, + freshness ?? timeline, + query.IS_WRITE ? true : undefined, + ); + + // Throws if e.g. the Shard was there by the moment we got its client + // above, but it probably disappeared (during migration) and appeared on + // some other Island. + const res = await query.run(client, { + ...annotation, + whyClient, + attempt: annotation.attempt + attempt, + }); + + if (query.IS_WRITE && freshness !== STALE_REPLICA) { + timeline.setPos( + await client.timelineManager.currentPos(), + maybeCall(client.timelineManager.maxLagMs), + ); + } + + return res; + }, + onAttemptError, + ); + } + + /** + * Throws if this Shard does not exist, or its Island is down, or something + * else is wrong with it. + */ + async assertDiscoverable(): Promise { + await this.client(MASTER); + } + + /** + * An extended Client selection logic. There are multiple reasons (8+ in total + * so far) why a master or a replica may be chosen to send the query to. We + * don't @Memoize, because the Shard may relocate to another Island during + * re-discovery, so we have to run this logic every time. + */ + private async clientImpl( + island: Island, + timeline: Timeline | typeof MASTER | typeof STALE_REPLICA, + isWrite: true | undefined, + ): Promise<[client: TClient, whyClient: WhyClient]> { + if (isWrite) { + return [this.withShard(island.master()), "master-bc-is-write"]; + } + + if (timeline === MASTER) { + return [this.withShard(island.master()), "master-bc-master-freshness"]; + } + + const replica = island.replica(); + + if (replica.role() !== "replica") { + return [this.withShard(replica), "master-bc-no-replicas"]; + } + + if (timeline === STALE_REPLICA) { + return [this.withShard(replica), "replica-bc-stale-replica-freshness"]; + } + + const isCaughtUp = timeline.isCaughtUp( + await replica.timelineManager.currentPos(), + ); + return isCaughtUp + ? [this.withShard(replica), isCaughtUp] + : [this.withShard(island.master()), "master-bc-replica-not-caught-up"]; + } + + /** + * Returns a Shard-aware Client from an Island Client. + */ + private withShard(client: TClient): TClient { + let shardClient = this.shardClients.get(client); + if (!shardClient) { + shardClient = client.withShard(this.no); + this.shardClients.set(client, shardClient); + } + + return shardClient; + } +} diff --git a/src/abstract/ShardError.ts b/src/abstract/ShardError.ts new file mode 100644 index 0000000..34acf53 --- /dev/null +++ b/src/abstract/ShardError.ts @@ -0,0 +1,19 @@ +/** + * This non-retriable error is thrown when the system cannot detect the target + * shard to work with (e.g. a null ID or a missing field or something else). + */ +export class ShardError extends Error { + constructor(message: string, where?: string) { + super(message); + + Object.defineProperty(this, "name", { + value: this.constructor.name, + writable: true, + enumerable: false, + }); + + if (where) { + this.stack += `\n on ${where}`; + } + } +} diff --git a/src/abstract/ShardIsNotDiscoverableError.ts b/src/abstract/ShardIsNotDiscoverableError.ts new file mode 100644 index 0000000..fd8d1d7 --- /dev/null +++ b/src/abstract/ShardIsNotDiscoverableError.ts @@ -0,0 +1,43 @@ +import type { Client } from "./Client"; +import type { Island } from "./Island"; +import type { SwallowedErrorLoggerProps } from "./Loggers"; +import { ShardError } from "./ShardError"; + +/** + * This non-retriable error is thrown when shardsDiscoverCache.cached() returns + * no shard with the requested number. + */ +export class ShardIsNotDiscoverableError extends ShardError { + constructor( + shardNo: number, + errors: SwallowedErrorLoggerProps[], + islands: Array>, + elapsed: number, + ) { + const now = new Date().toLocaleTimeString("en-US", { + hour12: false, + timeZoneName: "short", + }); + super( + `Shard ${shardNo} is not discoverable: no such Shard in the Cluster? some Islands are down? connections limit?` + + (errors.length > 0 + ? "\n" + + errors + .map( + ({ where, error, elapsed }) => + `- ${where}` + + (elapsed !== null ? ` (took ${elapsed} ms)` : "") + + ": " + + `${error?.toString() ?? error}`.trimEnd(), + ) + .sort() + .join("\n") + : ""), + "Islands " + + islands + .map(({ no, clients }) => `${no}@${clients[0].options.name}`) + .join(", ") + + `; cached discovery took ${elapsed} ms at ${now}`, + ); + } +} diff --git a/src/abstract/ShardNamer.ts b/src/abstract/ShardNamer.ts new file mode 100644 index 0000000..a6dfdae --- /dev/null +++ b/src/abstract/ShardNamer.ts @@ -0,0 +1,65 @@ +import type { MaybeCallable } from "../internal/misc"; + +/** + * Options for ShardNamer constructor. + */ +export interface ShardNamerOptions { + /** A format string to turn a Shard number to Shard name (e.g. "sh%04d"). */ + nameFormat: string; + /** A DB engine query that should return the names of Shards served by this + * Client. */ + discoverQuery: MaybeCallable; +} + +/** + * Client-specific logic on how to synchronously convert an ID into Shard number + * (only for the use cases when ID is prefixed with a Shard number), how to + * build Shard names, and how to extract Shard number from a Shard name. + */ +export abstract class ShardNamer { + /** + * Synchronously extracts Shard number from an ID prefix, for the use cases + * where IDs have this information. + */ + abstract shardNoByID(id: string): number; + + /** Number of decimal digits in an ID allocated for Shard number. Calculated + * dynamically based on `ShardNamerOptions#nameFormat` (e.g. for "sh%04d", it + * will be 4 since it expands to "sh0012"). */ + protected readonly shardNoPadLen: number; + + /** + * Initializes an instance of ShardNamer. + */ + constructor(readonly options: ShardNamerOptions) { + const match = this.shardNameByNo(0).match(/(\d+)/); + this.shardNoPadLen = match ? match[1].length : 0; + if (!this.shardNoPadLen) { + throw Error("Invalid nameFormat value"); + } + } + + /** + * Converts a Shard name to Shard number. Returns null if it's not a correct + * Shard name. + */ + shardNoByName(name: string): number | null { + const match = name?.match(/(\d+)/); + const no = match ? parseInt(match[1]) : null; + return no !== null && name === this.shardNameByNo(no) ? no : null; + } + + /** + * Builds the Shard name (e.g. for PG, "schema name") by Shard number using + * `ShardNamerOptions#nameFormat`. + * + * E.g. nameFormat="sh%04d" generates names like "sh0042". + */ + shardNameByNo(no: number): string { + return this.options.nameFormat.replace( + /%(0?)(\d+)[sd]/, + (_, zero: string, d: string) => + no.toString().padStart(zero ? parseInt(d) : 0, "0"), + ); + } +} diff --git a/src/abstract/Timeline.ts b/src/abstract/Timeline.ts new file mode 100644 index 0000000..8a1d093 --- /dev/null +++ b/src/abstract/Timeline.ts @@ -0,0 +1,125 @@ +const SEP = ":"; + +/** + * The reason why the decision that this replica timeline is "good enough" has + * been made. + */ +export type TimelineCaughtUpReason = + | false + | "replica-bc-master-state-unknown" + | "replica-bc-caught-up" + | "replica-bc-pos-expired"; + +/** + * Even when pos is expired, we still continue to serialize it for some time for + * better debugging. Having this gap allows the system trigger + * "replica-bc-pos-expired" reason of a replica choice longer. + */ +const SERIALIZE_EXPIRATION_GAP_MS = 600 * 1000; + +/** + * Tracks replication lag timeline position at master per "user" and Ent. + * - serialization format: "pos:expiresAt" + * - wipes expired records (expiration is calculated at assignment moment) + * + * How replication lag (timeline) tracking works: for each + * microshard+Ent+"user", we know the “last write-ahead log write position” + * which that user (typically, VC#principal) made recently. This info can be + * propagated through e.g. user's session and push notifications/subscriptions + * channels automatically (“serialized timeline” and “timelines merging”). So + * the next time the same user tries to read the data from the same Ent on the + * same microshard, Ent Framework makes a choice, whether the replica is “good + * enough” for this already; if not, it falls back to master read. I.e. the data + * is not granular to individual Ent ID, it’s granular to the + * user+Ent+microshard, and thus it is decoupled from IDs. + */ +export class Timeline { + private saved = false; + + constructor( + private state: + | "unknown" + | { readonly pos: bigint; readonly expiresAt: number } = "unknown", + ) {} + + static deserialize( + data: string | undefined, + prevTimeline: Timeline | null, + ): Timeline { + const parts = data ? data.split(SEP) : []; + const pos = BigInt(parts[0] || "0"); + + if ( + prevTimeline && + prevTimeline.state !== "unknown" && + prevTimeline.state.pos >= pos + ) { + // The previous timeline holds a more recent WAL position than the one + // we're deserializing, so we should respect it better. + return prevTimeline; + } + + return new this( + pos ? { pos, expiresAt: parseInt(parts[1] || "0") } : "unknown", + ); + } + + static cloneMap( + timelines: ReadonlyMap, + ): Map { + const copy = new Map(); + for (const [key, timeline] of timelines.entries()) { + if (timeline.state !== "unknown") { + copy.set(key, new Timeline(timeline.state)); + } + } + + return copy; + } + + serialize(): string | undefined { + return this.state === "unknown" || + Date.now() >= this.state.expiresAt + SERIALIZE_EXPIRATION_GAP_MS + ? undefined + : this.state.pos.toString() + SEP + this.state.expiresAt; + } + + setPos(pos: bigint, maxLagMs: number): void { + if (this.state !== "unknown" && pos < this.state.pos) { + // Tried to set a "worse" pos than current, so just ignore both the pos + // and expiresAt timestamp. + return; + } + + // Trying to set a "more recent" pos than current, or the same pos as + // current (e.g. when replication lag is tracked via timestamp simulation). + this.state = { pos, expiresAt: Date.now() + maxLagMs }; + this.saved = false; + } + + isCaughtUp(replicaPos: bigint): TimelineCaughtUpReason { + return this.state === "unknown" + ? "replica-bc-master-state-unknown" + : replicaPos >= this.state.pos + ? "replica-bc-caught-up" + : Date.now() >= this.state.expiresAt + ? "replica-bc-pos-expired" + : false; + } + + isSaved(): boolean { + if (this.state === "unknown" || Date.now() >= this.state.expiresAt) { + return true; + } + + return this.saved; + } + + setIsSaved(): void { + this.saved = true; + } + + reset(): void { + this.state = "unknown"; + } +} diff --git a/src/abstract/TimelineManager.ts b/src/abstract/TimelineManager.ts new file mode 100644 index 0000000..634303f --- /dev/null +++ b/src/abstract/TimelineManager.ts @@ -0,0 +1,60 @@ +import { maybeCall, type MaybeCallable } from "../internal/misc"; + +/** + * A side effect based container which holds the current master or replica + * timeline position. For master, the expectation is that the pos will be + * updated after each query only, so no need to use refreshMs. For replica, it's + * also updated after each query PLUS the class will call triggerRefresh() hook + * not more often than every refreshMs interval. + */ +export class TimelineManager { + private pos = BigInt(0); + private changeTime = 0; + private triggerRefreshPromise: Promise | null = null; + + constructor( + /** Time interval after which a replica is declared as "caught up" even if + * it's not caught up. This is to not read from master forever when + * something has happened with the replica. */ + public readonly maxLagMs: MaybeCallable, + /** Up to how often we call triggerRefresh(). */ + private refreshMs: MaybeCallable, + /** This method is called from time to time to refresh the data which is + * later returned by currentPos(). Makes sense for replica connections which + * execute queries rarely: for them, the framework triggers the update when + * the fresh data is needed. */ + private triggerRefresh: () => Promise, + ) {} + + /** + * Returns the current Client's replication timeline position (e.g. WAL + * position). + */ + async currentPos(): Promise { + const refreshMs = maybeCall(this.refreshMs); + if (performance.now() > this.changeTime + refreshMs) { + // Outdated pos; refresh it, also coalesce concurrent requests if any. + // Notice that we run this logic not only for replicas, but for masters + // too, because we can't distinguish the server role at this place. It's + // not a big deal and results into just one extra query within refreshMs + // time interval (which is typically ~1 second). + try { + this.triggerRefreshPromise ??= this.triggerRefresh(); + await this.triggerRefreshPromise; + } finally { + this.triggerRefreshPromise = null; + } + } + + return this.pos; + } + + /** + * Sets the actual timeline pos. Must be called by the Client after each + * interaction with the database. + */ + setCurrentPos(pos: bigint, force?: boolean): void { + this.pos = pos > this.pos || force ? pos : this.pos; + this.changeTime = performance.now(); + } +} diff --git a/src/abstract/__tests__/Cluster.test.ts b/src/abstract/__tests__/Cluster.test.ts new file mode 100644 index 0000000..15a0641 --- /dev/null +++ b/src/abstract/__tests__/Cluster.test.ts @@ -0,0 +1,110 @@ +import { MASTER } from ".."; +import { join } from "../../internal/misc"; +import { + TCPProxyServer, + TEST_CONFIG, + testCluster, +} from "../../pg/__tests__/test-utils"; +import { PgQueryInsert } from "../../pg/PgQueryInsert"; +import { PgSchema } from "../../pg/PgSchema"; +import { ID } from "../../types"; + +jest.useFakeTimers({ advanceTimers: true }); + +let connStuckServer: TCPProxyServer; + +beforeEach(async () => { + connStuckServer = new TCPProxyServer({ + host: TEST_CONFIG.host!, + port: TEST_CONFIG.port!, + delayOnConnect: 1000000, + }); +}); + +afterEach(async () => { + testCluster.options.islands = [{ no: 0, nodes: [TEST_CONFIG] }]; + testCluster.options.shardsDiscoverIntervalMs = 1000000; + testCluster.options.reloadIslandsIntervalMs = 50; + await testCluster.rediscover(); +}); + +test("Shard is not discoverable error when connection got stuck", async () => { + const STUCK_CONFIG = { + ...TEST_CONFIG, + ...(await connStuckServer.hostPort()), + connectionTimeoutMillis: 5000, + }; + testCluster.options.islands = [ + { + no: 0, + nodes: [ + { ...STUCK_CONFIG, isAlwaysLaggingReplica: false, nameSuffix: "0" }, + { ...STUCK_CONFIG, isAlwaysLaggingReplica: true, nameSuffix: "0" }, + ], + }, + { + no: 1, + nodes: [{ ...STUCK_CONFIG, nameSuffix: "1" }], + }, + ]; + + const shard = testCluster.shardByNo(0); + + const error1 = await join([ + jest.advanceTimersByTimeAsync(120000), + shard.client(MASTER), + ]) + .then(() => null) + .catch((error: unknown) => `${error}`); + expect(error1?.replace(/\d{3,}/g, "NNN")).toMatchSnapshot(); + + const error2 = await shard + .client(MASTER) + .then(() => null) + .catch((error: unknown) => `${error}`); + expect(error2?.replace(/\d{3,}/g, "NNN")).toMatchSnapshot(); +}); + +test("Shard is not discoverable error when no such shard", async () => { + const shard = testCluster.shardByNo(1000); + const error = await shard + .client(MASTER) + .then(() => null) + .catch((error: unknown) => `${error}`); + expect(error?.replace(/\d{3,}/g, "NNN")).toEqual( + "ShardIsNotDiscoverableError: Shard NNN is not discoverable: no such Shard in the Cluster? some Islands are down? connections limit?", + ); +}); + +test("batcher() memoizes", async () => { + const client = await testCluster.shardByNo(0).client(MASTER); + + const schemaA = new PgSchema("test", { id: { type: ID } }, []); + const schemaB = new PgSchema("test", { id: { type: Number } }, []); + const PgRunnerInsert = new PgQueryInsert(schemaA, { id: "42" }).RUNNER_CLASS; + + const batcher1 = client.batcher( + PgQueryInsert, + schemaA, + "", + false, + () => new PgRunnerInsert(schemaA, client), + ); + const batcher2 = client.batcher( + PgQueryInsert, + schemaA, + "", + false, + () => new PgRunnerInsert(schemaA, client), + ); + expect(batcher1).toBe(batcher2); + + const batcher3 = client.batcher( + PgQueryInsert, + schemaB, + "", + false, + () => new PgRunnerInsert(schemaB, client), + ); + expect(batcher1).not.toBe(batcher3); +}); diff --git a/src/abstract/__tests__/Island.test.ts b/src/abstract/__tests__/Island.test.ts new file mode 100644 index 0000000..fe9a8f4 --- /dev/null +++ b/src/abstract/__tests__/Island.test.ts @@ -0,0 +1,126 @@ +import { testCluster } from "../../pg/__tests__/test-utils"; +import type { Client, ClientConnectionIssue } from "../Client"; +import { Island } from "../Island"; + +const CLIENT1 = { + options: { name: "client1" }, + role: () => "master", + connectionIssue: () => null, +} as Client; + +const CLIENT2 = { + options: { name: "client2" }, + role: () => "replica", + connectionIssue: () => null, +} as Client; + +const CLIENT3 = { + options: { name: "client3" }, + role: () => "replica", + connectionIssue: () => null, +} as Client; + +const ISLAND_OPTIONS = { + no: 0, + createShard: () => testCluster.globalShard(), +}; + +const CONNECTION_ISSUE: ClientConnectionIssue = { + timestamp: new Date(), + cause: Error("test"), + postAction: "fail", + kind: "data-on-server-is-unchanged", + comment: "test", +}; + +test("fails on an empty island", async () => { + expect(() => new Island({ ...ISLAND_OPTIONS, clients: [] })).toThrowError(); +}); + +test("returns 1st client if it's the only one", async () => { + const island = new Island({ ...ISLAND_OPTIONS, clients: [CLIENT1] }); + expect(island.master()).toBe(CLIENT1); + expect(island.replica()).toBe(CLIENT1); +}); + +test("returns master and replica", async () => { + const island = new Island({ ...ISLAND_OPTIONS, clients: [CLIENT1, CLIENT2] }); + expect(island.master()).toBe(CLIENT1); + expect(island.replica()).toBe(CLIENT2); +}); + +test("replica() returns master if replica is unhealthy", async () => { + const island = new Island({ ...ISLAND_OPTIONS, clients: [CLIENT1, CLIENT2] }); + jest.spyOn(CLIENT2, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + expect(island.replica()).toBe(CLIENT1); +}); + +test("replica() returns master if both replicas are unhealthy", async () => { + const island = new Island({ + ...ISLAND_OPTIONS, + clients: [CLIENT1, CLIENT2, CLIENT3], + }); + jest.spyOn(CLIENT2, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + jest.spyOn(CLIENT3, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + expect(island.replica()).toBe(CLIENT1); +}); + +test("replica() returns another replica is one is unhealthy", async () => { + const island = new Island({ + ...ISLAND_OPTIONS, + clients: [CLIENT1, CLIENT2, CLIENT3], + }); + jest.spyOn(CLIENT2, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + expect(island.replica()).toBe(CLIENT3); +}); + +test("replica() returns replica if both master and replica are unhealthy", async () => { + const island = new Island({ ...ISLAND_OPTIONS, clients: [CLIENT1, CLIENT2] }); + jest.spyOn(CLIENT1, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + jest.spyOn(CLIENT2, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + expect(island.master()).toBe(CLIENT1); + expect(island.replica()).toBe(CLIENT2); +}); + +test("if there are two masters reported, master() return the healthy one", async () => { + const island = new Island({ ...ISLAND_OPTIONS, clients: [CLIENT1, CLIENT2] }); + jest.spyOn(CLIENT1, "role").mockReturnValue("master"); + jest.spyOn(CLIENT1, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + jest.spyOn(CLIENT2, "role").mockReturnValue("master"); + jest.spyOn(CLIENT2, "connectionIssue").mockReturnValue(null); + expect(island.replica()).toBe(CLIENT2); + expect(island.master()).toBe(CLIENT2); +}); + +test("master() always returns known master, even if it's unhealthy", async () => { + const island = new Island({ ...ISLAND_OPTIONS, clients: [CLIENT1, CLIENT2] }); + jest.spyOn(CLIENT1, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + expect(island.master()).toBe(CLIENT1); +}); + +test("master() returns replica as master if there is no master at all", async () => { + const island = new Island({ ...ISLAND_OPTIONS, clients: [CLIENT2, CLIENT3] }); + jest.spyOn(CLIENT2, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + expect(island.master()).toBe(CLIENT3); +}); + +test("master() returns unknown node as master if all other nodes reported themselves as replicas", async () => { + jest.spyOn(CLIENT1, "role").mockReturnValue("replica"); + jest.spyOn(CLIENT2, "role").mockReturnValue("replica"); + jest.spyOn(CLIENT3, "role").mockReturnValue("unknown"); + const island = new Island({ + ...ISLAND_OPTIONS, + clients: [CLIENT1, CLIENT2, CLIENT3], + }); + expect(island.master()).toBe(CLIENT3); + jest.spyOn(CLIENT3, "connectionIssue").mockReturnValue(CONNECTION_ISSUE); + expect(island.master()).toBe(CLIENT3); +}); + +test("master() and replica() still work when nodes switch suddenly after reclassification", async () => { + const island = new Island({ ...ISLAND_OPTIONS, clients: [CLIENT1, CLIENT2] }); + jest.spyOn(CLIENT1, "role").mockReturnValue("replica"); + jest.spyOn(CLIENT2, "role").mockReturnValue("master"); + expect(island.master()).toBe(CLIENT2); + expect(island.replica()).toBe(CLIENT1); +}); diff --git a/src/abstract/__tests__/LocalCache.test.ts b/src/abstract/__tests__/LocalCache.test.ts new file mode 100644 index 0000000..25bd25d --- /dev/null +++ b/src/abstract/__tests__/LocalCache.test.ts @@ -0,0 +1,29 @@ +import delay from "delay"; +import waitForExpect from "wait-for-expect"; +import { LocalCache } from "../LocalCache"; + +jest.useFakeTimers({ advanceTimers: true }); + +test("writes and reads", async () => { + const KEY = `key${Date.now()}`; + + const cache = new LocalCache<{ some: string }>({ + dir: "/tmp/ent-framework-local-cache-test", + loggers: { swallowedErrorLogger: () => {} }, + expirationMs: 1000000, + }); + expect(await cache.get(KEY)).toBeNull(); + + await cache.set(KEY, { some: "42" }); + await delay(100); + expect(await cache.get(KEY)).toEqual({ some: "42" }); + + await cache.set(KEY, { some: "101" }); + await delay(100); + expect(await cache.get(KEY)).toEqual({ some: "101" }); + + jest.advanceTimersByTime(cache.options.cleanupFirstRunDelayMs * 2); + jest.advanceTimersByTime(cache.options.expirationMs); + + await waitForExpect(async () => expect(await cache.get(KEY)).toBeNull()); +}); diff --git a/src/abstract/__tests__/Shard.test.ts b/src/abstract/__tests__/Shard.test.ts new file mode 100644 index 0000000..14ce6f0 --- /dev/null +++ b/src/abstract/__tests__/Shard.test.ts @@ -0,0 +1,8 @@ +import { testCluster } from "../../pg/__tests__/test-utils"; + +test("Shard.lastKnownIslandNo", async () => { + const shard = testCluster.shardByNo(1); + expect(shard.lastKnownIslandNo).toBeNull(); + await testCluster.rediscover(); + expect(shard.lastKnownIslandNo).toEqual(0); +}); diff --git a/src/abstract/__tests__/Timeline.test.ts b/src/abstract/__tests__/Timeline.test.ts new file mode 100644 index 0000000..0cadc12 --- /dev/null +++ b/src/abstract/__tests__/Timeline.test.ts @@ -0,0 +1,46 @@ +import { Timeline } from "../Timeline"; + +jest.useFakeTimers({ advanceTimers: true }); + +test("serialize and deserialize", async () => { + const timeline = new Timeline(); + + expect(timeline.serialize()).toEqual(undefined); + + timeline.setPos(BigInt(42), 10000 /* ms */); + expect(timeline.serialize()).toEqual( + Timeline.deserialize(timeline.serialize(), null).serialize(), + ); + expect(timeline.isCaughtUp(BigInt(50))).toBeTruthy(); + expect(timeline.isCaughtUp(BigInt(40))).not.toBeTruthy(); + + timeline.setPos(BigInt(5), 10000 /* ms */); // 5 < 52, so it's a no-op + expect(timeline.isCaughtUp(BigInt(50))).toBeTruthy(); + expect(timeline.isCaughtUp(BigInt(40))).not.toBeTruthy(); +}); + +test("cloneMap", () => { + const map = new Map(); + map.set("unk", new Timeline("unknown")); + map.set("p1", new Timeline({ pos: BigInt(1), expiresAt: Date.now() })); + + const copy = Timeline.cloneMap(map); + + expect(copy.get("unk") === map.get("unk")).toBeFalsy(); + expect(copy.get("unk")).toBeUndefined(); + + expect(copy.get("p1") === map.get("p1")).toBeFalsy(); + expect(copy.get("p1")!.serialize()).toEqual(map.get("p1")!.serialize()); + + const prevP1Ser = map.get("p1")!.serialize(); + map.get("p1")!.setPos(BigInt(42), 42); + expect(copy.get("p1")!.serialize()).toEqual(prevP1Ser); +}); + +test("isCaughtUp pos expiration", async () => { + const timeline = new Timeline(); + timeline.setPos(BigInt(1), 20000); + expect(timeline.isCaughtUp(BigInt(0))).toEqual(false); + jest.advanceTimersByTime(30000); + expect(timeline.isCaughtUp(BigInt(0))).toEqual("replica-bc-pos-expired"); +}); diff --git a/src/abstract/__tests__/__snapshots__/Cluster.test.ts.snap b/src/abstract/__tests__/__snapshots__/Cluster.test.ts.snap new file mode 100644 index 0000000..f3f0aa5 --- /dev/null +++ b/src/abstract/__tests__/__snapshots__/Cluster.test.ts.snap @@ -0,0 +1,15 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Shard is not discoverable error when connection got stuck 1`] = ` +"ShardIsNotDiscoverableError: Shard 0 is not discoverable: no such Shard in the Cluster? some Islands are down? connections limit? +- TestPgClient(test-pool(replica=false)-0).shardNos (took NNN ms): Error: Connection terminated due to connection timeout +- TestPgClient(test-pool(replica=false)-1).shardNos (took NNN ms): Error: Connection terminated due to connection timeout +- TestPgClient(test-pool(replica=true)-0).shardNos (took NNN ms): Error: Connection terminated due to connection timeout" +`; + +exports[`Shard is not discoverable error when connection got stuck 2`] = ` +"ShardIsNotDiscoverableError: Shard 0 is not discoverable: no such Shard in the Cluster? some Islands are down? connections limit? +- TestPgClient(test-pool(replica=false)-0).shardNos (took NNN ms): Error: Connection terminated due to connection timeout +- TestPgClient(test-pool(replica=false)-1).shardNos (took NNN ms): Error: Connection terminated due to connection timeout +- TestPgClient(test-pool(replica=true)-0).shardNos (took NNN ms): Error: Connection terminated due to connection timeout" +`; diff --git a/src/abstract/index.ts b/src/abstract/index.ts new file mode 100644 index 0000000..ea3e199 --- /dev/null +++ b/src/abstract/index.ts @@ -0,0 +1,24 @@ +/** + * @file Automatically generated by barrelsby. + */ + +export * from "./Batcher"; +export * from "./Client"; +export * from "./ClientError"; +export * from "./Cluster"; +export * from "./Island"; +export * from "./Loader"; +export * from "./LocalCache"; +export * from "./Loggers"; +export * from "./Query"; +export * from "./QueryAnnotation"; +export * from "./QueryBase"; +export * from "./QueryPing"; +export * from "./Runner"; +export * from "./Schema"; +export * from "./Shard"; +export * from "./ShardError"; +export * from "./ShardIsNotDiscoverableError"; +export * from "./ShardNamer"; +export * from "./Timeline"; +export * from "./TimelineManager"; diff --git a/src/abstract/internal/misc.ts b/src/abstract/internal/misc.ts new file mode 100644 index 0000000..0362355 --- /dev/null +++ b/src/abstract/internal/misc.ts @@ -0,0 +1,3 @@ +export const OP_TIMELINE_POS_REFRESH = "TIMELINE_POS_REFRESH"; +export const OP_PING = "PING"; +export const OP_SHARD_NOS = "SHARD_NOS"; diff --git a/src/ent/BaseEnt.ts b/src/ent/BaseEnt.ts new file mode 100644 index 0000000..83ece1f --- /dev/null +++ b/src/ent/BaseEnt.ts @@ -0,0 +1,40 @@ +import type { Client } from "../abstract/Client"; +import type { Cluster } from "../abstract/Cluster"; +import type { Schema } from "../abstract/Schema"; +import type { Table, UniqueKey } from "../types"; +import { CacheMixin } from "./mixins/CacheMixin"; +import { ConfigMixin } from "./mixins/ConfigMixin"; +import type { HelpersClass } from "./mixins/HelpersMixin"; +import { HelpersMixin } from "./mixins/HelpersMixin"; +import { PrimitiveMixin } from "./mixins/PrimitiveMixin"; + +/** + * This is a helper function to create new Ent classes. Run once per each + * Ent+Cluster on app boot. See examples in __tests__/TestObjects.ts and + * EntTest.ts. + * + * Since all Ent objects are immutable (following the modern practices), + * 1. Ent is not a DataMapper pattern; + * 2. Ent is not an ActiveRecord; + * 3. At last, Ent is not quite a DAO pattern too. + * + * We assume that Ents are very simple (we don't need triggers or multi-ent + * touching mutations), because we anyway have a GraphQL layer on top of it. + * + * Finally, a naming decision has been made: we translate database field names + * directly to Ent field names, no camelCase. This has proven its simplicity + * benefits in the past: the less translation layers we have, the easier it is + * to debug and develop. + */ +export function BaseEnt< + TTable extends Table, + TUniqueKey extends UniqueKey, + TClient extends Client, +>( + cluster: Cluster, + schema: Schema, +): HelpersClass { + return HelpersMixin( + CacheMixin(PrimitiveMixin(ConfigMixin(class {}, cluster, schema))), + ); +} diff --git a/src/ent/Configuration.ts b/src/ent/Configuration.ts new file mode 100644 index 0000000..adaea78 --- /dev/null +++ b/src/ent/Configuration.ts @@ -0,0 +1,144 @@ +import type { + FieldOfIDType, + FieldOfIDTypeRequired, + Row, + Table, +} from "../types"; +import type { ShardAffinity } from "./ShardAffinity"; +import type { + AfterMutationTrigger, + AfterUpdateTrigger, + BeforeMutationTrigger, + BeforeUpdateTrigger, + DeleteTrigger, + DepsBuilder, + InsertTrigger, +} from "./Triggers"; +import type { Ent } from "./types"; +import type { Validation, ValidationRules } from "./Validation"; +import type { VC } from "./VC"; + +/** + * Strongly typed configuration framework to force TS auto-infer privacy + * callbacks arguments types (which are not Ents, but row-like inputs). + * + * Motivation: + * 1. We MUST resolve privacyXyz rules below lazily, at actual operation; + * otherwise in case of cyclic Ent dependencies between EntA and EntB, one of + * them will be magically undefined. + * 2. We can’t define these parameter as BaseEnt arguments: privacy rules may + * refer the derived Ent itself and other Ents and thus produce cyclic + * dependencies. TS doesn't allow to work with such cyclic dependencies + * during the class is defining. + * 3. Configuration can’t be just returned from a virtual method, because in TS, + * type inference in return values is poor: + * https://github.com/Microsoft/TypeScript/issues/31273 + */ +export class Configuration { + /** Defines how to locate a Shard at Ent insert time. See ShardAffinity for + * more details. + * + * 1. GLOBAL_SHARD: places the Ent in the global Shard (0). + * 2. `[]`: places the Ent in a random Shard. The "randomness" of the "random + * Shard" is deterministic by the Ent's unique key at the moment of + * insertion (if it's defined; otherwise completely random). This helps two + * racy insert operations running concurrently to choose the same Shard for + * the Ent to be created in, so only one of them will win, instead of both + * winning and mistakenly creating the Ent duplicates. I.e. having the same + * value in unique key forces the engine to target the same "random" Shard. + * 3. `["field1", "field2", ...]`: places the Ent in the Shard that is pointed + * to by the value in field1 (if it's null, then field2 etc.). + * + * A special treatment is applied if a fieldN value in (3) points to the + * global Shard. In such a case, the Shard for the current Ent is chosen + * deterministic-randomly at insert time, as if [] is passed. This allows the + * Ent to refer other "owning" Ents of different types, some of which may be + * located in the global Shard. Keep in mind that, to locate such an Ent + * pointing to another Ent in the global Shard, an inverse for fieldN must be + * defined in most of the cases. */ + declare readonly shardAffinity: ShardAffinity>; + /** Inverses allow cross-Shard foreign keys & cross-Shard selection. If a + * field points to an Ent in another Shard, and we're e.g. selecting by a + * value in this field, inverses allow to locate Shard(s) of the Ent. */ + declare readonly inverses?: { + [k in FieldOfIDTypeRequired]?: { name: string; type: string }; + }; + /** If defined, forces all Ents of this class to have the value of that field + * equal to VC's principal at load time. This is a very 1st unavoidable check + * in the privacy rules chain, thus it's bullet-proof. */ + declare readonly privacyTenantPrincipalField?: Validation["tenantPrincipalField"]; + /** An attempt to load this Ent using an omni VC will "lower" that VC to the + * principal returned. Omni VC is always lowered. + * 1. If an Ent is returned, the lowered principal will be Ent#vc.principal. + * It is a way to delegate principal inference to another Ent. + * 2. If a string is returned, then it's treated as a principal ID. + * 3. If a null is returned, then a guest principal will be used. + * 4. Returning an omni principal or VC will result in a run-time error. */ + declare readonly privacyInferPrincipal: + | (( + vc: VC, + row: Row, + ) => Promise | string | null> | string | null) + | string + | null; + + /** Privacy rules checked on every row loaded from the DB. */ + declare readonly privacyLoad: ValidationRules["load"]; + /** Privacy rules checked before a row is inserted to the DB. + * - It the list is empty, then only omni VC can insert; it's typically a good + * option for Ents representing e.g. a user. + * - If no update/delete rules are defined, then privacyInsert rules are also + * run on update/delete by default. + * - Unless empty, the rules must include at least one Require() predicate, + * they can't entirely consist of AllowIf(). This is because for write rules + * (privacyInsert, privacyUpdate, privacyDelete) it's important to make sure + * that ALL rules permit the operation, not only one of them allows it; this + * is what Require() is exactly for. */ + declare readonly privacyInsert: ValidationRules["insert"]; + /** Privacy rules checked before a row is updated in the DB. + * - If not defined, privacyInsert rules are used. + * - The rules must include at least one Require() predicate. */ + declare readonly privacyUpdate?: ValidationRules["update"]; + /** Privacy rules checked before a row is deleted in the DB. + * - If not defined, privacyInsert rules are used. + * - The rules must include at least one Require() predicate. */ + declare readonly privacyDelete?: ValidationRules["delete"]; + /** Custom field values validators run before any insert/update. */ + declare readonly validators?: ValidationRules["validate"]; + /** Triggers run before every insert. */ + declare readonly beforeInsert?: Array>; + /** Triggers run before every update. */ + declare readonly beforeUpdate?: Array< + | BeforeUpdateTrigger + | [DepsBuilder, BeforeUpdateTrigger] + >; + /** Triggers run before every delete. */ + declare readonly beforeDelete?: Array>; + /** Triggers run before every insert/update/delete. Each trigger may also be + * passed as "React useEffect-like" tuple where the callback is executed only + * if the deps are modified. */ + declare readonly beforeMutation?: Array< + | BeforeMutationTrigger + | [DepsBuilder, BeforeMutationTrigger] + >; + /** Triggers run after every delete. */ + declare readonly afterInsert?: Array>; + /** Triggers run after every update. */ + declare readonly afterUpdate?: Array< + | AfterUpdateTrigger + | [DepsBuilder, AfterUpdateTrigger] + >; + /** Triggers run after every delete. */ + declare readonly afterDelete?: Array>; + /** Triggers run after every insert/update/delete. Each trigger may also be + * passed as "React useEffect-like" tuple where the callback is executed only + * if the deps are modified. */ + declare readonly afterMutation?: Array< + | AfterMutationTrigger + | [DepsBuilder, AfterMutationTrigger] + >; + + constructor(cfg: Configuration) { + Object.assign(this, cfg); + } +} diff --git a/src/ent/IDsCache.ts b/src/ent/IDsCache.ts new file mode 100644 index 0000000..3ceeee5 --- /dev/null +++ b/src/ent/IDsCache.ts @@ -0,0 +1,24 @@ +import QuickLRU from "quick-lru"; +import { objectId } from "../internal/objectId"; + +const SEP = ":"; + +interface EntClassAlike { + SCHEMA: object; +} + +export abstract class IDsCache { + private ids = new QuickLRU({ maxSize: 2000 }); + + has(Ent: EntClassAlike, id: string): boolean { + return this.ids.has(objectId(Ent) + SEP + id); + } + + add(Ent: EntClassAlike, id: string, value: boolean = true): void { + this.ids.set(objectId(Ent) + SEP + id, value); + } + + get(Ent: EntClassAlike, id: string): boolean | undefined { + return this.ids.get(objectId(Ent) + SEP + id); + } +} diff --git a/src/ent/Inverse.ts b/src/ent/Inverse.ts new file mode 100644 index 0000000..0006440 --- /dev/null +++ b/src/ent/Inverse.ts @@ -0,0 +1,213 @@ +import { Memoize } from "fast-typescript-memoize"; +import type { Client } from "../abstract/Client"; +import type { Cluster } from "../abstract/Cluster"; +import type { Query } from "../abstract/Query"; +import type { Schema } from "../abstract/Schema"; +import type { Shard } from "../abstract/Shard"; +import type { DesperateAny } from "../internal/misc"; +import { join } from "../internal/misc"; +import type { FieldOfIDTypeRequired, Table } from "../types"; +import { ID } from "../types"; +import type { ShardAffinity } from "./ShardAffinity"; +import { GLOBAL_SHARD } from "./ShardAffinity"; +import type { VC } from "./VC"; + +/** + * No DB unique indexes can include a nullable field and be really unique, so we + * simulate id1=NULL via just storing "0" in the Inverse, and Inverse abstracts + * this fact from the caller. + */ +const ZERO_NULL = "0"; + +/** + * Represents an Inverse assoc manager which knows how to modify/query Inverses. + * Parameter `name` is the Inverse's schema name (in relational databases, most + * likely a table name), and `type` holds both the name of the "parent" entity + * and the field name of the child (e.g. "org2users" when a field "org_id" in + * EntUser refers an EntOrg row). + */ +export class Inverse { + private cluster; + private shardAffinity; + private name; + private inverseSchema; + public readonly id2Field; + public readonly type; + + constructor({ + cluster, + shardAffinity, + id2Schema, + id2Field, + name, + type, + }: { + cluster: Cluster; + shardAffinity: ShardAffinity; + id2Schema: Schema; + id2Field: FieldOfIDTypeRequired; + name: string; + type: string; + }) { + this.cluster = cluster; + this.shardAffinity = shardAffinity; + this.inverseSchema = Inverse.buildInverseSchema(id2Schema, name); + this.id2Field = id2Field; + this.name = name; + this.type = type; + } + + /** + * Runs before a row with a pre-generated id2 was inserted to the main schema. + * Returns true if the Inverse row was actually inserted in the DB. + */ + async beforeInsert( + vc: VC, + id1: string | null, + id2: string, + ): Promise { + if (this.id2ShardIsInferrableFromShardAffinity(id1)) { + return false; + } + + const id = await this.run( + vc, + this.shard(id1), + this.inverseSchema.insert({ + type: this.type, + id1: id1 ?? ZERO_NULL, + id2, + }), + ); + return id !== null; + } + + /** + * Runs after a row was updated in the main schema. + */ + async afterUpdate( + vc: VC, + id1: string | null, + id2: string, + oldID1: string | null, + ): Promise { + if (id1 === oldID1) { + return; + } + + await join([ + this.afterDelete(vc, oldID1, id2), + this.beforeInsert(vc, id1, id2), + ]); + } + + /** + * Runs after a row was deleted in the main schema. + */ + async afterDelete(vc: VC, id1: string | null, id2: string): Promise { + if (this.id2ShardIsInferrableFromShardAffinity(id1)) { + return; + } + + const row = await this.run( + vc, + this.shard(id1), + this.inverseSchema.loadBy({ + type: this.type, + id1: id1 ?? ZERO_NULL, + id2, + }), + ); + if (row) { + await this.run(vc, this.shard(id1), this.inverseSchema.delete(row[ID])); + } + } + + /** + * Returns all id2s by a particular (id1, type) pair. The number of resulting + * rows is limited to not overload the database. + */ + async id2s(vc: VC, id1: string | null): Promise { + const rows = await this.run( + vc, + this.shard(id1), + this.inverseSchema.selectBy({ + type: this.type, + id1: id1 ?? ZERO_NULL, + }), + ); + return rows.map((row) => row.id2).sort(); + } + + /** + * Creates an Inverse schema which derives its id field's autoInsert from the + * passed id2 schema. The returned schema is heavily cached, so batching for + * it works efficiently even for different id2 schemas and different Inverse + * types (actually, it would work the same way even without `@Memoize` since + * Runner batches by schema hash, not by schema object instance, but anyways). + */ + @Memoize( + (id2Schema: Schema, name: string) => + id2Schema.table[ID].autoInsert + name, + ) + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + private static buildInverseSchema( + id2Schema: Schema, + name: string, + ) { + return new id2Schema.constructor( + name, + { + id: { type: ID, autoInsert: id2Schema.table[ID].autoInsert }, + created_at: { type: Date, autoInsert: "now()" }, + type: { type: String }, + id1: { type: ID }, + id2: { type: ID }, + }, + ["type", "id1", "id2"], + ); + } + + /** + * If the field is already mentioned in shardAffinity, and the referred parent + * object (id1) exists, we won't need to create an Inverse, because the engine + * will be able to infer the target Shard from shardAffinity. This method + * would return true in such a case. In fact, we could've still create an + * Inverse for this case, but in sake of keeping the database lean, we don't + * do it (useful when a field holds a reference to an "optionally sharded" + * Ent, like sometimes it point so an Ent which is sharded, and sometimes on + * an Ent in the global Shard). + */ + private id2ShardIsInferrableFromShardAffinity(id1: string | null): boolean { + return ( + id1 !== null && + this.cluster.shard(id1) !== this.cluster.globalShard() && + this.shardAffinity !== GLOBAL_SHARD && + this.shardAffinity.includes(this.id2Field) + ); + } + + /** + * A shortcut to run a query on the Shard of id1. + */ + private async run( + vc: VC, + shard: Shard, + query: Query, + ): Promise { + return shard.run( + query, + vc.toAnnotation(), + vc.timeline(shard, `${this.name}:${this.type}`), + vc.freshness, + ); + } + + /** + * Returns a target Shard for an id. + */ + private shard(id: string | null): Shard { + // id1=NULL Inverse is always put to the global Shard. + return id ? this.cluster.shard(id) : this.cluster.globalShard(); + } +} diff --git a/src/ent/QueryCache.ts b/src/ent/QueryCache.ts new file mode 100644 index 0000000..833fc88 --- /dev/null +++ b/src/ent/QueryCache.ts @@ -0,0 +1,160 @@ +import QuickLRU from "quick-lru"; +import { MASTER } from "../abstract/Shard"; +import { appendCaller } from "../internal/misc"; +import type { VC } from "./VC"; +import { VCWithQueryCache } from "./VCFlavor"; + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const OPS = [ + "loadNullable", + "loadByNullable", + "selectBy", + "select", + "count", + "exists", +] as const; + +export type AnyClass = new (...args: never[]) => unknown; + +type Op = (typeof OPS)[number]; + +/** + * Caches Ents loaded by a particular VC. I.e. the same query running for the + * same VC twice will quickly return the same Ents. This is typically enabled on + * web servers only, to deliver the fastest UI response. + */ +export class QueryCache { + private maxQueries?: number; + private byEntClass?: WeakMap< + AnyClass, + Partial>>> + >; + public readonly whyOff?: string; + + /** + * Creates the QueryCache object. It enable caching only if VCWithQueryCache + * was manually added to the VC by the user, otherwise caching is a no-op. + */ + constructor(vc: VC) { + if (vc.freshness === MASTER) { + this.whyOff = "MASTER freshness"; + return; + } + + const flavor = vc.flavor(VCWithQueryCache); + if (!flavor) { + this.whyOff = "No VCWithQueryCache flavor"; + } else if (flavor.options.maxQueries <= 0) { + this.whyOff = "VCWithQueryCache#maxQueries is not positive"; + } else { + this.maxQueries = flavor.options.maxQueries; + } + } + + /** + * Saves a Promise to the cache slot for `op`. If this Promise rejects, the + * slot will automatically be cleared (we don't cache rejected Promises to not + * have a risk of caching a transient DB error). + */ + set( + EntClass: AnyClass, + op: Op, + key: string, + value: Promise | undefined, + ): this { + if (!this.maxQueries) { + // Caching is turned off. + return this; + } + + let byOp = (this.byEntClass ??= new WeakMap()).get(EntClass); + if (!byOp) { + byOp = {}; + this.byEntClass.set(EntClass, byOp); + } + + const slot = (byOp[op] ??= new QuickLRU({ maxSize: this.maxQueries })); + if (value !== undefined) { + slot.set(key, value); + // As a side effect of value rejection, clear the cache slot. Note: + // although we don't re-throw in the callback, this does not swallow the + // rejection, because we don't save the result of value.catch() anywhere + // and don't return it. It's a pure side effect. + value.catch(() => slot!.delete(key)); + } else { + slot.delete(key); + } + + return this; + } + + /** + * Deletes cache slots or keys for an Ent. If key is null, skips the deletion. + * If key is undefined (i.e. not passed), then deletes all slots. + */ + delete(EntClass: AnyClass, ops: readonly Op[], key?: string | null): this { + if (key === null) { + return this; + } + + const byOp = this.byEntClass?.get(EntClass); + if (!byOp) { + return this; + } + + for (const op of ops) { + if (key === undefined) { + delete byOp[op]; + } else { + byOp[op]?.delete(key); + } + } + + return this; + } + + /** + * This method is non-async on intent. We store Promises in the cache, not end + * values, because we want the code to join awaiting an ongoing operation in + * case it's inflight already. + */ + get( + EntClass: AnyClass, + op: Op, + key: string, + ): Promise | undefined { + const byOp = this.byEntClass?.get(EntClass); + if (!byOp) { + return undefined; + } + + return byOp[op]?.get(key) as Promise | undefined; + } + + /** + * Read-through caching pattern. + */ + async through( + EntClass: AnyClass, + op: Op, + key: string, + creator: () => Promise, + ): Promise { + if (!this.maxQueries) { + // Caching is turned off. + return creator(); + } + + let value = this.get(EntClass, op, key); + if (value === undefined) { + value = creator(); + this.set(EntClass, op, key, value); + } + + try { + return await value; + } catch (e) { + throw appendCaller(e); + } + } +} diff --git a/src/ent/ShardAffinity.ts b/src/ent/ShardAffinity.ts new file mode 100644 index 0000000..54c4a21 --- /dev/null +++ b/src/ent/ShardAffinity.ts @@ -0,0 +1,24 @@ +import type { ID } from "../types"; + +/** + * The table is located in the global Shard (0). + */ +export const GLOBAL_SHARD = "global_shard"; + +/** + * Defines Ent Shard collocation to some Ent's field when this Ent is inserted. + * - The Shard can always be Shard 0 ("global Shard"), be inferred based on the + * value in other Ent field during the insertion ("colocation"), or, in case + * colocation inference didn't succeed, be chosen pseudo-randomly at insertion + * time ("random Shard"). + * - E.g. a random Shard can also be chosen in case an empty array is passed to + * Shard affinity (like "always fallback"), or when a field's value points to + * a global Shard. + * - Passing ID to ShardAffinity is prohibited by TS. + */ +export type ShardAffinity< + TField extends string, + TF = Exclude, +> = + | typeof GLOBAL_SHARD + | (TField extends typeof ID ? readonly TF[] : readonly [TF, ...TF[]]); diff --git a/src/ent/ShardLocator.ts b/src/ent/ShardLocator.ts new file mode 100644 index 0000000..c914056 --- /dev/null +++ b/src/ent/ShardLocator.ts @@ -0,0 +1,312 @@ +import uniq from "lodash/uniq"; +import type { Client } from "../abstract/Client"; +import type { Cluster } from "../abstract/Cluster"; +import type { Shard } from "../abstract/Shard"; +import { ShardError } from "../abstract/ShardError"; +import { inspectCompact, mapJoin } from "../internal/misc"; +import type { Table } from "../types"; +import { ID } from "../types"; +import { EntNotFoundError } from "./errors/EntNotFoundError"; +import type { Inverse } from "./Inverse"; +import type { ShardAffinity } from "./ShardAffinity"; +import { GLOBAL_SHARD } from "./ShardAffinity"; +import type { VC } from "./VC"; +import { GUEST_ID } from "./VC"; + +/** + * Knows how to locate Shard(s) based on various inputs. In some contexts, we + * expect exactly one Shard returned, and in other contexts, multiple Shards are + * okay. + */ +export class ShardLocator< + TClient extends Client, + TTable extends Table, + TField extends string, +> { + private cluster; + private entName; + private shardAffinity; + private uniqueKey; + private inverses; + private globalShard; + private idAndShardAffinity; + + constructor({ + cluster, + entName, + shardAffinity, + uniqueKey, + inverses, + }: { + cluster: Cluster; + entName: string; + shardAffinity: ShardAffinity; + uniqueKey: readonly string[] | undefined; + inverses: ReadonlyArray>; + }) { + this.cluster = cluster; + this.entName = entName; + this.shardAffinity = shardAffinity; + this.idAndShardAffinity = [ + ID, + ...(this.shardAffinity instanceof Array ? this.shardAffinity : []), + ]; + this.uniqueKey = uniqueKey; + this.inverses = inverses; + this.globalShard = cluster.globalShard(); + } + + /** + * Called in a context when we must know exactly 1 Shard to work with (e.g. + * INSERT, UPSERT etc.). If op === "insert" (fallback to random Shard), then + * returns a random Shard in case when it can't infer the Shard number from + * the input (used in e.g. INSERT operations); otherwise throws ShardError + * (happens in e.g. UPSERT). + * + * The "randomness" of the "random Shard" is deterministic by the Ent's unique + * key (if it's defined), so Ents with the same unique key will map to the + * same "random" Shard (considering the total number of discovered Shards is + * unchanged). Notice that this logic applies at INSERT time: since we often + * times add Shards to the Cluster, we can't rely on it consistently at SELECT + * time (but relying at INSERT time is more or less fine: it protects against + * most of "unique key violation" problems, although still doesn't prevent all + * of them for a fraction of the second when the number of Shards has just + * been changed). + */ + async singleShardForInsert( + input: Record, + op: "insert" | "upsert", + ): Promise> { + let shard = await this.singleShardFromAffinity(input, op); + + if (!shard && op === "insert") { + shard = await this.cluster.randomShard( + this.uniqueKey?.length + ? this.uniqueKey.map((field) => input[field]) + : undefined, + ); + } + + if (!shard) { + throw new ShardError( + this.buildShardErrorMessage({ + op, + fields: + this.shardAffinity instanceof Array ? this.shardAffinity : [ID], + input, + }), + ); + } + + return shard; + } + + /** + * Called in a context when multiple Shards may be involved, e.g. when + * selecting Ents referred by some Inverses. May also return the empty list of + * Shards when, although there are fields with Inverses in input (i.e. the + * filtering is correct), there are no Inverse rows existing in the database. + */ + async multiShardsFromInput( + vc: VC, + input: Record, + op: string, + ): Promise>> { + const singleShard = await this.singleShardFromAffinity(input, op); + if (singleShard) { + return [singleShard]; + } + + // Scan Inverses from left to right (assuming the leftmost Inverses are + // lower in cardinality) and check whether our input has a filtering field + // defined for that Inverse. If so, locate Shards based on that 1st found + // field only (because it makes no sense to move to the next Inverse if we + // can use a previous Inverse already). + let hadInputFieldWithInverse = false; + const shards = new Set>(); + for (const inverse of this.inverses) { + const field = inverse.id2Field; + const id1 = input[field]; + if (id1 !== undefined) { + hadInputFieldWithInverse = true; + await mapJoin(id1 instanceof Array ? id1 : [id1], async (id1) => { + let id2s: string[]; + try { + id2s = await inverse.id2s(vc, id1); + } catch (e: unknown) { + throw e instanceof ShardError + ? new EntNotFoundError(this.entName, { [field]: id1 }, e) + : e; + } + + for (const id2 of id2s) { + const shard = await this.singleShardFromID(field, id2, op); + if (shard) { + shards.add(shard); + } + } + }); + break; + } + } + + if (!hadInputFieldWithInverse) { + const inverseFields = this.inverses.map(({ id2Field }) => id2Field); + throw new ShardError( + this.buildShardErrorMessage({ + op, + fields: uniq([ + ...(this.shardAffinity instanceof Array + ? this.shardAffinity + : [ID]), + ...inverseFields, + ]), + input, + }), + ); + } + + return [...shards]; + } + + /** + * A wrapper for Cluster#shard() which injects Ent name to the exception (in + * case of e.g. "Cannot locate Shard" exception). This is just a convenience + * for debugging. + * + * If this method returns null, that means the caller should give up trying to + * load the Ent with this ID, because it won't find it anyways (e.g. when we + * try to load a sharded Ent using an ID from the global Shard). This is + * identical to the case of an Ent not existing in the database. + */ + async singleShardFromID( + field: string, + id: string | null | undefined, + op: string, + ): Promise | null> { + try { + let shard: Shard; + + // GLOBAL_SHARD has precedence over a Shard number from ID (or any other + // fields, since global tables may refer to global tables only). This allows + // to move some previously sharded objects to the global Shard while doing + // some refactoring. + if (this.shardAffinity === GLOBAL_SHARD) { + shard = this.globalShard; + } else if (id === GUEST_ID) { + throw new ShardError( + this.buildShardErrorMessage({ + op, + why: "most likely you're trying to use a guest VC's principal instead of an ID", + }), + ); + } else { + if (id === null || id === undefined) { + throw new ShardError( + this.buildShardErrorMessage({ + op, + why: `you should not pass null or undefined value in "${field}" field`, + }), + ); + } + + shard = this.cluster.shard(id); + + if (shard.no === this.globalShard.no) { + // We're trying to load a sharded Ent using an ID from the global + // Shard. We know for sure that there will be no such Ent there then. + return null; + } + } + + // We want to throw ShardError early to wrap the possible exception with + // EntNotFoundError below. This is a little kludge, since on success, it + // will call into Shard#options.locateClient() twice (here and when + // running the actual query). Also, assertDiscoverable() is used only in + // this single place. + await shard.assertDiscoverable(); + + return shard; + } catch (e: unknown) { + throw e instanceof ShardError + ? new EntNotFoundError(this.entName, { [field]: id }, e) + : e; + } + } + /** + * All shards for this particular Ent depending on its affinity. + */ + async allShards(): Promise>> { + return this.shardAffinity === GLOBAL_SHARD + ? [this.globalShard] + : this.cluster.nonGlobalShards(); + } + + /** + * Infers Shard number from shardAffinity info and the input record. + * - Returns null if it can't do this; the caller should likely throw in this + * case (although not always). + * - If a field's value is an array of IDs from multiple Shards, then returns + * the 1st inferred Shard still. This is the current limitation: we don't + * even try to infer multiple Shards if we have some affinity fields in the + * request, which e.g. simplifies Ent creation logic. Cross-Shard logic is + * only enabled when using Inverses; see multiShardsFromInput(). + */ + private async singleShardFromAffinity( + input: Record, + op: string, + ): Promise | null> { + // For a low number of a very global objects only. ATTENTION: GLOBAL_SHARD + // has precedence over a Shard number from ID! This allows to move some + // previously sharded objects to the global Shard while doing some + // refactoring. + if (this.shardAffinity === GLOBAL_SHARD) { + return this.globalShard; + } + + // Explicit info about which Shard to use. + if (input["$shardOfID"] !== undefined) { + return this.singleShardFromID( + "$shardOfID", + input["$shardOfID"]?.toString(), + op, + ); + } + + // An explicit list of fields is passed in SHARD_AFFINITY. + for (const fromField of this.idAndShardAffinity) { + const fromValue = input[fromField]; + const value = fromValue instanceof Array ? fromValue[0] : fromValue; + if (typeof value === "string" && value) { + return this.singleShardFromID(fromField, value, op); + } + } + + // Couldn't detect Shard number from any of the sources. + return null; + } + + /** + * A helper to build uniform ShardError error messages. + */ + private buildShardErrorMessage({ + op, + why, + fields, + input, + }: { op: string } & ( + | { why: string; fields?: never; input?: never } + | { why?: never; fields: readonly string[]; input: object } + )): string { + throw new ShardError( + `${this.entName}: cannot detect shard in "${op}" query: ` + + (typeof why === "string" + ? why + : (fields.length > 1 + ? `at least one of non-empty "${fields.join(", ")}" fields` + : `non-empty "${fields[0]}" field`) + + " must be present at TOP LEVEL of the input, but got " + + inspectCompact(input)), + ); + } +} diff --git a/src/ent/TimelineStorage.ts b/src/ent/TimelineStorage.ts new file mode 100644 index 0000000..b911455 --- /dev/null +++ b/src/ent/TimelineStorage.ts @@ -0,0 +1,46 @@ +import defaults from "lodash/defaults"; +import type { MaybeCallable, PickPartial } from "../internal/misc"; +import { VC } from "./VC"; + +export interface TimelineStorageOptions { + merge?: (dataStrs: string[]) => string; + maxChunksPerPrincipal?: MaybeCallable; +} + +/** + * An abstract class that defines the interface for loading and storing + * timelines per VC principals. + */ +export abstract class TimelineStorage { + /** Default values for the constructor options. */ + static readonly DEFAULT_OPTIONS: Required< + PickPartial + > = { + merge: (dataStrs: string[]) => { + const vc = VC.createGuestPleaseDoNotUseCreationPointsMustBeLimited(); + vc.deserializeTimelines(...dataStrs); + return vc.serializeTimelines()!; + }, + maxChunksPerPrincipal: 10, + }; + + /** Client configuration options. */ + readonly options: Required; + + /** + * Loads the timelines from the storage for a given principal. + */ + abstract load(principal: string): Promise; + + /** + * Saves the timelines in the storage for a given principal. + */ + abstract save(principal: string, dataStr: string): Promise; + + /** + * Initializes an instance of TimelineStorage. + */ + constructor(options: TimelineStorageOptions) { + this.options = defaults({}, options, TimelineStorage.DEFAULT_OPTIONS); + } +} diff --git a/src/ent/Triggers.ts b/src/ent/Triggers.ts new file mode 100644 index 0000000..cc80c25 --- /dev/null +++ b/src/ent/Triggers.ts @@ -0,0 +1,396 @@ +import { deepEqual } from "../internal/deepEqual"; +import type { Flatten, Writeable } from "../internal/misc"; +import { join } from "../internal/misc"; +import type { + InsertInput, + Row, + RowWithID, + Table, + UpdateInput, + Value, +} from "../types"; +import type { VC } from "./VC"; + +/** + * Table -> trigger's before- and after-insert input. Below, we use InsertInput + * and not Row, because before and even after some INSERT, we may still not know + * some values of the row (they can be filled by the DB in e.g. autoInsert + * clause). InsertInput is almost a subset of Row, but it has stricter symbol + * keys: e.g. if some symbol key is non-optional in INSERT (aka doesn't have + * autoInsert), it will always be required in InsertInput too. + */ +export type TriggerInsertInput = Flatten< + InsertInput & RowWithID +>; + +/** + * Table -> trigger's before-update input. + */ +export type TriggerUpdateInput = Flatten< + UpdateInput +>; + +/** + * Table -> trigger's before- and after-update NEW row. Ephemeral (symbol) + * fields may or may not be passed depending on what the user passes to the + * update method. + */ +export type TriggerUpdateNewRow = Flatten< + Readonly< + Row & { + [K in keyof TTable & symbol]?: Value | undefined; + } + > +>; + +/** + * Table -> trigger's before- and after-update (or delete) OLD row. Ephemeral + * (symbol) fields are marked as always presented, but "never" typed, so they + * will be available for dereferencing in newOrOldRow of before/after mutation + * triggers without guard-checking of op value. + */ +export type TriggerUpdateOrDeleteOldRow = Flatten< + Readonly & Record> +>; + +/** + * Triggers could be used to simulate "transactional best-effort behavior" in a + * non-transactional combination of some services. Imagine we have a relational + * database and a queue service; each time we change something in the query, we + * want to schedule the ID to the queue. Queue service is faulty: if a queueing + * operation fails, we don't want the data to be stored to the DB afterwards. + * Queries are faulty too, but it's okay for us to have something added to the + * queue even if the corresponding query failed after it (a queue worker will + * just do a no-op since it anyway rechecks the source of truth in relational + * DBs). Queue service is like a write-ahead log for DB which always has + * not-less records than the DB. In this case, we have the following set of + * triggers: + * + * 1. beforeInsert: schedules ID to the queue (ID is known, see below why) + * 2. beforeUpdate: schedules ID to the queue + * 3. afterDelete: optionally schedule ID removal to the queue (notice "after") + * + * Notice that ID is always known in all cases, even in insertBefore triggers, + * because we split an INSERT operation into gen_id+insert parts, and the + * triggers are executed in between. + * + * Triggers are invoked sequentially. Any exception thrown in a before-trigger + * is propagated to the caller, and the DB operation is skipped. + * + * Triggers for beforeInsert and beforeUpdate can change their input parameter, + * the change will apply to the database. + * + * Naming convention for trigger arguments: + * 1. input: whatever is passed to the operation. Notice that due to us having + * autoInsert/autoUpdate fields, the set of fields can be incomplete here! + * 1. oldRow: the entire row in the DB which was there before the operation. All + * the fields will be presented there. + * 2. newRow: a row in the DB as it will looks like after the operation. Notice + * that it can be non precise, because we don't always reload the updated row + * from the database! What we do is just field by field application of input + * properties to oldRow. + */ + +export type InsertTrigger = ( + vc: VC, + args: { input: TriggerInsertInput }, // always knows ID even in beforeInsert +) => Promise | unknown; + +export type BeforeUpdateTrigger = ( + vc: VC, + args: { + newRow: TriggerUpdateNewRow; + oldRow: TriggerUpdateOrDeleteOldRow; + input: TriggerUpdateInput; + }, +) => Promise | unknown; + +export type AfterUpdateTrigger = ( + vc: VC, + args: { + newRow: TriggerUpdateNewRow; + oldRow: TriggerUpdateOrDeleteOldRow; + }, +) => Promise | unknown; + +export type DeleteTrigger = ( + vc: VC, + args: { + oldRow: TriggerUpdateOrDeleteOldRow; + }, +) => Promise | unknown; + +export type BeforeMutationTrigger = ( + vc: VC, + args: + | { + op: "INSERT"; + newOrOldRow: Readonly>; + input: TriggerInsertInput; + } + | { + op: "UPDATE"; + newOrOldRow: TriggerUpdateNewRow; + input: TriggerUpdateInput; + } + | { + op: "DELETE"; + newOrOldRow: TriggerUpdateOrDeleteOldRow; + /** We allow people to modify input of a DELETE operation, although it + * will be a no-op. This is for convenience: if we remained it + * read-only, then people would need to check `if (op !== "DELETE") ...` + * in their beforeMutation triggers, which is a boilerplate. */ + input: Writeable>; + }, +) => Promise | unknown; + +export type AfterMutationTrigger = ( + vc: VC, + args: + | { + op: "INSERT"; + newOrOldRow: Readonly>; + } + | { + op: "UPDATE"; + newOrOldRow: TriggerUpdateNewRow; + } + | { + op: "DELETE"; + newOrOldRow: TriggerUpdateOrDeleteOldRow; + }, +) => Promise | unknown; + +export type DepsBuilder = ( + vc: VC, + row: Flatten>>, +) => unknown[] | Promise; + +export class Triggers { + constructor( + private beforeInsert: Array>, + private beforeUpdate: Array< + [DepsBuilder | null, BeforeUpdateTrigger] + >, + private beforeDelete: Array>, + private beforeMutation: Array< + [DepsBuilder | null, BeforeMutationTrigger] + >, + private afterInsert: Array>, + private afterUpdate: Array< + [DepsBuilder | null, AfterUpdateTrigger] + >, + private afterDelete: Array>, + private afterMutation: Array< + [DepsBuilder | null, AfterMutationTrigger] + >, + ) {} + + hasInsertTriggers(): boolean { + return ( + this.beforeInsert.length > 0 || + this.beforeMutation.length > 0 || + this.afterInsert.length > 0 || + this.afterMutation.length > 0 + ); + } + + hasUpdateTriggers(): boolean { + return ( + this.beforeUpdate.length > 0 || + this.beforeMutation.length > 0 || + this.afterUpdate.length > 0 || + this.afterMutation.length > 0 + ); + } + + async wrapInsert( + func: (input: InsertInput & RowWithID) => Promise, + vc: VC, + input: InsertInput & RowWithID, + ): Promise { + if (!this.hasInsertTriggers()) { + return func(input); + } + + for (const triggerBeforeInsert of this.beforeInsert) { + // We clone the input to make different triggers calls independent: if the + // trigger e.g. stores input somewhere by reference, we don't want the + // next trigger to affect that place. + input = { ...input }; + await triggerBeforeInsert(vc, { input }); + } + + for (const [_, triggerBeforeMutation] of this.beforeMutation) { + input = { ...input }; + await triggerBeforeMutation(vc, { + op: "INSERT", + newOrOldRow: input, + input, + }); + } + + const output = await func(input); + + if (!output) { + // Insert failed (unique key constraint failed); don't run after-triggers. + return output; + } + + for (const triggerAfterInsert of this.afterInsert) { + await triggerAfterInsert(vc, { input }); + } + + for (const [_, triggerAfterMutation] of this.afterMutation) { + await triggerAfterMutation(vc, { op: "INSERT", newOrOldRow: input }); + } + + return output; + } + + async wrapUpdate( + func: (input: UpdateInput) => Promise, + vc: VC, + oldRow: TriggerUpdateOrDeleteOldRow, + input: UpdateInput, + ): Promise { + if (!this.hasUpdateTriggers()) { + return func(input); + } + + let newRow = buildUpdateNewRow(oldRow, input); + + for (const [depsBuilder, triggerBeforeUpdate] of this.beforeUpdate) { + if (await depsBuilderApproves(depsBuilder, vc, oldRow, newRow)) { + await triggerBeforeUpdate(vc, { newRow, oldRow, input }); + // Each call to triggerBefore() may potentially change the input, so we + // need to rebuild newRow each time to feed it to the next call of + // triggerBefore() and to the rest of triggerAfter. + newRow = buildUpdateNewRow(oldRow, input); + } + } + + for (const [depsBuilder, triggerBeforeMutation] of this.beforeMutation) { + if (await depsBuilderApproves(depsBuilder, vc, oldRow, newRow)) { + await triggerBeforeMutation(vc, { + op: "UPDATE", + newOrOldRow: newRow, + input, + }); + newRow = buildUpdateNewRow(oldRow, input); + } + } + + const output = await func(input); + + if (!output) { + // Update failed (no row with such ID); don't call after-triggers. + return output; + } + + for (const [depsBuilder, triggerAfterUpdate] of this.afterUpdate) { + if (await depsBuilderApproves(depsBuilder, vc, oldRow, newRow)) { + await triggerAfterUpdate(vc, { + newRow, + oldRow: oldRow as TriggerUpdateOrDeleteOldRow, + }); + } + } + + for (const [depsBuilder, triggerAfterMutation] of this.afterMutation) { + if (await depsBuilderApproves(depsBuilder, vc, oldRow, newRow)) { + await triggerAfterMutation(vc, { + op: "UPDATE", + newOrOldRow: newRow, + }); + } + } + + return output; + } + + async wrapDelete( + func: () => Promise, + vc: VC, + oldRow: TriggerUpdateOrDeleteOldRow, + ): Promise { + for (const triggerBeforeDelete of this.beforeDelete) { + await triggerBeforeDelete(vc, { oldRow }); + } + + for (const [_, triggerBeforeMutation] of this.beforeMutation) { + await triggerBeforeMutation(vc, { + op: "DELETE", + newOrOldRow: oldRow, + input: oldRow, + }); + } + + const output = await func(); + + if (!output) { + // Delete failed (no row with such ID); don't call after-triggers. + return output; + } + + for (const triggerAfterDelete of this.afterDelete) { + await triggerAfterDelete(vc, { oldRow }); + } + + for (const [_, triggerAfterMutation] of this.afterMutation) { + await triggerAfterMutation(vc, { + op: "DELETE", + newOrOldRow: oldRow, + }); + } + + return output; + } +} + +/** + * Simulates an update for a row, as if it's applied to the Ent. + * @ignore + */ +export function buildUpdateNewRow( + oldRow: Row, + input: UpdateInput, +): TriggerUpdateNewRow { + const newRow = { ...oldRow } as TriggerUpdateNewRow; + + for (const k of Object.getOwnPropertyNames(input)) { + if (input[k] !== undefined) { + (newRow as Record)[k] = input[k]; + } + } + + for (const k of Object.getOwnPropertySymbols(input)) { + if (input[k] !== undefined) { + (newRow as Record)[k] = input[k]; + } + } + + return newRow; +} + +/** + * Returns true if depsBuilder approves running the trigger (i.e. the deps are + * changed, or no depsBuilder is presented at all). + */ +async function depsBuilderApproves( + depsBuilder: DepsBuilder | null, + vc: VC, + oldRow: Row, + newRow: Row, +): Promise { + if (!depsBuilder) { + return true; + } + + const [depsOld, depsNew] = await join([ + depsBuilder(vc, oldRow), + depsBuilder(vc, newRow), + ]); + + return !deepEqual(depsOld, depsNew); +} diff --git a/src/ent/VC.ts b/src/ent/VC.ts new file mode 100644 index 0000000..2091ece --- /dev/null +++ b/src/ent/VC.ts @@ -0,0 +1,655 @@ +import { inspect } from "util"; +import delay from "delay"; +import { Memoize } from "fast-typescript-memoize"; +import compact from "lodash/compact"; +import type { Client } from "../abstract/Client"; +import type { Handler } from "../abstract/Loader"; +import { Loader } from "../abstract/Loader"; +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import type { Shard } from "../abstract/Shard"; +import { MASTER, STALE_REPLICA } from "../abstract/Shard"; +import { Timeline } from "../abstract/Timeline"; +import { minifyStack } from "../internal/misc"; +import type { TimelineStorage } from "./TimelineStorage"; +import { VCCaches } from "./VCCaches"; +import type { VCFlavor } from "./VCFlavor"; +import { VCWithStacks } from "./VCFlavor"; +import { VCTrace } from "./VCTrace"; + +/** + * Guest VC: has minimum permissions. Typically if the user is not logged in, + * this VC is used. + */ +export const GUEST_ID = "guest"; + +/** + * Temporary "omniscient" VC. Any Ent can be loaded with it, but this VC is + * replaced with lower-pri VC as soon as possible. E.g. when some Ent is loaded + * with omni VC, its ent.vc is assigned to either this Ent's "owner" VC + * (accessible via VC pointing field) or, if not detected, to guest VC. + */ +export const OMNI_ID = "omni"; + +/** + * Useful for debugging, to identify unique VC objects. + */ +let instanceNumber = 0; + +/** + * VC - Viewer Context. + * + * VC is set per HTTP request (or per worker job) in each Ent and represents the + * person who is about to run some database operation. It can represent a user, + * or a guest, or a bot observing that Ent. + * + * Depending on the Ent's Configuration object and privacy rules, it may allow + * the user to load/insert/update/etc. or to traverse to related objects. + */ +export class VC { + // + // WARNING! + // + // DO NOT use ES2020 #-private properties for VC, use only native TS private + // properties. This is because VC is often times cloned by core libraries + // (e.g. Apollo GraphQL context could be a VC, or Express Request user is VC + // etc.). ES2020 #-private properties are simulated via WeakMap during + // transpilation which requires a call to VC's constructor. And while cloning + // via Object.create()/Object.setPrototypeOf(), the constructor is not called. + // + + private annotationCache?: QueryAnnotation; + private caches: VCCaches | undefined = undefined; + private instanceNumber = (instanceNumber++).toString(); // string makes it visible in Chrome memory dump profiler + + /** + * Please please don't call this method except one or two core places. The + * idea is that we create an "origin" VC once and then derive all other VCs + * from it (possibly upgrading or downgrading permissions, controlling + * master/replica read policy etc.). It's also good to trace the entire chain + * of calls and reasons, why some object was accessed. + */ + static createGuestPleaseDoNotUseCreationPointsMustBeLimited({ + trace, + cachesExpirationMs, + }: { + trace?: string; + cachesExpirationMs?: number; + } = {}): VC { + return new VC( + new VCTrace(trace), + GUEST_ID, + null, + new Map(), + new Map(), + { heartbeat: async () => {}, delay }, + true, + cachesExpirationMs ?? 0, + ); + } + + /** + * This is to show VCs in console.log() and inspect() nicely. + */ + [inspect.custom](): string { + return `<${this.toString()}>`; + } + + /** + * Some IDs are cached in VC (e.g. is this ID readable? is it writable? is + * this VC an admin VC?). Also, people may define their own VC-local caches. + */ + cache(Class: { new (vc: VC): TInstance }): TInstance; + + /** + * Same as the above overload, but allows to use a custom creating function. + * This is useful when e.g. cached values are async-created. + */ + cache(tag: symbol, creator: (vc: VC) => TInstance): TInstance; + + // The actual implementation of the above overloads. + cache( + ClassOrTag: { new (vc: VC): TInstance } | symbol, + creator?: (vc: VC) => TInstance, + ): TInstance { + this.caches ??= new VCCaches(this.cachesExpirationMs); + + let cache = this.caches.get(ClassOrTag) as TInstance | undefined; + if (!cache) { + cache = + typeof ClassOrTag === "function" + ? new ClassOrTag(this) + : creator!(this); + this.caches.set(ClassOrTag, cache); + } + + return cache; + } + + /** + * Returns a cached instance of Loader whose actual code is defined in + * HandlerClass. In case there is no such Loader yet, creates it. + */ + loader(HandlerClass: { + new (vc: VC): Handler; + $loader?: symbol; + }): Loader { + let symbol = HandlerClass.$loader; + if (!symbol) { + symbol = HandlerClass.$loader = Symbol(HandlerClass.name); + } + + return this.cache(symbol, () => new Loader(() => new HandlerClass(this))); + } + + /** + * Returns Shard+schemaName timeline which tracks replica staleness for the + * particular schema name (most likely, table). + */ + timeline(shard: Shard, schemaName: string): Timeline { + const key = shard.no + ":" + schemaName; + let timeline = this.timelines.get(key); + if (timeline === undefined) { + timeline = new Timeline(); + this.timelines.set(key, timeline); + } + + return timeline; + } + + /** + * Serializes Shard timelines (master WAL positions) to a string format. The + * method always returns a value which is compatible to + * deserializeTimelines() input. + */ + serializeTimelines(): string | undefined { + const timelines: Record = {}; + for (const [key, timeline] of this.timelines) { + const timelineStr = timeline.serialize(); + if (timelineStr) { + timelines[key] = timelineStr; + } + } + + // Not a single write has been done in this VC; skip serialization. + if (Object.keys(timelines).length === 0) { + return undefined; + } + + return JSON.stringify(timelines); + } + + /** + * Restores all replication timelines in the VC based on the serialized info + * provided. Returns the new VC derived from the current one, but with empty + * caches. + * + * This method has a side effect of changing the timelines of the current VC + * (and actually all parent VCs), because it reflects the changes in the + * global DB state as seen by the current VC's principal. It restores + * previously serialized timelines to the existing VC and all its parent VCs + * which share the same principal. (The latter happens, because + * `this.timelines` map is passed by reference to all derived VCs starting + * from the one which sets principal; see `new VC(...)` clauses all around and + * toLowerInternal() logic.) The timelines are merged according to WAL + * positions (larger WAL positions win). + */ + deserializeTimelines(...dataStrs: ReadonlyArray): VC { + let deserialized = false; + for (const dataStr of dataStrs) { + if (dataStr) { + const data = JSON.parse(dataStr) as Record; + for (const [key, timelineStr] of Object.entries(data)) { + const oldTimeline = this.timelines.get(key) ?? null; + this.timelines.set( + key, + Timeline.deserialize(timelineStr, oldTimeline), + ); + deserialized = true; + } + } + } + + return deserialized ? this.withEmptyCache() : this; + } + + /** + * Saves the timelines to the storage. The function minimizes the number of + * writes to the storage: + * + * - Calling saveTimelines() the 2nd time without making changes to the + * timelines is a no-op. + * - Calling saveTimelines() after loadTimelines() without making changes to + * the timelines is a no-op. + */ + async saveTimelines(storage: TimelineStorage): Promise { + if (!this.isLoggedIn()) { + throw new Error( + `One does not simply save timelines for a non-logged VC: ${this.toString()}`, + ); + } + + const dataStr = this.serializeTimelines(); + if (!dataStr) { + return undefined; + } + + const needsSave = [...this.timelines.values()].some( + (timeline) => !timeline.isSaved(), + ); + if (needsSave) { + // Save to the storage only if there were some changes in timelines. In + // most of the cases (read-only requests) there are no changes, so we save + // one write operation. + await storage.save(this.principal, dataStr); + for (const timeline of this.timelines.values()) { + timeline.setIsSaved(); + } + } + + return dataStr; + } + + /** + * Loads the timelines from the storage and deserializes them into the current + * VC. Returns the new VC derived from the current one, but with empty caches. + */ + async loadTimelines(storage: TimelineStorage): Promise { + if (!this.isLoggedIn()) { + throw new Error( + `One does not simply load timelines for a non-logged VC: ${this.toString()}`, + ); + } + + const dataStrs = await storage.load(this.principal); + const vc = this.deserializeTimelines(...dataStrs); + for (const timeline of vc.timelines.values()) { + timeline.setIsSaved(); + } + + return vc; + } + + /** + * Returns a new VC derived from the current one, but with empty cache. + */ + withEmptyCache(): VC { + return new VC( + this.trace, + this.principal, + this.freshness, + this.timelines, + this.flavors, + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ); + } + + /** + * Returns a new VC derived from the current one, but with master freshness. + * Master freshness is inherited by ent.vc after an Ent is loaded. + */ + @Memoize() + withTransitiveMasterFreshness(): VC { + if (this.freshness === MASTER) { + return this; + } + + return new VC( + this.trace, + this.principal, + MASTER, + this.timelines, + this.flavors, + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ); + } + + /** + * Returns a new VC derived from the current one, but which forces an Ent to + * be loaded always from replica. Freshness is NOT inherited by Ents (not + * transitive): e.g. if an Ent is loaded with STALE_REPLICA freshness, its + * ent.vc will have the DEFAULT freshness. + * + * Also, if an Ent is inserted with a VC of STALE_REPLICA freshness, its VC + * won't remember it, so next immediate reads will go to a replica and not to + * the master. + */ + withOneTimeStaleReplica(): VC { + if (this.freshness === STALE_REPLICA) { + return this; + } + + return new VC( + this.trace, + this.principal, + STALE_REPLICA, + this.timelines, + this.flavors, + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ); + } + + /** + * Creates a new VC with default freshness (i.e. not sticky to master or + * replica, auto-detected on request). Generally, it's not a good idea to use + * this derivation since we lose some bit of internal knowledge from the past + * history of the VC, but for e.g. tests or benchmarks, it's fine. + */ + withDefaultFreshness(): VC { + if (this.freshness === null) { + return this; + } + + return new VC( + this.trace, + this.principal, + null, + this.timelines, + this.flavors, + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ); + } + + /** + * Returns a new VC derived from the current one adding some more flavors to + * it. If no flavors were added, returns the same VC (`this`). + */ + withFlavor(prepend: "prepend", ...flavors: Array): this; + withFlavor(...flavors: Array): this; + withFlavor(...args: unknown[]): VC { + const prepend = args[0] === "prepend" ? args.shift() : undefined; + const pairs = (args as Array) + .filter((flavor): flavor is VCFlavor => flavor !== undefined) + .map((flavor) => [flavor.constructor, flavor] as const); + return pairs.length > 0 + ? new VC( + this.trace, + this.principal, + this.freshness, + this.timelines, + new Map( + prepend === "prepend" + ? [ + ...pairs, + ...[...this.flavors].filter(([cons]) => + pairs.every(([newCons]) => cons !== newCons), + ), + ] + : [ + ...this.flavors, + // Keys in pairs override the previous flavors, do we don't + // need to filter here as above (performance). + ...pairs, + ], + ), + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ) + : this; + } + + /** + * Returns a new VC derived from the current one removing the specified flavors. + * If no flavors were removed, returns the same VC (`this`). + */ + withoutFlavor( + ...flavorClasses: Array VCFlavor> + ): this { + if (!flavorClasses.length) { + return this; + } + + const newFlavors = new Map(this.flavors); + let removed = false; + + for (const flavorClass of flavorClasses) { + if (newFlavors.delete(flavorClass)) { + removed = true; + } + } + + return removed + ? (new VC( + this.trace, + this.principal, + this.freshness, + this.timelines, + newFlavors, + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ) as this) + : this; + } + + /** + * Derives the VC with new trace ID. + */ + withNewTrace(trace: string | undefined): VC { + return new VC( + new VCTrace(trace), + this.principal, + this.freshness, + this.timelines, + this.flavors, + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ); + } + + /** + * Derives the VC with the provided heartbeater injected. + */ + withHeartbeater(heartbeater: VC["heartbeater"]): VC { + return new VC( + this.trace, + this.principal, + this.freshness, + this.timelines, + this.flavors, + heartbeater, + this.isRoot, + this.cachesExpirationMs, + ); + } + + /** + * Creates a new VC upgraded to omni permissions. This VC will not + * be placed to some Ent's ent.vc property; instead, it will be + * automatically downgraded to either the owning VC of this Ent or + * to a guest VC (see Ent.ts). + */ + @Memoize() + toOmniDangerous(): VC { + return new VC( + this.trace, + OMNI_ID, + this.freshness, + this.timelines, + this.flavors, + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ); + } + + /** + * Creates a new VC downgraded to guest permissions. + */ + @Memoize() + public toGuest(): VC { + return new VC( + this.trace, + GUEST_ID, + this.freshness, + this.timelines, + this.flavors, + this.heartbeater, + this.isRoot, + this.cachesExpirationMs, + ); + } + /** + * Checks if it's an omni VC. + */ + isOmni(): boolean { + return this.principal === OMNI_ID; + } + + /** + * Checks if it's a guest VC. + */ + isGuest(): boolean { + return this.principal === GUEST_ID; + } + + /** + * Checks if it's a regular user (i.e. owning) VC. + */ + isLoggedIn(): boolean { + return !this.isOmni() && !this.isGuest(); + } + + /** + * Returns VC's flavor of the particular type. + */ + flavor( + flavor: new (...args: never[]) => TFlavor, + ): TFlavor | null { + return (this.flavors.get(flavor) as TFlavor | undefined) ?? null; + } + + /** + * Used for debugging purposes. + */ + toString(withInstanceNumber = false): string { + const flavorsStr = compact([ + withInstanceNumber && this.instanceNumber, + ...[...this.flavors.values()].map((flavor) => flavor.toDebugString()), + ]).join(","); + return ( + `vc:${this.principal}` + + (flavorsStr ? `(${flavorsStr})` : "") + + (this.freshness === MASTER + ? ":master" + : this.freshness === STALE_REPLICA + ? ":stale_replica" + : "") + ); + } + + /** + * Returns a debug annotation of this VC. + */ + toAnnotation(): QueryAnnotation { + if (!this.annotationCache) { + this.annotationCache = { + // DON'T alter trace here anyhow, or it would break the debugging chain. + trace: this.trace.trace, + debugStack: "", + // vc.toString() returns a textual VC with all flavors mixed in + vc: this.toString(), + whyClient: undefined, + attempt: 0, + }; + } + + if (this.flavor(VCWithStacks)) { + // This is expensive, only enabled explicitly for this flavor. + return { + ...this.annotationCache, + debugStack: minifyStack(Error().stack?.toString() ?? "", 1), + }; + } + + return this.annotationCache; + } + + /** + * Used internally by Ent framework to lower permissions of an injected VC. + * For guest, principal === null. + * - freshness is always reset to default one it VC is demoted + * - isRoot is changed to false once a root VC is switched to a per-user VC + */ + @Memoize() + public toLowerInternal(principal: string | null): VC { + const newPrincipal = principal ? principal.toString() : GUEST_ID; + + if (this.principal === newPrincipal && this.freshness !== STALE_REPLICA) { + // Speed optimization (this happens most of the time): a VC is already + // user-owned, and freshness is default or MASTER. + return this; + } + + const switchesToPrincipalFirstTime = + this.isRoot && newPrincipal !== GUEST_ID && newPrincipal !== OMNI_ID; + const newIsRoot = this.isRoot && !switchesToPrincipalFirstTime; + + // Create an independent timelines map only when we switch to a non-root VC + // the 1st time (e.g. in the beginning of HTTP connection). + const newTimelines = switchesToPrincipalFirstTime + ? Timeline.cloneMap(this.timelines) + : this.timelines; + + // A special case: demote STALE_REPLICA freshness to default (it's not + // transitive and applies only till the next derivation). + const newFreshness = + this.freshness === STALE_REPLICA ? null : this.freshness; + + // Something has changed (most commonly omni->principal or omni->guest). + return new VC( + this.trace, + newPrincipal, + newFreshness, + newTimelines, + this.flavors, + this.heartbeater, + newIsRoot, + this.cachesExpirationMs, + ); + } + + /** + * Private constructor disallows inheritance and manual object creation. + */ + private constructor( + /** Trace information to quickly find all the requests done by this VC in + * debug logs. Trace is inherited once VC is derived. */ + private readonly trace: VCTrace, + /** A principal (typically user ID) represented by this VC. */ + public readonly principal: string, + /** Allows to set VC to always use either a master or a replica DB. E.g. if + * freshness=MASTER, then all the timeline data is ignored, and all the + * requests are sent to master. */ + public readonly freshness: null | typeof MASTER | typeof STALE_REPLICA, + /** Replication WAL position per Shard & Ent. Used to make decisions, + * should a request be sent to a replica or to the master. */ + private timelines: Map, + /** Sticky objects attached to the VC (and inherited when deriving). */ + private flavors: ReadonlyMap, + /** The heartbeat callback is called before each primitive operation. It + * plays the similar role as AbortController: when called, it may throw + * sometimes (signalled externally). Delay callback can also be passed since + * it's pretty common use case to wait for some time and be aborted on a + * heartbeat exception. */ + public readonly heartbeater: { + readonly heartbeat: () => Promise; + readonly delay: (ms: number) => Promise; + }, + /** If true, it's the initial "root" VC which is not yet derived to any + * user's VC. */ + private isRoot: boolean, + /** If nonzero, VC#cache() will return the values which will be auto-removed + * when VC#cache() hasn't been called for more than this time. */ + private cachesExpirationMs: number, + ) {} +} diff --git a/src/ent/VCCaches.ts b/src/ent/VCCaches.ts new file mode 100644 index 0000000..9d86df7 --- /dev/null +++ b/src/ent/VCCaches.ts @@ -0,0 +1,40 @@ +import { WeakTicker } from "../internal/WeakTicker"; + +/** + * Holds an auto-expiring map of VC caches. + */ +export class VCCaches extends Map { + constructor(private expirationMs: number) { + super(); + } + + /** + * Calls the Map's get() and defers cache clearing to the next WeakTicker + * tick (i.e. schedules clearing on inactivity). + */ + override get(key: TKey): TValue | undefined { + if (this.expirationMs > 0) { + weakTicker.schedule(this, this.expirationMs); + } + + return super.get(key); + } + + /** + * Called periodically after VC#cache() was called at least once. + */ + onTick(tickNoSinceScheduling: number): "keep" | "unschedule" { + if (tickNoSinceScheduling === 0) { + // Skip the very 1st tick after the most recent scheduling. Starting from + // the 2nd tick, we can be sure that query cache hasn't been accessed + // within at least expirationMs. + return "keep"; + } + + this.clear(); + return "unschedule"; + } +} + +// Clears VC caches after some time of inactivity. +const weakTicker = new WeakTicker(); diff --git a/src/ent/VCFlavor.ts b/src/ent/VCFlavor.ts new file mode 100644 index 0000000..3833ce0 --- /dev/null +++ b/src/ent/VCFlavor.ts @@ -0,0 +1,34 @@ +/** + * VCFlavor is some piece of info which is transitively attached to a VC and is + * preserved when VC derivation (upgrade/downgrade) is happening. This piece of + * info may be just an object of a separate class with no data (acts like a + * boolean flag), or it can be an object with payload. + * + * For each flavor type, only a single VCFlavor object may exist. + */ +export abstract class VCFlavor { + static readonly _tag: "VCFlavorClass"; + declare readonly _tag: "VCFlavorInstance"; + + /** + * Appended to the end of VC.toString() result. + */ + toDebugString(): string { + return ""; + } +} + +/** + * If turned on, the debug logs will contain caller stack traces for each Ent + * query. This is expensive, use in dev mode only! + */ +export class VCWithStacks extends VCFlavor {} + +/** + * If set, Ent cache is enabled for operations in this VC. + */ +export class VCWithQueryCache extends VCFlavor { + constructor(public readonly options: { maxQueries: number }) { + super(); + } +} diff --git a/src/ent/VCTrace.ts b/src/ent/VCTrace.ts new file mode 100644 index 0000000..3dedf06 --- /dev/null +++ b/src/ent/VCTrace.ts @@ -0,0 +1,48 @@ +/** + * The upper bound of Date.now() is Number.MAX_SAFE_INTEGER which is 2^53 - 1, + * so we have only 10 bits from BigInt (2^63 - 1) left to represent the random + * part of the trace value. + */ +const RANDOM_BITS = 10; +const RANDOM_BITS_MASK = Math.pow(2, RANDOM_BITS) - 1; + +/** + * A "trace" objects which allows to group database related stuff while logging + * it. Traces are inherited during VC derivation, similar to flavors, but + * they're a part of VC core interface to allow faster access. + */ +export class VCTrace { + readonly trace: string; + + constructor(trace?: string) { + this.trace = trace ?? this.createRandomTrace(); + } + + /** + * In case the trace was created by this tool, tries to extract the date of + * its creation. As a sanity check, verifies that this date is not too far + * away from the present time. + */ + tryExtractCreationDate(): Date | null { + try { + const ts = BigInt(this.trace) >> BigInt(RANDOM_BITS); + const minTs = Date.now() - 1000 * 3600 * 24 * 365; + const maxTs = Date.now() + 1000 * 3600 * 24; + return BigInt(minTs) < ts && ts < BigInt(maxTs) + ? new Date(Number(ts)) + : null; + } catch { + return null; + } + } + + /** + * Returns a stringified uint63 (0 - 9223372036854775807). + */ + private createRandomTrace(): string { + return ( + (BigInt(Date.now()) << BigInt(RANDOM_BITS)) | + BigInt(Math.trunc(Math.random() * RANDOM_BITS_MASK) & RANDOM_BITS_MASK) + ).toString(); + } +} diff --git a/src/ent/Validation.ts b/src/ent/Validation.ts new file mode 100644 index 0000000..1fe9970 --- /dev/null +++ b/src/ent/Validation.ts @@ -0,0 +1,258 @@ +import flatten from "lodash/flatten"; +import pickBy from "lodash/pickBy"; +import { + ID, + type InsertFieldsRequired, + type InsertInput, + type Row, + type Table, + type UpdateInput, +} from "../types"; +import { EntNotInsertableError } from "./errors/EntNotInsertableError"; +import { EntNotReadableError } from "./errors/EntNotReadableError"; +import { EntNotUpdatableError } from "./errors/EntNotUpdatableError"; +import { EntValidationError } from "./errors/EntValidationError"; +import type { AbstractIs } from "./predicates/AbstractIs"; +import type { AllowIf } from "./rules/AllowIf"; +import type { DenyIf } from "./rules/DenyIf"; +import { evaluate } from "./rules/evaluate"; +import { Require } from "./rules/Require"; +import type { Rule } from "./rules/Rule"; +import { buildUpdateNewRow } from "./Triggers"; +import type { VC } from "./VC"; + +export type LoadRule = AllowIf | DenyIf; + +/** + * For safety, we enforce all Require rules to be in the end of the + * insert/update/delete privacy list, and have at least one of them. In + * TypeScript, it's not possible to create [...L[], R, ...R[]] type + * (double-variadic) when both L[] and R[] are open-ended (i.e. tuples with + * unknown length), so we have to brute-force. + */ +export type WriteRules = + | [] + | [Require, ...Array>] + | [LoadRule, Require, ...Array>] + | [ + LoadRule, + LoadRule, + Require, + ...Array>, + ] + | [ + LoadRule, + LoadRule, + LoadRule, + Require, + ...Array>, + ] + | [ + LoadRule, + LoadRule, + LoadRule, + LoadRule, + Require, + ...Array>, + ]; + +export type ValidationRules = { + readonly tenantPrincipalField?: InsertFieldsRequired & string; + readonly inferPrincipal: (vc: VC, row: Row) => Promise; + readonly load: Validation["load"]; + readonly insert: Validation["insert"]; + readonly update?: Validation["update"]; + readonly delete?: Validation["delete"]; + readonly validate?: Array>>; +}; + +export class Validation { + readonly tenantPrincipalField?: ValidationRules["tenantPrincipalField"]; + readonly inferPrincipal: ValidationRules["inferPrincipal"]; + readonly load: Array>>; + readonly insert: WriteRules>; + readonly update: WriteRules>; + readonly delete: WriteRules>; + readonly validate: Array>>; + + constructor( + private entName: string, + rules: ValidationRules, + ) { + this.tenantPrincipalField = rules.tenantPrincipalField; + this.inferPrincipal = rules.inferPrincipal; + this.load = rules.load; + this.insert = rules.insert; + this.update = rules.update || (this.insert as typeof this.update); + this.delete = rules.delete || this.update; + this.validate = (rules.validate || []).map((pred) => new Require(pred)); + } + + async validateLoad(vc: VC, row: Row): Promise { + await this.validatePrivacyImpl( + "load", + this.load, + vc, + row, + "sequential", + EntNotReadableError, + ); + } + + async validateInsert(vc: VC, input: InsertInput): Promise { + await this.validateUserInputImpl(vc, input, input); + await this.validatePrivacyImpl( + "insert", + this.insert, + vc, + input, + "parallel", + EntNotInsertableError, + ); + } + + async validateUpdate( + vc: VC, + old: Row, + input: UpdateInput, + privacyOnly = false, + ): Promise { + // Simulate the update, as if it's applied to the ent. + const newRow = buildUpdateNewRow(old, input); + + if (!privacyOnly) { + await this.validateUserInputImpl( + vc, + newRow as InsertInput, + input, + ); + } + + await this.validatePrivacyImpl( + "update", + this.update, + vc, + newRow, + "parallel", + EntNotUpdatableError, + ); + } + + async validateDelete(vc: VC, row: Row): Promise { + await this.validatePrivacyImpl( + "delete", + this.delete, + vc, + row, + "parallel", + EntNotUpdatableError, // same exception as for update + ); + } + + private async validatePrivacyImpl( + op: string, + rules: Array>, + vc: VC, + row: object, + fashion: "parallel" | "sequential", + ExceptionClass: + | typeof EntNotReadableError + | typeof EntNotInsertableError + | typeof EntNotUpdatableError, + ): Promise { + if (vc.isOmni()) { + return; + } + + this.validateTenantUserIDImpl(vc, row, ExceptionClass); + + const { allow, cause } = + rules.length > 0 + ? await evaluate(vc, row, rules, fashion) + : { allow: false, cause: `No "${op}" rules defined` }; + if (allow) { + return; + } + + throw new ExceptionClass( + this.entName, + vc.toString(), + { [ID]: "?", ...row }, + cause, + ); + } + + private validateTenantUserIDImpl( + vc: VC, + row: object, + ExceptionClass: + | typeof EntNotReadableError + | typeof EntNotInsertableError + | typeof EntNotUpdatableError, + ): void { + if (this.tenantPrincipalField === undefined) { + return; + } + + const rowTenantUserID = (row as Record)[ + this.tenantPrincipalField + ]; + if (rowTenantUserID === vc.principal) { + return; + } + + throw new ExceptionClass( + this.entName, + vc.toString(), + { [ID]: "?", ...row }, + `${this.tenantPrincipalField} is expected to be ` + + JSON.stringify(vc.principal) + + ", but got " + + JSON.stringify(rowTenantUserID), + ); + } + + private async validateUserInputImpl( + vc: VC, + newRow: InsertInput, + input: object, + ): Promise { + // Validation error details (like field name and message) are propagated + // through results[].cause which is EntValidationError. + const { allow, results } = await evaluate( + vc, + newRow, + this.validate, + "parallel", + ); + if (allow) { + // Quick path (expected to fire most of the time). + return; + } + + // If some predicates failed, we ensure that they relate to the fields which + // we actually touched. This makes sense for e.g. UPDATE: if we don't update + // some field, it doesn't make sense to user-validate it. + const touchedFields = Object.keys(pickBy(input, (v) => v !== undefined)); + const errors = flatten( + results + .filter(({ decision }) => decision === "DENY") + .map(({ cause, rule }) => { + // It's safe to cast to AbstractIs, because it's how we build + // this.validate array in our constructor. + const { name, field, message } = rule.predicate as AbstractIs; + return cause instanceof EntValidationError + ? cause.errors + : cause === null + ? // The Predicate just returned false. + [{ field, message: message ?? `${name} returned false` }] + : // Some other error; we must not expose error message details, + // but can at least hint on the error class name. + [{ field, message: cause.name }]; + }), + ).filter(({ field }) => field === null || touchedFields.includes(field)); + if (errors.length > 0) { + throw new EntValidationError(this.entName, errors); + } + } +} diff --git a/src/ent/__tests__/Ent.composite-pk.test.ts b/src/ent/__tests__/Ent.composite-pk.test.ts new file mode 100644 index 0000000..a7181ff --- /dev/null +++ b/src/ent/__tests__/Ent.composite-pk.test.ts @@ -0,0 +1,110 @@ +import { recreateTestTables, testCluster } from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { ID } from "../../types"; +import { BaseEnt } from "../BaseEnt"; +import { CanReadOutgoingEdge } from "../predicates/CanReadOutgoingEdge"; +import { OutgoingEdgePointsToVC } from "../predicates/OutgoingEdgePointsToVC"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { Require } from "../rules/Require"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import { createVC } from "./test-utils"; + +const entTestUserSchema = new PgSchema( + 'ent.composite-pk"user', + { + userid: { type: ID, autoInsert: "id_gen()" }, + name: { type: String }, + }, + ["userid"], +); + +class EntTestUser extends BaseEnt(testCluster, entTestUserSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + userid bigint NOT NULL PRIMARY KEY, + name text + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.userid, + privacyLoad: [new AllowIf(new OutgoingEdgePointsToVC("userid"))], + privacyInsert: [], + privacyUpdate: [new Require(new OutgoingEdgePointsToVC("userid"))], + }); + } +} + +const entTestCompositeSchema = new PgSchema( + 'ent.composite-pk"composite', + { + user_id: { type: ID }, + some_id: { type: ID, autoInsert: "id_gen()" }, + name: { type: String }, + }, + ["user_id", "some_id"], +); + +class EntTestComposite extends BaseEnt(testCluster, entTestCompositeSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + user_id bigint NOT NULL, + some_id bigint NOT NULL, + name text, + PRIMARY KEY (user_id, some_id) + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: ["some_id"], + privacyInferPrincipal: async (_vc, row) => row.user_id, + privacyLoad: [ + new AllowIf(new CanReadOutgoingEdge("user_id", EntTestUser)), + ], + privacyInsert: [new Require(new True())], + }); + } +} + +let user: EntTestUser; + +beforeEach(async () => { + await recreateTestTables([EntTestUser, EntTestComposite]); + + user = await EntTestUser.insertReturning(createVC().toOmniDangerous(), { + name: "my-name", + }); + expect(user).toMatchObject({ + id: user.vc.principal, + userid: user.vc.principal, + }); +}); + +test("simple use case", async () => { + const composite = await EntTestComposite.insertReturning(user.vc, { + user_id: user.vc.principal, + name: "my-name", + }); + expect(composite).toMatchObject({ + id: `(${user.vc.principal},${composite.some_id})`, + user_id: user.vc.principal, + }); + + const rows = await EntTestComposite.select( + user.vc, + { user_id: composite.user_id, some_id: composite.some_id }, + 1, + ); + expect(rows).toHaveLength(1); + + await EntTestComposite.loadByX(user.vc, { + user_id: composite.user_id, + some_id: composite.some_id, + }); + + await EntTestComposite.loadX(user.vc, composite.id); +}); diff --git a/src/ent/__tests__/Ent.custom-fields.test.ts b/src/ent/__tests__/Ent.custom-fields.test.ts new file mode 100644 index 0000000..2bc54c6 --- /dev/null +++ b/src/ent/__tests__/Ent.custom-fields.test.ts @@ -0,0 +1,47 @@ +import { recreateTestTables, testCluster } from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { ID, JSONType } from "../../types"; +import { BaseEnt } from "../BaseEnt"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { Require } from "../rules/Require"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import { createVC } from "./test-utils"; + +const vc = createVC(); + +const entTestIssueSchema = new PgSchema( + 'ent.custom-fields"issues', + { + id: { type: ID, autoInsert: "id_gen()" }, + labels: { type: JSONType() }, + }, + [], +); + +class EntTestIssue extends BaseEnt(testCluster, entTestIssueSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL, + labels jsonb NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + }); + } +} + +beforeEach(async () => { + await recreateTestTables([EntTestIssue]); +}); + +test("JSONType", async () => { + const issue = await EntTestIssue.insertReturning(vc, { labels: ["a"] }); + expect(issue.labels).toEqual(["a"]); +}); diff --git a/src/ent/__tests__/Ent.errors.test.ts b/src/ent/__tests__/Ent.errors.test.ts new file mode 100644 index 0000000..93e8bef --- /dev/null +++ b/src/ent/__tests__/Ent.errors.test.ts @@ -0,0 +1,80 @@ +import { join, nullthrows } from "../../internal/misc"; +import { recreateTestTables, testCluster } from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { ID } from "../../types"; +import { BaseEnt } from "../BaseEnt"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { Require } from "../rules/Require"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import { createVC } from "./test-utils"; + +const vc = createVC(); + +const entTestNoUniqueIndexSchema = new PgSchema( + 'ent.errors"no_unique_index', + { + id: { type: ID, autoInsert: "id_gen()" }, + name: { type: String }, + }, + ["name"], +); + +class EntTestNoUniqueIndex extends BaseEnt( + testCluster, + entTestNoUniqueIndexSchema, +) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL, + name text NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + }); + } +} + +beforeEach(async () => { + await recreateTestTables([EntTestNoUniqueIndex]); +}); + +test('upsert() error does not duplicate "after 1 attempt" suffix in the stack', async () => { + const error = nullthrows( + await join([ + join([ + EntTestNoUniqueIndex.upsert(vc, { name: "name1" }), + EntTestNoUniqueIndex.upsert(vc, { name: "name2" }), + EntTestNoUniqueIndex.upsert(vc, { name: "name3" }), + ]), + EntTestNoUniqueIndex.upsert(vc, { name: "name4" }), + EntTestNoUniqueIndex.upsert(vc, { name: "name5" }), + join([ + EntTestNoUniqueIndex.upsert(vc, { name: "name6" }), + EntTestNoUniqueIndex.upsert(vc, { name: "name7" }), + EntTestNoUniqueIndex.upsert(vc, { name: "name8" }), + ]), + ]) + .then(() => null) + .catch((e: unknown) => e), + ); + + const stackTraceLines = (error as Error).stack?.split("\n"); + expect( + stackTraceLines?.filter((line) => line.includes("Cause:")).length, + ).toBe(3); + expect( + stackTraceLines?.filter((line) => + line.includes("at async Promise.all (index *)"), + ).length, + ).toBe(3); + expect((error as Error).stack).toMatch( + /.*Ent.errors.test.ts.*\n\s+on test-pool.*\n\s+after 1 attempt$/s, + ); +}); diff --git a/src/ent/__tests__/Ent.generic.test.ts b/src/ent/__tests__/Ent.generic.test.ts new file mode 100644 index 0000000..8ae6d07 --- /dev/null +++ b/src/ent/__tests__/Ent.generic.test.ts @@ -0,0 +1,852 @@ +import { collect } from "streaming-iterables"; +import { join } from "../../internal/misc"; +import { recreateTestTables, testCluster } from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { EnumType, ID } from "../../types"; +import { BaseEnt } from "../BaseEnt"; +import { CanDeleteOutgoingEdge } from "../predicates/CanDeleteOutgoingEdge"; +import { CanReadOutgoingEdge } from "../predicates/CanReadOutgoingEdge"; +import { CanUpdateOutgoingEdge } from "../predicates/CanUpdateOutgoingEdge"; +import { IncomingEdgeFromVCExists } from "../predicates/IncomingEdgeFromVCExists"; +import { OutgoingEdgePointsToVC } from "../predicates/OutgoingEdgePointsToVC"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { Require } from "../rules/Require"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import type { VC } from "../VC"; +import { createVC, expectToMatchSnapshot } from "./test-utils"; + +enum Industry { + IT = "it", + HEALTH = "health", +} + +enum Size { + ONE = 1, + TWO = 2, +} + +/** + * Company + */ +const entTestCompanySchema = new PgSchema( + 'ent.generic"company', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: EnumType<"some" | "other">() }, + industry: { type: EnumType() }, + size: { type: EnumType() }, + }, + ["name", "industry"], +); + +class EntTestCompany extends BaseEnt(testCluster, entTestCompanySchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL, + industry text NOT NULL, + size integer NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [ + new AllowIf(async function VCIsAllSeeing(vc, _company) { + const vcUser = await EntTestUser.loadX(vc, vc.principal); + return vcUser.is_alseeing; + }), + new AllowIf( + new IncomingEdgeFromVCExists(EntTestUser, "id", "company_id"), + ), + ], + privacyInsert: [], + }); + } +} + +/** + * User -> Company + */ +const entTestUserSchema = new PgSchema( + 'ent.generic"user', + { + id: { type: ID, autoInsert: "id_gen()" }, + company_id: { type: ID, allowNull: true, autoInsert: "NULL" }, + name: { type: String }, + url_name: { type: String, allowNull: true }, + is_alseeing: { type: Boolean, autoInsert: "false" }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + }, + ["name"], +); + +class EntTestUser extends BaseEnt(testCluster, entTestUserSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + company_id bigint DEFAULT NULL, + name text NOT NULL, + url_name text, + is_alseeing boolean, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + UNIQUE (name) + )`, + ]; + + some!: number; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.id, + privacyLoad: [ + new AllowIf(new OutgoingEdgePointsToVC("id")), + new AllowIf(new CanReadOutgoingEdge("company_id", EntTestCompany)), + ], + privacyInsert: [], + privacyUpdate: [new Require(new OutgoingEdgePointsToVC("id"))], + }); + } + + nameUpper(): string { + return this.name.toUpperCase(); + } +} + +/** + * Post -> User -> Company + */ +const entTestPostSchema = new PgSchema( + 'ent.generic"post', + { + post_id: { type: ID, autoInsert: "id_gen()" }, + user_id: { type: ID }, + title: { type: String }, + created_at: { type: Date, autoInsert: "now()" }, + }, + ["post_id"], +); + +class EntTestPost extends BaseEnt(testCluster, entTestPostSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + post_id bigint NOT NULL PRIMARY KEY, + user_id bigint NOT NULL, + title text NOT NULL, + created_at timestamptz NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: ["post_id"], + privacyInferPrincipal: async (_vc, row) => row.user_id, + privacyLoad: [ + new AllowIf(new CanReadOutgoingEdge("user_id", EntTestUser)), + ], + privacyInsert: [ + new Require(new CanUpdateOutgoingEdge("user_id", EntTestUser)), + ], + privacyUpdate: [ + new Require(async function VCInSameCompany(vc, post) { + // A post can be updated by anyone in the same company. + const postUser = await EntTestUser.loadX(vc, post.user_id); + const vcUser = await EntTestUser.loadX(vc, vc.principal); + return postUser.company_id === vcUser.company_id; + }), + ], + }); + } + + titleUpper(): string { + return this.title.toUpperCase(); + } + + async user(): Promise { + return EntTestUser.loadX(this.vc, this.user_id); + } +} + +/** + * Comment -> Post -> User -> Company + */ +const entTestCommentSchema = new PgSchema( + 'ent.generic"comment', + { + comment_id: { type: String, autoInsert: "id_gen()" }, + post_id: { type: ID }, + text: { type: String }, + }, + ["comment_id"], +); + +class EntTestComment extends BaseEnt(testCluster, entTestCommentSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + comment_id bigint NOT NULL PRIMARY KEY, + post_id bigint NOT NULL, + text text NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: ["post_id"], + privacyInferPrincipal: async (vc, row) => + EntTestPost.loadX(vc, row.post_id), + privacyLoad: [ + new AllowIf(new CanReadOutgoingEdge("post_id", EntTestPost)), + ], + privacyInsert: [ + new Require(new CanUpdateOutgoingEdge("post_id", EntTestPost)), + ], + privacyDelete: [ + new Require(new CanDeleteOutgoingEdge("post_id", EntTestPost)), + ], + }); + } + + textUpper(): string { + return this.text.toUpperCase(); + } +} + +/** + * Like -> Post -> User -> Company + */ +const entTestLikeSchema = new PgSchema( + 'ent.generic"like', + { + id: { type: ID, autoInsert: "id_gen()" }, + post_id: { type: ID }, + user_id: { type: ID }, + }, + ["post_id", "user_id"], +); + +class EntTestLike extends BaseEnt(testCluster, entTestLikeSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + post_id bigint NOT NULL, + user_id bigint NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: ["post_id"], + privacyInferPrincipal: async (_vc, row) => row.user_id, + privacyLoad: [ + new AllowIf(new CanReadOutgoingEdge("post_id", EntTestPost)), + ], + privacyInsert: [ + new Require(new CanUpdateOutgoingEdge("post_id", EntTestPost)), + ], + }); + } +} + +let vc: VC; +let vcOther: VC; + +beforeEach(async () => { + await recreateTestTables([ + EntTestCompany, + EntTestUser, + EntTestPost, + EntTestComment, + EntTestLike, + ]); + + const company = await EntTestCompany.insertReturning( + createVC().toOmniDangerous(), + { name: "some", industry: Industry.IT, size: Size.ONE }, + ); + expect(company.name).toEqual("some"); + expect(company.industry).toEqual(Industry.IT); + expect(company.size).toEqual(Size.ONE); + + const user = await EntTestUser.insertReturning(company.vc.toOmniDangerous(), { + company_id: company.id, + name: "John", + url_name: "john", + }); + expect(user.vc.principal).toEqual(user.id); + vc = user.vc; + + const otherUser = await EntTestUser.insertReturning( + company.vc.toOmniDangerous(), + { name: Date.now().toString(), url_name: "" }, + ); + vcOther = otherUser.vc; +}); + +test("simple use case", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + expect(user.url_name).toEqual("john"); + + const post = await EntTestPost.insertReturning(vc, { + user_id: user.id, + title: "something", + }); + expect(post.created_at).toBeInstanceOf(Date); + + const loadedUser = await post.user(); + expect(loadedUser.name).toEqual(user.name); +}); + +test("loadX", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + expect(user.nameUpper()).toEqual("JOHN"); +}); + +test("loadX coalescing produces same objects", async () => { + const [user1, user2] = await join([ + EntTestUser.loadX(vc, vc.principal), + EntTestUser.loadX(vc, vc.principal), + ]); + user1.some = 10; + expect(user2.some).toEqual(10); +}); + +test("loadX coalescing produces different objects for different vc", async () => { + const [user1, user2] = await join([ + EntTestUser.loadX(vc, vc.principal), + EntTestUser.loadX(vc.toOmniDangerous(), vc.principal), + ]); + user1.some = 10; + expect(user2.some).toBeUndefined(); +}); + +test("loadNullable with no access", async () => { + try { + await EntTestUser.loadNullable(vcOther, vc.principal); + fail("must throw an exception"); + } catch (e: unknown) { + expectToMatchSnapshot("" + e); + } +}); + +test("load child with no access", async () => { + const post = await EntTestPost.insertReturning(vc, { + user_id: vc.principal, + title: "some_post", + }); + const comment = await EntTestComment.insertReturning(vc, { + post_id: post.id, + text: "some_comment", + }); + try { + await EntTestComment.loadNullable(vcOther, comment.id); + fail("must throw an exception"); + } catch (e: unknown) { + expectToMatchSnapshot("" + e); + } +}); + +test("loadByX", async () => { + const user = await EntTestUser.loadByX(vc, { name: "John" }); + expect(user.url_name).toEqual("john"); + expect(user.nameUpper()).toEqual("JOHN"); + expect(await EntTestUser.loadByNullable(vc, { name: "zzz" })).toBeNull(); +}); + +test("loadBy with no access", async () => { + try { + await EntTestUser.loadByX(vcOther, { name: "John" }); + fail("must throw an exception"); + } catch (e: unknown) { + expectToMatchSnapshot("" + e); + } +}); + +test("selectBy", async () => { + const post = await EntTestPost.insertReturning(vc, { + user_id: vc.principal, + title: "some_post", + }); + const like = await EntTestLike.insertReturning(vc, { + post_id: post.id, + user_id: vc.principal, + }); + const likes = await EntTestLike.selectBy(vc, { + post_id: post.id, + }); + expect(likes.map((ent) => ent.id)).toEqual([like.id]); +}); + +test("select and count", async () => { + const post = await EntTestPost.insertReturning(vc, { + user_id: vc.principal, + title: "post", + }); + await join([ + EntTestComment.insertReturning(vc, { post_id: post.id, text: "c2" }), + EntTestComment.insertReturning(vc, { post_id: post.id, text: "c3" }), + ]); + + const comments = await EntTestComment.select( + vc, + { post_id: post.id, text: ["c1", "c2", "c3"] }, + 2, + [{ text: "DESC" }], + ); + expect(comments.length).toEqual(2); + expect(comments[0].textUpper()).toEqual("C3"); + expect(comments[1].textUpper()).toEqual("C2"); + + const comments2 = await EntTestComment.select( + vc, + { id: [comments[0].id, comments[1].id] }, + 2, + [{ text: "DESC" }], + ); + expect(comments2.length).toEqual(2); + + const error1 = await EntTestComment.select( + vc, + { post_id: [] }, + Number.MAX_SAFE_INTEGER, + ).catch((e: unknown) => `${e}`); + expect(error1).toMatchSnapshot(); + + const error2 = await EntTestComment.select( + vc, + { post_id: "" }, + Number.MAX_SAFE_INTEGER, + ).catch((e: unknown) => `${e}`); + expect(error2).toMatchSnapshot(); + + const count = await EntTestComment.count(vc, { + post_id: post.id, + text: ["c1", "c2", "c3"], + }); + expect(count).toEqual(2); + + const exists1 = await EntTestComment.exists(vc, { + post_id: post.id, + text: ["c1", "c2", "c3"], + }); + expect(exists1).toStrictEqual(true); + + const exists2 = await EntTestComment.exists(vc, { + post_id: post.id, + text: ["cNonExistent"], + }); + expect(exists2).toStrictEqual(false); +}); + +test("selectChunked", async () => { + const post = await EntTestPost.insertReturning(vc, { + user_id: vc.principal, + title: "post", + }); + await join([ + EntTestComment.insertReturning(vc, { post_id: post.id, text: "c2" }), + EntTestComment.insertReturning(vc, { post_id: post.id, text: "c3" }), + EntTestComment.insertReturning(vc, { post_id: post.id, text: "c4" }), + EntTestComment.insertReturning(vc, { post_id: post.id, text: "c5" }), + ]); + + const commentChunks = await collect( + EntTestComment.selectChunked( + vc, + { post_id: post.id, $not: { text: "c4" } }, + 2, + Number.MAX_SAFE_INTEGER, + ), + ); + expect(commentChunks.length).toEqual(2); + expect(commentChunks[0].length).toEqual(2); + expect(commentChunks[1].length).toEqual(1); + + const noChunks = await collect( + EntTestComment.selectChunked( + vc, + { post_id: post.id, text: "nothing" }, + 2, + Number.MAX_SAFE_INTEGER, + ), + ); + expect(noChunks).toEqual([]); +}); + +test("custom shard", async () => { + const post = await EntTestPost.insertReturning(vc, { + user_id: vc.principal, + title: "something", + }); + const idInOtherNonGlobalShard = post.id.replace( + /^(\d0+)(\d)/s, + (_, m1, m2) => m1 + (((parseInt(m2) - 1 + 1) % 2) + 1).toString(), + ); + + const posts = await EntTestPost.select( + vc, + { id: post.id, $shardOfID: idInOtherNonGlobalShard }, + Number.MAX_SAFE_INTEGER, + ); + expect(posts).toHaveLength(0); + + const error = await EntTestPost.select( + vc, + { id: post.id, $shardOfID: "" }, + Number.MAX_SAFE_INTEGER, + ).catch((e: unknown) => `${e}`); + expect(error).toMatchSnapshot(); +}); + +test("upsertReturning overwrites", async () => { + const user = await EntTestUser.upsertReturning(vc.toOmniDangerous(), { + name: "John", + url_name: "new_value", + }); + expect(user).toMatchObject({ name: "John", url_name: "new_value" }); +}); + +test("upsertReturning creates new ent", async () => { + const newUser = await EntTestUser.upsertReturning(vc.toOmniDangerous(), { + name: "Someone", + url_name: "someone", + }); + expect(newUser).toMatchObject({ name: "Someone", url_name: "someone" }); + expect(newUser.vc.principal).toEqual(newUser.id); +}); + +test("updateReturningX", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + const newUser = await user.updateReturningX({ url_name: "new" }); + expect(newUser).toMatchObject({ name: "John", url_name: "new" }); + expect(newUser.nameUpper()).toEqual("JOHN"); +}); + +test("updateChanged ignores $literal when all fields are unchanged", async () => { + let user = await EntTestUser.loadX(vc, vc.principal); + expect( + await user.updateChanged({ + url_name: "john", + $literal: ["url_name=?", "new"], + }), + ).toStrictEqual(null); + user = await EntTestUser.loadX(vc, vc.principal); + expect(user).toMatchObject({ name: "John", url_name: "john" }); + + expect( + await user.updateChanged({ + $literal: ["url_name=?", "new"], + }), + ).toStrictEqual(null); + user = await EntTestUser.loadX(vc, vc.principal); + expect(user).toMatchObject({ name: "John", url_name: "john" }); +}); + +test("updateChanged applies $literal when there are changed fields", async () => { + let user = await EntTestUser.loadX(vc, vc.principal); + expect( + await user.updateChanged({ + name: "Doe", + $literal: ["url_name=?", "new"], + }), + ).toStrictEqual(["name"]); + user = await EntTestUser.loadX(vc, vc.principal); + expect(user).toMatchObject({ name: "Doe", url_name: "new" }); +}); + +test("updateChangedReturningX", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + const newUser1 = await user.updateChangedReturningX({ url_name: "new" }); + expect(newUser1).toMatchObject({ name: "John", url_name: "new" }); + expect(newUser1 === user).toBeFalsy(); + const newUser2 = await newUser1.updateChangedReturningX({ url_name: "new" }); + expect(newUser2 === newUser1).toBeTruthy(); +}); + +test("updateOriginal with CAS", async () => { + // There are way more tests for CAS in pg/__tests__, with all corner cases + // covered; here we just illustrate the basic syntax. + const user = await EntTestUser.loadX(vc, vc.principal); + + expect( + await user.updateOriginal({ + url_name: "skip", + $cas: { updated_at: new Date(42) }, + }), + ).toBeFalsy(); + + expect( + await user.updateOriginal({ + url_name: "new", + $cas: { updated_at: user.updated_at }, + }), + ).toBeTruthy(); + + expect( + await user.updateOriginal({ + url_name: "skip2", + $cas: ["updated_at"], + }), + ).toBeFalsy(); + + let newUser = await EntTestUser.loadX(vc, vc.principal); + expect(newUser).toMatchObject({ url_name: "new" }); + expect( + await newUser.updateOriginal({ + url_name: "newest", + $cas: "skip-if-someone-else-changed-updating-ent-props", + }), + ).toBeTruthy(); + + newUser = await EntTestUser.loadX(vc, vc.principal); + expect( + await newUser.updateOriginal({ + url_name: "newest", + $cas: ["updated_at"], + }), + ).toBeTruthy(); +}); + +test("updateChanged with CAS", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + expect( + await user.updateChanged({ + url_name: "skip-by-cas", + $cas: { updated_at: new Date(42) }, + }), + ).toStrictEqual(false); + expect( + await user.updateChanged({ + url_name: user.url_name, // skipped since no fields are changed + $cas: { updated_at: new Date(42) }, // CAS doesn't matter + }), + ).toStrictEqual(null); + expect( + await user.updateChanged({ + url_name: user.url_name, // skipped since no fields are changed + $cas: ["updated_at"], // CAS doesn't matter + }), + ).toStrictEqual(null); + expect( + await user.updateChanged({ + url_name: "new", // field changed + $cas: ["updated_at"], // CAS succeeded + }), + ).toStrictEqual(["url_name"]); +}); + +test("updateChangedReturningX with CAS", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + const original = { ...user }; + expect( + await user.updateChangedReturningX({ + url_name: "skip-by-cas", + $cas: { updated_at: new Date(42) }, + }), + ).toMatchObject({ url_name: original.url_name }); + expect( + await user.updateChangedReturningX({ + url_name: user.url_name, // skipped since no fields are changed + $cas: { updated_at: new Date(42) }, // CAS doesn't matter + }), + ).toMatchObject({ + url_name: original.url_name, + }); + expect( + await user.updateChangedReturningX({ + url_name: user.url_name, // skipped since no fields are changed + $cas: { updated_at: user.updated_at }, // CAS doesn't matter + }), + ).toMatchObject({ + url_name: original.url_name, + }); + expect( + await user.updateChangedReturningX({ + url_name: "new", // field changed + $cas: { updated_at: user.updated_at }, // CAS succeeded + }), + ).toMatchObject({ + url_name: "new", + }); +}); + +test("delete", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + expect(await user.deleteOriginal()).toBeTruthy(); + expect(await EntTestUser.loadNullable(vc, vc.principal)).toBeNull(); + expect(await user.deleteOriginal()).toBeFalsy(); +}); + +test("can read post of the same company user", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + const post = await EntTestPost.insertReturning(vc, { + user_id: user.id, + title: "something", + }); + + const user2 = await EntTestUser.insertReturning(vc.toOmniDangerous(), { + company_id: user.company_id, + name: "Jane", + url_name: "jane", + }); + const post2 = await EntTestPost.loadX(user2.vc, post.id); + expect(post2.title).toEqual("something"); +}); + +test("can update comment of the same company user", async () => { + const user = await EntTestUser.loadX(vc, vc.principal); + const post = await EntTestPost.insertReturning(vc, { + user_id: user.id, + title: "something", + }); + const comment = await EntTestComment.insertReturning(vc, { + post_id: post.id, + text: "some", + }); + + // Add user to the company. + const user2 = await EntTestUser.insertReturning(vc.toOmniDangerous(), { + company_id: user.company_id, + name: "Jane", + url_name: "jane", + }); + + // Check that this user can update comments of other same-company users. + const commentViaUser2 = await EntTestComment.loadX(user2.vc, comment.id); + await commentViaUser2.updateOriginal({ text: "other" }); + await commentViaUser2.deleteOriginal(); +}); + +test("cannot create posts for different users", async () => { + const userAllseeing = await EntTestUser.insertReturning( + vc.toOmniDangerous(), + { is_alseeing: true, name: "All-seeing", url_name: "all-seeing" }, + ); + try { + await EntTestPost.insertReturning(userAllseeing.vc, { + user_id: vc.principal, + title: "something", + }); + fail("must throw an exception"); + } catch (e: unknown) { + expectToMatchSnapshot("" + e); + } +}); + +test("heisenbug: two different schema field sets make schema hash different", async () => { + const schema1 = new PgSchema( + EntTestUser.SCHEMA.name, + { + id: { type: ID, autoInsert: "id_gen()" }, + company_id: { type: ID, allowNull: true, autoInsert: "NULL" }, + }, + [], + ); + + const schema2 = new PgSchema( + EntTestUser.SCHEMA.name, + { + id: { type: ID, autoInsert: "id_gen()" }, + name: { type: String }, + }, + [], + ); + + class Ent1 extends BaseEnt(testCluster, schema1) { + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.id, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [], + privacyUpdate: [new Require(new True())], + }); + } + } + + class Ent2 extends BaseEnt(testCluster, schema2) { + company_id!: string; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.id, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [], + privacyUpdate: [new Require(new True())], + }); + } + } + + const row1 = await Ent1.loadX(vc, vc.principal); + const row2 = await Ent2.loadX(vc, vc.principal); + expect(row1.company_id).toBeTruthy(); + expect(row2.company_id).toBeUndefined(); // heisenbug was here +}); + +test("attempt to use guest VC to load Ents", async () => { + const fakeNull = null as unknown as string; + await expect( + EntTestPost.loadNullable(vc.toGuest(), vc.toGuest().principal), + ).rejects.toThrowErrorMatchingSnapshot(); + await expect( + EntTestPost.loadX(vc.toGuest(), vc.toGuest().principal), + ).rejects.toThrowErrorMatchingSnapshot(); + await expect( + EntTestPost.select(vc.toGuest(), { post_id: vc.toGuest().principal }, 1), + ).rejects.toThrowErrorMatchingSnapshot(); + await expect( + EntTestPost.select(vc.toGuest(), { post_id: fakeNull }, 1), + ).rejects.toThrowErrorMatchingSnapshot(); + await expect( + EntTestPost.loadNullable(vc.toGuest(), fakeNull), + ).rejects.toThrowErrorMatchingSnapshot(); +}); + +test("write without affecting timeline", async () => { + const preInsertTimeline = vc.serializeTimelines(); + const vcDerived = vc.withOneTimeStaleReplica(); + await EntTestPost.insertReturning(vcDerived, { + user_id: vc.principal, + title: "some_post", + }); + expect(vcDerived.serializeTimelines()).toEqual(preInsertTimeline); + expect(vc.serializeTimelines()).toEqual(preInsertTimeline); +}); + +test("should support inserting simple Ents with custom IDs", async () => { + const ent = await EntTestUser.insertReturning(vc.toOmniDangerous(), { + name: "Test", + url_name: null, + }); + await ent.deleteOriginal(); + const newEnt = await EntTestUser.insertReturning(vc.toOmniDangerous(), ent); + expect(newEnt.id).toEqual(ent.id); +}); + +test("delete orphaned comment", async () => testCommentDeletion(true)); + +test("delete comment", async () => testCommentDeletion(false)); + +async function testCommentDeletion( + makeCommentOrphaned: boolean, +): Promise { + const post = await EntTestPost.insertReturning(vc, { + user_id: vc.principal, + title: "some_post", + }); + const comment = await EntTestComment.insertReturning(vc, { + post_id: post.id, + text: "some_comment", + }); + if (makeCommentOrphaned) { + await post.deleteOriginal(); + } + + await comment.deleteOriginal(); +} diff --git a/src/ent/__tests__/Ent.shards.test.ts b/src/ent/__tests__/Ent.shards.test.ts new file mode 100644 index 0000000..1510b6b --- /dev/null +++ b/src/ent/__tests__/Ent.shards.test.ts @@ -0,0 +1,119 @@ +import { MASTER } from "../../abstract/Shard"; +import { recreateTestTables, testCluster } from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { ID } from "../../types"; +import { BaseEnt } from "../BaseEnt"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { Require } from "../rules/Require"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import { createVC } from "./test-utils"; + +const ID_FROM_EXISTING_SHARD = "500010000000"; +const ID_FROM_UNKNOWN_SHARD = "511110000000"; +const PARENT_ID_FROM_UNKNOWN_SHARD = "511117654321"; +const UNIVERSE_FROM_UNKNOWN_SHARD = "511119999999"; + +const vc = createVC(); + +const entTestUniverseSchema = new PgSchema( + 'ent.shards"universe', + { + id: { type: ID, autoInsert: "id_gen()" }, + }, + [], +); + +class EntTestUniverse extends BaseEnt(testCluster, entTestUniverseSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + }); + } +} + +const entTestHumanSchema = new PgSchema( + 'ent.shards"human', + { + id: { type: ID, autoInsert: "id_gen()" }, + parent_id: { type: ID }, + universe_id: { type: ID, allowNull: true }, + name: { type: String }, + }, + [], +); + +class EntTestHuman extends BaseEnt(testCluster, entTestHumanSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + parent_id bigint NOT NULL, + universe_id bigint, + name text + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: ["parent_id"], + inverses: { + universe_id: { name: "inverses", type: "universe2humans" }, + }, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + }); + } +} + +beforeEach(async () => { + await recreateTestTables([EntTestUniverse, EntTestHuman]); + + testCluster.options.runOnShardErrorRetryCount = 2; + testCluster.options.runOnShardErrorRediscoverClusterDelayMs = 1000; +}); + +test("unknown shard failure, singleShardFromID", async () => { + await expect( + EntTestHuman.loadNullable(vc, ID_FROM_UNKNOWN_SHARD), + ).rejects.toThrowErrorMatchingSnapshot(); + expect(testCluster.options.loggers.runOnShardErrorLogger).toBeCalledTimes(1); // not retried on intent +}); + +test("unknown shard failure, singleShardForInsert", async () => { + await expect( + EntTestHuman.insertIfNotExists(vc, { + parent_id: PARENT_ID_FROM_UNKNOWN_SHARD, + universe_id: null, + name: "test", + }), + ).rejects.toThrowErrorMatchingSnapshot(); + expect(testCluster.options.loggers.runOnShardErrorLogger).toBeCalledTimes(1); // not retried on intent +}); + +test("unknown shard failure, multiShardsFromInput, no inverses", async () => { + await expect( + EntTestHuman.select(vc, { universe_id: UNIVERSE_FROM_UNKNOWN_SHARD }, 1), + ).rejects.toThrowErrorMatchingSnapshot(); +}); + +test("shard relocation error when accessing a table should be retried", async () => { + testCluster.options.runOnShardErrorRediscoverClusterDelayMs = 1; + + const master = await testCluster.shard(ID_FROM_EXISTING_SHARD).client(MASTER); + await master.rows("DROP TABLE %T CASCADE", EntTestHuman.SCHEMA.name); + + await expect( + EntTestHuman.loadNullable(vc, ID_FROM_EXISTING_SHARD), + ).rejects.toThrow(/undefined_table/); + expect(testCluster.options.loggers.runOnShardErrorLogger).toBeCalledTimes(3); +}); diff --git a/src/ent/__tests__/Ent.timeline.test.ts b/src/ent/__tests__/Ent.timeline.test.ts new file mode 100644 index 0000000..1eaca91 --- /dev/null +++ b/src/ent/__tests__/Ent.timeline.test.ts @@ -0,0 +1,104 @@ +import { MASTER, STALE_REPLICA } from "../../abstract/Shard"; +import { + recreateTestTables, + TEST_CONFIG, + TEST_ISLANDS, + testCluster, +} from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { ID } from "../../types"; +import { BaseEnt } from "../BaseEnt"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { Require } from "../rules/Require"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import { createVC } from "./test-utils"; + +jest.useFakeTimers({ advanceTimers: true }); + +const vc = createVC("no-cache"); + +const entTestPersonSchema = new PgSchema( + 'ent.timeline"person', + { + id: { type: ID, autoInsert: "id_gen()" }, + name: { type: String }, + }, + [], +); + +class EntTestPerson extends BaseEnt(testCluster, entTestPersonSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL, + name text NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + }); + } +} + +beforeEach(async () => { + testCluster.options.islands = TEST_ISLANDS; + testCluster.options.shardsDiscoverIntervalMs = 1000000; + await testCluster.rediscover(); + await recreateTestTables([EntTestPerson]); +}); + +test("replication lag tracking is simulated through timestamp", async () => { + testCluster.options.islands = [ + { + no: 0, + nodes: [ + { ...TEST_CONFIG, role: "master" }, + { ...TEST_CONFIG, role: "replica" }, + ], + }, + ]; + await testCluster.rediscover(); + + const master = await testCluster.shardByNo(0).client(MASTER); + const masterQuerySpy = jest.spyOn(master, "query"); + const replica = await testCluster.shardByNo(0).client(STALE_REPLICA); + const replicaQuerySpy = jest.spyOn(replica, "query"); + + const id1 = await EntTestPerson.insert(vc, { name: "a" }); + expect(masterQuerySpy).toBeCalledTimes(1); + expect(masterQuerySpy.mock.calls[0][0].annotations[0].whyClient).toBe( + "master-bc-is-write", + ); + + await EntTestPerson.loadX(vc, id1); + expect(replicaQuerySpy).toBeCalledTimes(0); + expect(masterQuerySpy).toBeCalledTimes(2); + expect(masterQuerySpy.mock.calls[1][0].annotations[0].whyClient).toBe( + "master-bc-replica-not-caught-up", + ); + + await jest.advanceTimersByTimeAsync(11000); + await EntTestPerson.loadX(vc, id1); + expect(replicaQuerySpy).toBeCalledTimes(1); + + await jest.advanceTimersByTimeAsync(30000); + jest.clearAllMocks(); + + const id2 = await EntTestPerson.insert(vc, { name: "b" }); + expect(masterQuerySpy).toBeCalledTimes(1); + expect(masterQuerySpy.mock.calls[0][0].annotations[0].whyClient).toBe( + "master-bc-is-write", + ); + + await EntTestPerson.loadX(vc, id2); + expect(replicaQuerySpy).toBeCalledTimes(0); + expect(masterQuerySpy).toBeCalledTimes(2); + expect(masterQuerySpy.mock.calls[1][0].annotations[0].whyClient).toBe( + "master-bc-replica-not-caught-up", + ); +}); diff --git a/src/ent/__tests__/Inverse.test.ts b/src/ent/__tests__/Inverse.test.ts new file mode 100644 index 0000000..d412745 --- /dev/null +++ b/src/ent/__tests__/Inverse.test.ts @@ -0,0 +1,646 @@ +import delay from "delay"; +import range from "lodash/range"; +import sortBy from "lodash/sortBy"; +import uniq from "lodash/uniq"; +import { collect } from "streaming-iterables"; +import { MASTER } from "../../abstract/Shard"; +import { ShardError } from "../../abstract/ShardError"; +import { inspectCompact, join, mapJoin } from "../../internal/misc"; +import { + recreateTestTables, + testCluster, + TEST_CONFIG, + TCPProxyServer, +} from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { ID } from "../../types"; +import { BaseEnt } from "../BaseEnt"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { Require } from "../rules/Require"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import type { VC } from "../VC"; +import { createVC } from "./test-utils"; + +const TABLE_INVERSE = 'inverse"test_inverse'; + +/** + * An Ent in GLOBAL_SHARD. + */ +const entTestCompanySchema = new PgSchema( + 'inverse"test_company', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + }, + ["name"], +); + +class EntTestCompany extends BaseEnt(testCluster, entTestCompanySchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + }); + } +} + +/** + * An Ent in a random Shard with high-cardinality Inverses. + */ +const entTestUserSchema = new PgSchema( + 'inverse"test_user', + { + id: { type: ID, autoInsert: "id_gen()" }, + company_id: { type: ID, allowNull: true }, + team_id: { type: ID, allowNull: true }, + name: { type: String }, + }, + ["company_id", "team_id"], +); + +class EntTestUser extends BaseEnt(testCluster, entTestUserSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + company_id bigint, + team_id bigint, + name text NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: [], + inverses: { + company_id: { name: TABLE_INVERSE, type: "company2users" }, + team_id: { name: TABLE_INVERSE, type: "team2users" }, + }, + privacyInferPrincipal: async (_vc, row) => row.id, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + }); + } +} + +/** + * An Ent which is either sharded (if owner_id points to EntTestUser) or in + * GLOBAL_SHARD (if owner_id points to EntTestCompany). + */ +const entTestTopicSchema = new PgSchema( + 'inverse"test_topic', + { + id: { type: ID, autoInsert: "id_gen()" }, + owner_id: { type: ID }, // either EntTestUser (sharded) or EntTestCompany (global) + slug: { type: String }, + sleep: { type: Number, allowNull: true, autoInsert: "0" }, + }, + ["owner_id", "slug"], +); + +class EntTestTopic extends BaseEnt(testCluster, entTestTopicSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + owner_id bigint NOT NULL, + slug text NOT NULL, + sleep integer NOT NULL, + UNIQUE (owner_id, slug) + )`, + `CREATE OR REPLACE FUNCTION pg_sleep_trigger() RETURNS trigger LANGUAGE plpgsql SET search_path FROM CURRENT AS + $$ BEGIN PERFORM pg_sleep(NEW.sleep); RETURN NEW; END $$`, + "CREATE TRIGGER pg_sleep_trigger BEFORE INSERT ON %T FOR EACH ROW EXECUTE PROCEDURE pg_sleep_trigger()", + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: ["owner_id"], + inverses: { + owner_id: { name: TABLE_INVERSE, type: "owner2topics" }, + }, + privacyInferPrincipal: async (_vc, row) => row.owner_id, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + beforeInsert: [ + async (_vc, { input }) => { + if (input.slug === "throwInBeforeInsert") { + throw Error("thrown in beforeInsert"); + } + }, + ], + afterMutation: [ + async (_vc, { newOrOldRow }) => { + if (newOrOldRow.slug === "throwInAfterMutation") { + throw Error("thrown in throwInAfterMutation"); + } + }, + ], + }); + } +} + +let vc: VC; +let myCompany: EntTestCompany; +let proxyServer: TCPProxyServer; + +beforeEach(async () => { + testCluster.options.runOnShardErrorRetryCount = 2; + testCluster.options.islands = [{ no: 0, nodes: [TEST_CONFIG] }]; + await testCluster.rediscover(); + + await recreateTestTables( + [EntTestCompany, EntTestUser, EntTestTopic], + TABLE_INVERSE, + ); + + vc = createVC(); + myCompany = await EntTestCompany.insertReturning(vc, { name: "my-company" }); + + proxyServer = new TCPProxyServer({ + host: TEST_CONFIG.host!, + port: TEST_CONFIG.port!, + }); +}); + +test("CRUD", async () => { + const companyID1 = "1000000000000000010"; + const teamID1 = "1000100000000000020"; + + let user = await EntTestUser.insertReturning(vc, { + company_id: companyID1, + team_id: teamID1, + name: "u1", + }); + + expect(await EntTestUser.INVERSES[0].id2s(vc, companyID1)).toEqual([user.id]); + expect(await EntTestUser.INVERSES[1].id2s(vc, teamID1)).toEqual([user.id]); + + const companyID2 = "1000100000000000012"; + const teamID2 = "1000000000000000022"; + + user = await user.updateReturningX({ + company_id: companyID2, + team_id: teamID2, + }); + + expect(await EntTestUser.INVERSES[0].id2s(vc, companyID1)).toEqual([]); + expect(await EntTestUser.INVERSES[1].id2s(vc, teamID1)).toEqual([]); + expect(await EntTestUser.INVERSES[0].id2s(vc, companyID2)).toEqual([user.id]); + expect(await EntTestUser.INVERSES[1].id2s(vc, teamID2)).toEqual([user.id]); + + await user.deleteOriginal(); + + expect(await EntTestUser.INVERSES[0].id2s(vc, companyID2)).toEqual([]); + expect(await EntTestUser.INVERSES[1].id2s(vc, teamID2)).toEqual([]); +}); + +test("nullable fields", async () => { + const user = await EntTestUser.insertReturning(vc, { + company_id: null, + team_id: null, + name: "u2", + }); + + expect(await EntTestUser.INVERSES[0].id2s(vc, null)).toEqual([user.id]); + expect(await EntTestUser.INVERSES[1].id2s(vc, null)).toEqual([user.id]); + + await user.deleteOriginal(); + + expect(await EntTestUser.INVERSES[0].id2s(vc, null)).toEqual([]); + expect(await EntTestUser.INVERSES[1].id2s(vc, null)).toEqual([]); +}); + +test("cross-shard select", async () => { + const companyIDNull = null; + const teamID = "1000100000000000020"; + + const user = await EntTestUser.insertReturning(vc, { + company_id: companyIDNull, + team_id: teamID, + name: "u1", + }); + + expect( + await EntTestUser.select(vc, { company_id: companyIDNull }, 42), + ).toMatchObject([{ id: user.id, name: "u1" }]); + + expect(await EntTestUser.select(vc, { team_id: teamID }, 42)).toMatchObject([ + { id: user.id, name: "u1" }, + ]); + + const companyID2 = "1000100000000000012"; + + const user2 = await EntTestUser.insertReturning(vc, { + company_id: companyID2, + team_id: teamID, + name: "u2", + }); + + const users = sortBy( + await EntTestUser.select(vc, { team_id: teamID }, 42), + (u) => u.name, + ); + expect(users).toMatchObject([ + { id: user.id, name: "u1" }, + { id: user2.id, name: "u2" }, + ]); +}); + +test("cross-shard selectChunked", async () => { + const companyID = "1000100000000000912"; + await mapJoin(range(1, 3), async (shardNo) => + mapJoin(range(5), async () => + createUserInShard({ + vc, + shardNo, + startCompanyID: companyID, + startTeamID: "1000000000000000001", + increment: "teamID", + }), + ), + ); + const chunkLens = await mapJoin( + collect(EntTestUser.selectChunked(vc, { company_id: companyID }, 2, 8)), + async (chunk) => chunk.length, + ); + expect(chunkLens).toEqual([2, 2, 1, 2, 1]); +}); + +test("optionally sharded colocation", async () => { + const user = await EntTestUser.insertReturning(vc, { + company_id: null, + team_id: null, + name: "u1", + }); + const inverse = EntTestTopic.INVERSES[0]; + + const company = await EntTestCompany.insertReturning(vc, { + name: "c1", + }); + + const topic1 = await EntTestTopic.insertReturning(vc, { + owner_id: user.id, + slug: "topic1", + }); + expect(await inverse.id2s(vc, user.id)).toEqual([]); // no inverse created since it's in shardAffinity + expect(await EntTestTopic.select(vc, { owner_id: user.id }, 1)).toMatchObject( + [{ id: topic1.id }], + ); + expect( + await EntTestTopic.loadByX(vc, { owner_id: user.id, slug: "topic1" }), + ).toMatchObject({ id: topic1.id }); + + const topic2 = await EntTestTopic.insertReturning(vc, { + owner_id: company.id, + slug: "topic2", + }); + expect(await inverse.id2s(vc, company.id)).toEqual([topic2.id]); + expect( + await EntTestTopic.select(vc, { owner_id: company.id }, 1), + ).toMatchObject([{ id: topic2.id }]); + expect( + await EntTestTopic.loadByX(vc, { owner_id: company.id, slug: "topic2" }), + ).toMatchObject({ id: topic2.id }); + + expect( + await EntTestTopic.loadByNullable(vc, { + owner_id: "1000000000000000000", + slug: "topic2", + }), + ).toBeNull(); +}); + +test("race condition in insert/loadBy", async () => { + for (let i = 0; i < 5; i++) { + const COUNT = 2; + const company = await EntTestCompany.insertReturning(vc, { + name: "c" + i, + }); + const insertedTopicIDs = uniq( + await mapJoin(range(0, COUNT), async () => { + const key = { + owner_id: company.id, + slug: "topic", + }; + const id = await EntTestTopic.insertIfNotExists(vc, key); + if (id) { + return id; + } + + return (await EntTestTopic.loadByX(vc, key)).id; + }), + ); + expect(insertedTopicIDs).toHaveLength(1); + } +}); + +test("inverses are deleted when ent insert DB operation fails", async () => { + testCluster.options.runOnShardErrorRetryCount = 0; + await mapJoin(testCluster.nonGlobalShards(), async (shard) => { + const master = await shard.client(MASTER); + await master.rows( + "DROP TABLE IF EXISTS %T CASCADE", + EntTestTopic.SCHEMA.name, + ); + }); + await expect( + EntTestTopic.insertIfNotExists(vc, { + owner_id: myCompany.id, + slug: "topic", + }), + ).rejects.toThrow(/undefined_table/); + const inverse = EntTestTopic.INVERSES[0]; + expect(await inverse.id2s(vc, myCompany.id)).toEqual([]); +}); + +test("inverses are deleted when beforeInsert trigger throws any error", async () => { + await expect( + EntTestTopic.insertIfNotExists(vc, { + owner_id: myCompany.id, + slug: "throwInBeforeInsert", + }), + ).rejects.toThrow(Error); + const inverse = EntTestTopic.INVERSES[0]; + expect(await inverse.id2s(vc, myCompany.id)).toEqual([]); +}); + +test("inverses are NOT deleted when throwInAfterMutation trigger throws any error", async () => { + await expect( + EntTestTopic.insertIfNotExists(vc, { + owner_id: myCompany.id, + slug: "throwInAfterMutation", + }), + ).rejects.toThrow(Error); + const inverse = EntTestTopic.INVERSES[0]; + expect(await inverse.id2s(vc, myCompany.id)).toHaveLength(1); +}); + +test("inverses are NOT deleted when connection aborts at ent creation, and there were no retries", async () => { + testCluster.options.runOnShardErrorRetryCount = 0; + testCluster.options.islands = [ + { + no: 0, + nodes: [{ ...TEST_CONFIG, ...(await proxyServer.hostPort()) }], + }, + ]; + await testCluster.rediscover(); + + const promise = EntTestTopic.insertIfNotExists(vc, { + owner_id: myCompany.id, + slug: "pg_sleep", + sleep: 5, + }).catch((e: unknown) => inspectCompact(e)); + + await delay(2000); + await proxyServer.abortConnections(); + expect(await promise).toContain("Connection terminated unexpectedly"); + + // On an accidental disconnect (and when INSERT wasn't retried), we don't + // know, whether the DB applied the insert or not, so we should expect Ent + // Framework to NOT delete the inverse. + testCluster.options.runOnShardErrorRetryCount = 10; + expect( + await EntTestTopic.INVERSES[0].id2s( + vc.withOneTimeStaleReplica(), + myCompany.id, + ), + ).toHaveLength(1); +}); + +test("inverses are NOT deleted when connection aborts at ent creation, and insert returned null due to a successful retry", async () => { + testCluster.options.runOnShardErrorRetryCount = 5; + testCluster.options.islands = [ + { + no: 0, + nodes: [{ ...TEST_CONFIG, ...(await proxyServer.hostPort()) }], + }, + ]; + await testCluster.rediscover(); + + const promise = EntTestTopic.insertIfNotExists(vc, { + owner_id: myCompany.id, + slug: "pg_sleep", + sleep: 5, + }).catch((e: unknown) => inspectCompact(e)); + + await delay(2000); + await proxyServer.abortConnections(); + + // Success after a retry, but it returns null thinking that there was such a + // row already in the DB. From the caller perspective, we can't distinguish + // whether the row existed there originally, or it was silently created during + // the 1st attempt (right before the disconnect) and we didn't know about it. + expect(await promise).toBeNull(); + + // On the 1st attempt of the insert(), there is an accidental disconnect. In + // fact, the INSERT succeeds, so on a retry, it returns null instead of an ID + // (because there is such a row inserted already - unique key violation). In + // this case, Ent Framework must not undo inverses creation! Otherwise, there + // will remain a row in the DB with no inverses. + testCluster.options.runOnShardErrorRetryCount = 10; + expect( + await EntTestTopic.INVERSES[0].id2s( + vc.withOneTimeStaleReplica(), + myCompany.id, + ), + ).toHaveLength(1); +}); + +test("inverses are NOT deleted when ent insertion is requested with an existing ID", async () => { + const user = await EntTestUser.insertReturning(vc, { + company_id: myCompany.id, + team_id: null, + name: "my-user", + }); + + const user2id = await EntTestUser.insertIfNotExists(vc, { + id: user.id, + company_id: myCompany.id, + team_id: null, + name: "my-user2", + }); + expect(user2id).toBeNull(); + + expect(await EntTestUser.INVERSES[0].id2s(vc, myCompany.id)).toHaveLength(1); + await EntTestUser.loadByX(vc, { company_id: myCompany.id, team_id: null }); +}); + +test("loadBy with multiple shard candidates", async () => { + const companyID = "1000000000000000001"; + // Creates an inverse from companyID to Shard 1. + await createUserInShard({ + vc, + shardNo: 1, + startCompanyID: companyID, + startTeamID: "1000000000000000001", + increment: "teamID", + }); + // Creates an inverse from companyID to Shard 2. + const user2 = await createUserInShard({ + vc, + shardNo: 2, + startCompanyID: companyID, + startTeamID: "1000000000000010001", + increment: "teamID", + }); + // Now for companyID, we'll have 2 Shard candidates (1 and 2), and they'll + // both be rechecked. Shard 1 will return empty results, and Shard 2 will + // return the user. Make sure that we really get a first non-empty response. + await EntTestUser.loadByX(vc, { + company_id: companyID, + team_id: user2.team_id, + }); +}); + +test("multiShardsFromInput returns minimal number of Shard candidates", async () => { + const sharedTeamID = "1000000000000000011"; + // Add the 1st inverse to the 1st Shard. + const user1 = await createUserInShard({ + vc, + shardNo: 1, + startCompanyID: "1000000000000000001", + startTeamID: sharedTeamID, + increment: "companyID", + }); + // Add the 2nd inverse to the 2nd Shard from the same sharedTeamID + const user2 = await createUserInShard({ + vc, + shardNo: 2, + startCompanyID: "1000000000000100001", + startTeamID: sharedTeamID, + increment: "companyID", + }); + // Since company_id is mentioned earlier in the list of fields with inverses, + // and we filter by it, ShardLocator should only resolve Shard candidates + // based on company_id and never based on team_id. And we know that only one + // user (= one Shard) is associated with a user1.company_id. + expect(user1.company_id).not.toEqual(user2.company_id); + const shards = await EntTestUser.SHARD_LOCATOR.multiShardsFromInput( + vc, + { company_id: user1.company_id, team_id: sharedTeamID }, + "loadBy", + ); + expect(shards).toHaveLength(1); +}); + +test("exception", async () => { + await expect(EntTestUser.select(vc, { name: "u2" }, 42)).rejects.toThrow( + ShardError, + ); +}); + +test("id2s single", async () => { + const companyID1 = "1000000000000000010"; + const teamID = "1000000000000000090"; + const u1 = await EntTestUser.insertReturning(vc, { + company_id: companyID1, + team_id: teamID, + name: "u1", + }); + const master = await testCluster.globalShard().client(MASTER); + master.resetSnapshot(); + const id2s = await EntTestUser.INVERSES[0].id2s( + vc.withEmptyCache(), + companyID1, + ); + expect(id2s).toEqual([u1.id]); + master.toMatchSnapshot(); +}); + +test("id2s batched", async () => { + const companyID1 = "1000000000000000010"; + const companyID2 = "1000000000000000020"; + const teamID = "1000000000000000090"; + + const [u1, u2, u3, u4, u5] = await join([ + EntTestUser.insertReturning(vc, { + company_id: companyID1, + team_id: null, + name: "u1", + }), + EntTestUser.insertReturning(vc, { + company_id: companyID1, // same companyID1 + team_id: null, + name: "u2", + }), + EntTestUser.insertReturning(vc, { + company_id: companyID2, + team_id: null, + name: "u3", + }), + EntTestUser.insertReturning(vc, { + company_id: null, + team_id: null, + name: "u4", + }), + EntTestUser.insertReturning(vc, { + company_id: null, + team_id: teamID, + name: "u5", + }), + ]); + + const master = await testCluster.globalShard().client(MASTER); + master.resetSnapshot(); + const [company1UserIDs, company2UserIDs, companyNullUserIDs, teamUserIDs] = + await join([ + EntTestUser.INVERSES[0].id2s(vc.withEmptyCache(), companyID1), + EntTestUser.INVERSES[0].id2s(vc.withEmptyCache(), companyID2), + EntTestUser.INVERSES[0].id2s(vc.withEmptyCache(), null), + EntTestUser.INVERSES[1].id2s(vc.withEmptyCache(), teamID), + ]); + expect(company1UserIDs).toEqual([u1.id, u2.id].sort()); + expect(company2UserIDs).toEqual([u3.id].sort()); + expect(companyNullUserIDs).toEqual([u4.id, u5.id].sort()); + expect(teamUserIDs).toEqual([u5.id].sort()); + master.toMatchSnapshot(); +}); + +/** + * Tries to insert EntTestUser multiple times until it succeeds inserting it to + * shardNo. Since Shard number generation is randomly-deterministic by unique + * key fields (which are [company_id, team_id] in this case), we need to vary + * either company_id or team_id when inserting more users: we allow the caller + * to specify, which one to increment. + */ +async function createUserInShard({ + vc, + shardNo, + startCompanyID, + startTeamID, + increment, +}: { + vc: VC; + shardNo: number; + startCompanyID: string; + startTeamID: string; + increment: "companyID" | "teamID"; +}): Promise { + for (let i = 1; i <= 100; i++) { + const user = await EntTestUser.insertReturning(vc, { + company_id: ( + BigInt(startCompanyID) + BigInt(increment === "companyID" ? i : 0) + ).toString(), + team_id: ( + BigInt(startTeamID) + BigInt(increment === "teamID" ? i : 0) + ).toString(), + name: "u" + i, + }); + if (testCluster.shard(user.id).no === shardNo) { + return user; + } else { + await user.deleteOriginal(); + } + } + + throw Error(`Weird: couldn't create an EntTestUser in shard ${shardNo}`); +} diff --git a/src/ent/__tests__/QueryCache.test.ts b/src/ent/__tests__/QueryCache.test.ts new file mode 100644 index 0000000..11eab7c --- /dev/null +++ b/src/ent/__tests__/QueryCache.test.ts @@ -0,0 +1,53 @@ +import { testCluster } from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { BaseEnt } from "../BaseEnt"; +import { True } from "../predicates/True"; +import { QueryCache } from "../QueryCache"; +import { AllowIf } from "../rules/AllowIf"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import { createVC } from "./test-utils"; + +const entTestCompanySchema = new PgSchema( + 'query-cache"company', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + }, + ["name"], +); + +class EntTestCompany extends BaseEnt(testCluster, entTestCompanySchema) { + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [], + }); + } +} + +test("simple deletion", async () => { + const vc = createVC(); + const cache = vc.cache(QueryCache); + + await cache.through(EntTestCompany, "select", "key", async () => "res1"); + + const res1 = await cache.through( + EntTestCompany, + "select", + "key", + async () => "other", + ); + expect(res1).toEqual("res1"); + + cache.delete(EntTestCompany, ["select"]); + + const res2 = await cache.through( + EntTestCompany, + "select", + "key", + async () => "other", + ); + expect(res2).toEqual("other"); +}); diff --git a/src/ent/__tests__/ShardLocator.test.ts b/src/ent/__tests__/ShardLocator.test.ts new file mode 100644 index 0000000..9891123 --- /dev/null +++ b/src/ent/__tests__/ShardLocator.test.ts @@ -0,0 +1,123 @@ +import groupBy from "lodash/groupBy"; +import range from "lodash/range"; +import { mapJoin } from "../../internal/misc"; +import { testCluster } from "../../pg/__tests__/test-utils"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import { ShardLocator } from "../ShardLocator"; + +test("singleShardForInsert with colocation affinity", async () => { + const shardLocator = new ShardLocator({ + cluster: testCluster, + entName: "post", + shardAffinity: ["user_id"], + uniqueKey: ["user_id", "title"], + inverses: [], + }); + const s1 = await shardLocator.singleShardForInsert( + { user_id: "100020000000" }, + "insert", + ); + expect(s1).toEqual(testCluster.shard("100020000000")); +}); + +test("singleShardForInsert with GLOBAL_SHARD affinity", async () => { + const shardLocator = new ShardLocator({ + cluster: testCluster, + entName: "test", + shardAffinity: GLOBAL_SHARD, + uniqueKey: ["user_id", "title"], + inverses: [], + }); + const s1 = await shardLocator.singleShardForInsert( + { user_id: "100020000000" }, + "insert", + ); + expect(s1).toEqual(testCluster.globalShard()); +}); + +test("singleShardForInsert with by-unique-key random shard", async () => { + const shardLocator = new ShardLocator({ + cluster: testCluster, + entName: "test", + shardAffinity: ["company_id"], + uniqueKey: ["user_id", "title"], + inverses: [], + }); + const s1 = await shardLocator.singleShardForInsert( + { company_id: null, user_id: "100020000000", title: new Date(12345) }, + "insert", + ); + for (let i = 0; i < 10; i++) { + const s2 = await shardLocator.singleShardForInsert( + { company_id: null, user_id: "100020000000", title: new Date(12345) }, + "insert", + ); + expect(s2).toEqual(s1); + } +}); + +test("singleShardForInsert with truly random shard", async () => { + const shardLocator = new ShardLocator({ + cluster: testCluster, + entName: "test", + shardAffinity: ["user_id"], + uniqueKey: undefined, + inverses: [], + }); + const shardNos = groupBy( + await mapJoin( + range(0, 1000), + async () => + (await shardLocator.singleShardForInsert({ user_id: null }, "insert")) + .no, + ), + ); + expect(Object.keys(shardNos)).toHaveLength( + (await testCluster.nonGlobalShards()).length, + ); +}); + +test("singleShardForInsert with complex filter by ID", async () => { + const shardLocator = new ShardLocator({ + cluster: testCluster, + entName: "test", + shardAffinity: ["user_id"], + uniqueKey: undefined, + inverses: [], + }); + const shard = await shardLocator.singleShardForInsert( + { user_id: "100020000000", id: { $gt: "100010000000" } }, + "insert", + ); + expect(shard.no).toEqual(testCluster.shard("100020000000").no); +}); + +test("singleShardForInsert with simple filter by ID", async () => { + const shardLocator = new ShardLocator({ + cluster: testCluster, + entName: "test", + shardAffinity: ["user_id"], + uniqueKey: undefined, + inverses: [], + }); + const shard = await shardLocator.singleShardForInsert( + { user_id: "100020000000", id: "100010000000" }, + "insert", + ); + expect(shard.no).toEqual(testCluster.shard("100010000000").no); +}); + +test("singleShardForInsert with simple filter by ID arr", async () => { + const shardLocator = new ShardLocator({ + cluster: testCluster, + entName: "test", + shardAffinity: ["user_id"], + uniqueKey: undefined, + inverses: [], + }); + const shard = await shardLocator.singleShardForInsert( + { user_id: "100020000000", id: ["100010000000", "100030000001"] }, + "insert", + ); + expect(shard.no).toEqual(testCluster.shard("100010000000").no); +}); diff --git a/src/ent/__tests__/Triggers.test.ts b/src/ent/__tests__/Triggers.test.ts new file mode 100644 index 0000000..bf28926 --- /dev/null +++ b/src/ent/__tests__/Triggers.test.ts @@ -0,0 +1,496 @@ +import { inspect } from "util"; +import { recreateTestTables, testCluster } from "../../pg/__tests__/test-utils"; +import { PgSchema } from "../../pg/PgSchema"; +import { ID } from "../../types"; +import { BaseEnt } from "../BaseEnt"; +import { CanReadOutgoingEdge } from "../predicates/CanReadOutgoingEdge"; +import { CanUpdateOutgoingEdge } from "../predicates/CanUpdateOutgoingEdge"; +import { OutgoingEdgePointsToVC } from "../predicates/OutgoingEdgePointsToVC"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { Require } from "../rules/Require"; +import { GLOBAL_SHARD } from "../ShardAffinity"; +import { type VC } from "../VC"; +import { createVC, expectToMatchSnapshot } from "./test-utils"; + +const $EPHEMERAL = Symbol("$EPHEMERAL"); +const $EPHEMERAL2 = Symbol("$EPHEMERAL2"); + +/** + * User + */ +const entTestUserSchema = new PgSchema( + 'ent.triggers"user', + { + id: { type: ID, autoInsert: "id_gen()" }, + name: { type: String }, + url_name: { type: String, allowNull: true }, + is_alseeing: { type: Boolean, autoInsert: "false" }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + }, + ["name"], +); + +class EntTestUser extends BaseEnt(testCluster, entTestUserSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + company_id bigint DEFAULT NULL, + name text NOT NULL, + url_name text, + is_alseeing boolean, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + UNIQUE (name) + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: (_vc, row) => row.id, + privacyLoad: [new AllowIf(new OutgoingEdgePointsToVC("id"))], + privacyInsert: [], + privacyUpdate: [new Require(new OutgoingEdgePointsToVC("id"))], + }); + } + + nameUpper(): string { + return this.name.toUpperCase(); + } +} + +/** + * Headline -> User -> Company + */ +const entTestHeadlineSchema = new PgSchema( + 'ent.triggers"headline', + { + id: { type: ID, autoInsert: "id_gen()" }, + user_id: { type: ID }, + headline: { type: String }, + name: { type: String, allowNull: true, autoInsert: "NULL" }, + [$EPHEMERAL]: { type: String, allowNull: true }, // required, but nullable + [$EPHEMERAL2]: { type: Number, autoInsert: "NULL" }, // optional (can be skipped), but if present, must be non-nullable + }, + [], +); + +class EntTestHeadline extends BaseEnt(testCluster, entTestHeadlineSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + user_id bigint NOT NULL, + headline text NOT NULL, + name text + )`, + ]; + static readonly TRIGGER_CALLS: Array<{ + type: string; + op?: string; + oldRow?: unknown; + newRow?: unknown; + newOrOldRow?: unknown; + input?: unknown; + }> = []; + + static override configure() { + return new this.Configuration({ + shardAffinity: [], + privacyInferPrincipal: async (_vc, row) => row.user_id, + privacyLoad: [ + new AllowIf(new CanReadOutgoingEdge("user_id", EntTestUser)), + ], + privacyInsert: [ + new Require(new CanUpdateOutgoingEdge("user_id", EntTestUser)), + ], + beforeInsert: [ + (_vc, { input }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "beforeInsert1", + input: { ...input }, + }); + input.headline += " added-by-beforeInsert1"; + expectRequired(input[$EPHEMERAL], "yes"); + if (input[$EPHEMERAL2]) { + input[$EPHEMERAL2] += 1000; + } + }, + async (_vc, { input }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "beforeInsert2", + input: { ...input }, + }); + input.headline += " added-by-beforeInsert2"; + }, + ], + beforeUpdate: [ + (_vc, { newRow, oldRow, input }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "beforeUpdate", + oldRow, + newRow, + input: { ...input }, + }); + input.headline = newRow.headline + " added-by-beforeUpdate"; + expectRequired(newRow[$EPHEMERAL], "no"); + expectRequired(input[$EPHEMERAL], "no"); + if (input[$EPHEMERAL2]) { + input[$EPHEMERAL2] += 1000000; + } + }, + [ + (_vc, row) => [row.name], + async (_vc, { newRow, oldRow, input }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "beforeUpdate (if name changed)", + newRow, + oldRow, + input: { ...input }, + }); + }, + ], + ], + beforeDelete: [ + async (_vc, { oldRow }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "beforeDelete", + oldRow, + }); + }, + ], + beforeMutation: [ + (_vc, { op, newOrOldRow, input }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "beforeMutation", + op, + newOrOldRow, + input: { ...input }, + }); + op === "INSERT" && + expectRequired(newOrOldRow[$EPHEMERAL], "yes") && + expectRequired(input[$EPHEMERAL], "yes"); + op === "UPDATE" && + expectRequired(newOrOldRow[$EPHEMERAL], "no") && + expectRequired(input[$EPHEMERAL], "no"); + op === "DELETE" && + expectRequired(newOrOldRow[$EPHEMERAL], "no") && + expectRequired(input[$EPHEMERAL], "no"); + expectRequired(newOrOldRow[$EPHEMERAL], "no"); + expectRequired(input[$EPHEMERAL], "no"); + expectRequired(newOrOldRow[$EPHEMERAL2], "no"); + if (input.user_id) { + input.user_id = input.user_id + ""; // input is not readonly + } + }, + [ + (_vc, row) => [row.name], + async (_vc, { op, newOrOldRow, input }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "beforeMutation (if name changed or INSERT/DELETE)", + op, + newOrOldRow, + input: { ...input }, + }); + }, + ], + ], + afterInsert: [ + async (_vc, { input }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "afterInsert", + input: { ...input }, + }); + expectRequired(input[$EPHEMERAL], "yes"); + expectRequired(input[$EPHEMERAL2], "no"); + }, + ], + afterUpdate: [ + (_vc, { newRow, oldRow }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "afterUpdate", + oldRow, + newRow, + }); + expectRequired(newRow[$EPHEMERAL], "no"); + expectRequired(newRow[$EPHEMERAL2], "no"); + }, + [ + (_vc, row) => [row.name], + async (_vc, { newRow, oldRow }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "afterUpdate (if name changed)", + oldRow, + newRow, + }); + }, + ], + ], + afterDelete: [ + async (_vc, { oldRow }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "afterDelete", + oldRow, + }); + }, + ], + afterMutation: [ + (_vc, { op, newOrOldRow }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "afterMutation", + op, + newOrOldRow, + }); + op === "INSERT" && expectRequired(newOrOldRow[$EPHEMERAL], "yes"); + op === "UPDATE" && expectRequired(newOrOldRow[$EPHEMERAL], "no"); + op === "DELETE" && expectRequired(newOrOldRow[$EPHEMERAL], "no"); + expectRequired(newOrOldRow[$EPHEMERAL], "no"); + expectRequired(newOrOldRow[$EPHEMERAL2], "no"); + }, + [ + (_vc, row) => [row.name], + async (_vc, { op, newOrOldRow }) => { + EntTestHeadline.TRIGGER_CALLS.push({ + type: "afterMutation (if name changed or INSERT/DELETE)", + op, + newOrOldRow, + }); + }, + ], + ], + }); + } +} + +/** + * Country + */ +const entTestCountrySchema = new PgSchema( + 'ent.triggers"country', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String, allowNull: true, autoInsert: "NULL" }, + }, + ["name"], +); + +class EntTestCountry extends BaseEnt(testCluster, entTestCountrySchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text, + UNIQUE(name) + )`, + ]; + + static readonly TRIGGER_CALLS: Array<{ + type: string; + old?: unknown; + new?: unknown; + input?: unknown; + }> = []; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + beforeInsert: [ + async (_vc, { input }) => { + this.TRIGGER_CALLS.push({ + type: "beforeInsert", + input: { ...input }, + }); + }, + ], + beforeUpdate: [ + async (_vc, { newRow, oldRow, input }) => { + this.TRIGGER_CALLS.push({ + type: "beforeUpdate", + old: oldRow, + new: newRow, + input: { ...input }, + }); + }, + ], + beforeDelete: [ + async (_vc, { oldRow }) => { + this.TRIGGER_CALLS.push({ + type: "beforeDelete", + old: oldRow, + }); + }, + ], + afterInsert: [ + async (_vc, { input }) => { + this.TRIGGER_CALLS.push({ + type: "afterInsert", + input: { ...input }, + }); + }, + ], + afterUpdate: [ + async (_vc, { newRow, oldRow }) => { + this.TRIGGER_CALLS.push({ + type: "afterUpdate", + old: oldRow, + new: newRow, + }); + }, + ], + afterDelete: [ + async (_vc, { oldRow }) => { + this.TRIGGER_CALLS.push({ + type: "afterDelete", + old: oldRow, + }); + }, + ], + afterMutation: [ + async (_vc, { newOrOldRow }) => { + this.TRIGGER_CALLS.push({ + type: "afterMutation", + input: newOrOldRow, + }); + }, + ], + }); + } +} + +let vc: VC; + +beforeEach(async () => { + await recreateTestTables([EntTestUser, EntTestHeadline, EntTestCountry]); + + const user = await EntTestUser.insertReturning(createVC().toOmniDangerous(), { + name: "John", + url_name: "john", + }); + expect(user.vc.principal).toEqual(user.id); + vc = user.vc; +}); + +test("triggers", async () => { + EntTestHeadline.TRIGGER_CALLS.splice(0); + await EntTestHeadline.insertReturning(vc, { + user_id: vc.principal, + headline: "xyz", + [$EPHEMERAL]: null, + }); + expectToMatchSnapshot( + inspect(EntTestHeadline.TRIGGER_CALLS), + "0: insert happened", + ); + + EntTestHeadline.TRIGGER_CALLS.splice(0); + const headline = await EntTestHeadline.insertReturning(vc, { + user_id: vc.principal, + headline: "abc", + [$EPHEMERAL]: "eph", + [$EPHEMERAL2]: 42, + }); + expect(headline.headline).toEqual( + "abc added-by-beforeInsert1 added-by-beforeInsert2", + ); + expectToMatchSnapshot( + inspect(EntTestHeadline.TRIGGER_CALLS), + "1: insert happened", + ); + + EntTestHeadline.TRIGGER_CALLS.splice(0); + const headline2 = await headline.updateReturningX({ + headline: "xyz-updated", + name: "new-name", + [$EPHEMERAL2]: 101, + }); + expect(headline2.headline).toEqual("xyz-updated added-by-beforeUpdate"); + expectToMatchSnapshot( + inspect(EntTestHeadline.TRIGGER_CALLS), + "2: update happened", + ); + + EntTestHeadline.TRIGGER_CALLS.splice(0); + const headline3 = await headline.updateReturningX({ + [$EPHEMERAL]: "eph3", + }); + expect(headline3.headline).toEqual( + "abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate", + ); + expectToMatchSnapshot( + inspect(EntTestHeadline.TRIGGER_CALLS), + "3: noop-update happened", + ); + + EntTestHeadline.TRIGGER_CALLS.splice(0); + await headline.updateChanged({ + [$EPHEMERAL]: "eph4", + }); + expectToMatchSnapshot( + inspect(EntTestHeadline.TRIGGER_CALLS), + "4: updateChanged happened", + ); + + EntTestHeadline.TRIGGER_CALLS.splice(0); + await headline.deleteOriginal(); + expect(await EntTestHeadline.loadNullable(vc, headline.id)).toBeNull(); + expectToMatchSnapshot( + inspect(EntTestHeadline.TRIGGER_CALLS), + "5: delete happened", + ); +}); + +test("skip after triggers if operation soft fails", async () => { + await EntTestCountry.insertReturning(vc, { name: "zzz" }); + + EntTestCountry.TRIGGER_CALLS.splice(0); + const abc = await EntTestCountry.insertReturning(vc, { name: "abc" }); + expectToMatchSnapshot( + inspect(EntTestCountry.TRIGGER_CALLS), + "1: insert happened", + ); + + EntTestCountry.TRIGGER_CALLS.splice(0); + await EntTestCountry.insertIfNotExists(vc, { name: "abc" }); + expectToMatchSnapshot( + inspect(EntTestCountry.TRIGGER_CALLS), + "2: insert soft-failed on unique key conflict", + ); + + await abc.deleteOriginal(); + + EntTestCountry.TRIGGER_CALLS.splice(0); + await abc.updateOriginal({ name: "zzz" }); + expectToMatchSnapshot( + inspect(EntTestCountry.TRIGGER_CALLS), + "3: update soft-failed on non-existing row", + ); + + EntTestCountry.TRIGGER_CALLS.splice(0); + await abc.deleteOriginal(); + expectToMatchSnapshot( + inspect(EntTestCountry.TRIGGER_CALLS), + "4: delete soft-failed on non-existing row", + ); +}); + +test("inserting Ents with custom IDs when they have beforeInsert triggers", async () => { + const ent = await EntTestCountry.insertReturning(vc.toOmniDangerous(), { + name: "Test", + }); + await ent.deleteOriginal(); + const newEnt = await EntTestCountry.insertReturning( + vc.toOmniDangerous(), + ent, + ); + expect(newEnt.id).toEqual(ent.id); +}); + +function expectRequired( + _value: T, + _flag: [T] extends [never] ? "no" : undefined extends T ? "no" : "yes", +): true { + return true; +} diff --git a/src/ent/__tests__/VC.test.ts b/src/ent/__tests__/VC.test.ts new file mode 100644 index 0000000..05a91bf --- /dev/null +++ b/src/ent/__tests__/VC.test.ts @@ -0,0 +1,188 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Timeline } from "../../abstract"; +import { TimelineStorage } from "../TimelineStorage"; +import { VC } from "../VC"; +import { VCFlavor } from "../VCFlavor"; +import { createVC } from "./test-utils"; + +class Cache extends Map { + constructor() { + super([]); + } +} + +class VCTest1 extends VCFlavor { + constructor(public value: string) { + super(); + } + + override toDebugString(): string { + return `VCTest1:${this.value}`; + } +} + +class VCTest2 extends VCFlavor { + constructor(public value: string) { + super(); + } + + override toDebugString(): string { + return `VCTest2:${this.value}`; + } +} + +class TestTimelineStorage extends TimelineStorage { + private db = new Map(); + + override async load(principal: string): Promise { + return this.db.get(principal) ?? []; + } + + override async save(principal: string, dataStr: string): Promise { + this.db.set(principal, [dataStr]); + } +} + +test("VC should be able to clone", () => { + const vc1 = createVC(); + vc1.cache(Cache).set("test", 42); + const vc2: VC = Object.assign(Object.create(VC.prototype), vc1); + expect(vc2.cache(Cache).get("test")).toEqual(42); +}); + +test("root flag of the VC is changed", () => { + const vc1root = createVC() as VC; + expect((vc1root as any).isRoot).toBeTruthy(); + + const vc2root = vc1root.toOmniDangerous(); + expect((vc2root as any).isRoot).toBeTruthy(); + expect( + (vc2root as any).timelines === (vc1root as any).timelines, + ).toBeTruthy(); + + const vc3 = vc1root.toLowerInternal("42"); + expect((vc3 as any).isRoot).toBeFalsy(); + expect((vc3 as any).timelines === (vc1root as any).timelines).toBeFalsy(); + + const vc4 = vc2root.toLowerInternal("42"); + expect((vc4 as any).isRoot).toBeFalsy(); + expect((vc4 as any).timelines === (vc2root as any).timelines).toBeFalsy(); + + const vc5 = vc4.toLowerInternal(null); // -> guest + expect((vc5 as any).isRoot).toBeFalsy(); + expect((vc5 as any).timelines === (vc4 as any).timelines).toBeTruthy(); + + const vc6 = vc5.toLowerInternal("11"); // -> other user + expect((vc6 as any).isRoot).toBeFalsy(); + expect((vc6 as any).timelines === (vc5 as any).timelines).toBeTruthy(); +}); + +test("VC flavor prepend and append", () => { + const vc = createVC().withFlavor(new VCTest1("some")); + const vc2 = vc.withFlavor(new VCTest2("t2")); + expect(vc2.toString()).toEqual("vc:guest(VCTest1:some,VCTest2:t2)"); + expect(vc2.withFlavor(new VCTest2("tNew")).toString()).toEqual( + "vc:guest(VCTest1:some,VCTest2:tNew)", + ); + expect(vc2.withFlavor("prepend", new VCTest2("tNew")).toString()).toEqual( + "vc:guest(VCTest2:tNew,VCTest1:some)", + ); +}); + +describe("VC.withoutFlavor", () => { + test("VC withoutFlavor removes single flavor", () => { + const vc = createVC() + .withFlavor(new VCTest1("test1")) + .withFlavor(new VCTest2("test2")); + const vcCopy = vc.withoutFlavor(VCTest1); + + expect(vcCopy.flavor(VCTest1)).toBeNull(); + expect(vcCopy.flavor(VCTest2)).toBeInstanceOf(VCTest2); + }); + + test("VC withoutFlavor removes multiple flavors", () => { + const vc = createVC() + .withFlavor(new VCTest1("test1")) + .withFlavor(new VCTest2("test2")); + const vcCopy = vc.withoutFlavor(VCTest1, VCTest2); + + expect(vcCopy.flavor(VCTest1)).toBeNull(); + expect(vcCopy.flavor(VCTest2)).toBeNull(); + }); + + test("VC withoutFlavor returns same instance when no flavors removed", () => { + const vc = createVC().withFlavor(new VCTest1("test1")); + const vcCopy = vc.withoutFlavor(VCTest2); + + expect(vcCopy).toBe(vc); + }); + + test("VC withoutFlavor returns same instance when no flavor classes provided", () => { + const vc = createVC().withFlavor(new VCTest1("test1")); + const vcCopy = vc.withoutFlavor(); + + expect(vcCopy).toBe(vc); + }); + + test("VC withoutFlavor handles non-existent flavor gracefully", () => { + const vc = createVC(); + const vcCopy = vc.withoutFlavor(VCTest1); + + expect(vcCopy).toBe(vc); + expect(vcCopy.flavor(VCTest1)).toBeNull(); + }); + + test("VC withoutFlavor should not mutate the original VC", () => { + const vc = createVC().withFlavor(new VCTest1("test1")); + const vcCopy = vc.withoutFlavor(VCTest1); + + expect(vcCopy).not.toBe(vc); + expect(vc.flavor(VCTest1)).toBeInstanceOf(VCTest1); + }); +}); + +test("VC saves and loads timelines", async () => { + let vc = createVC() + .toLowerInternal("test") + .deserializeTimelines( + JSON.stringify({ + "42:tbl": new Timeline({ + pos: BigInt(1), + expiresAt: Number.MAX_SAFE_INTEGER, + }).serialize(), + }), + ); + const storage = new TestTimelineStorage({ + merge: (dataStrs) => dataStrs.join(";"), + }); + const spySave = jest.spyOn(storage, "save"); + const spyLoad = jest.spyOn(storage, "load"); + await vc.saveTimelines(storage); + await vc.saveTimelines(storage); + expect(spySave).toHaveBeenCalledTimes(1); + expect(spySave).toHaveBeenCalledWith( + "test", + '{"42:tbl":"1:9007199254740991"}', + ); + spySave.mockReset(); + + vc = createVC().toLowerInternal("test"); + await vc.loadTimelines(storage); + expect(spyLoad).toHaveBeenCalledWith("test"); + await vc.saveTimelines(storage); + expect(spySave).toHaveBeenCalledTimes(0); +}); + +test("non-logged VC cannot save timelines", async () => { + let vc = createVC(); + const storage = new TestTimelineStorage({ + merge: (dataStrs) => dataStrs.join(";"), + }); + await expect(vc.saveTimelines(storage)).rejects.toThrow( + "One does not simply save timelines for a non-logged VC: vc:guest", + ); + vc = vc.toOmniDangerous(); + await expect(vc.loadTimelines(storage)).rejects.toThrow( + "One does not simply load timelines for a non-logged VC: vc:omni", + ); +}); diff --git a/src/ent/__tests__/Validation.privacy.test.ts b/src/ent/__tests__/Validation.privacy.test.ts new file mode 100644 index 0000000..d82d907 --- /dev/null +++ b/src/ent/__tests__/Validation.privacy.test.ts @@ -0,0 +1,544 @@ +import { PgSchema } from "../../pg/PgSchema"; +import type { Row } from "../../types"; +import { ID } from "../../types"; +import { EntNotReadableError } from "../errors/EntNotReadableError"; +import { Or } from "../predicates/Or"; +import { True } from "../predicates/True"; +import { AllowIf } from "../rules/AllowIf"; +import { DenyIf } from "../rules/DenyIf"; +import { Require } from "../rules/Require"; +import { Validation } from "../Validation"; +import { createVC, ValidationTester } from "./test-utils"; + +/* eslint-disable @typescript-eslint/no-unused-vars */ +const companySchema = new PgSchema( + 'ent.validation"privacy"company', + { + id: { type: ID, autoInsert: "gen_id()" }, + tenant_id: { type: ID }, + name: { type: String, autoInsert: "''" }, + }, + [], +); + +const vc = createVC(); + +test("0000: load succeeds when first rule allows", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [ + new AllowIf(async function First(_vc, _row) { + return tester.respond("First", true); + }), + new AllowIf(async function Second(_vc, _row) { + return tester.respond("Second", false); + }), + ], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + vc: vc.toLowerInternal("42"), + }), + ).toBe(true); +}); + +test("0010: load succeeds when any rule allows", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [ + new AllowIf(async function First(_vc, _row) { + return tester.respond("First", false); + }), + new AllowIf(async function Second(_vc, _row) { + return tester.respond("Second", true); + }), + ], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + }), + ).toBe(true); +}); + +test("0020: load fails when first rule throws", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [ + new AllowIf(async function First(_vc, _row) { + return tester.respond("First", Error("wild")); + }), + ], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + }), + ).toBe(false); +}); + +test("0030: insert succeeds when all require allow", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new Require(async function First(_vc, row) { + return tester.respond("First", true, row); + }), + new Require(async function Second(_vc, row) { + return tester.respond("Second", true, row); + }), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + vc: vc.toLowerInternal("42"), + }), + ).toBe(true); +}); + +test("0040: insert fails when any require denies", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new Require(async function First(_vc, _row) { + return tester.respond("First", true); + }), + new Require(async function Second(_vc, _row) { + return tester.respond("Second", false); + }), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0041: update fails when any require denies", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [], + update: [ + new Require(async function First(_vc, _row) { + return tester.respond("First", true); + }), + new Require(async function Second(_vc, _row) { + return tester.respond("Second", false); + }), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateUpdate", + updateInput: {}, + vc: vc.toLowerInternal("42"), + }), + ).toBe(false); +}); + +test("0042: delete fails when any require denies", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [], + delete: [ + new Require(async function First(_vc, _row) { + return tester.respond("First", true); + }), + new Require(async function Second(_vc, _row) { + return tester.respond("Second", false); + }), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateDelete", + vc: vc.toLowerInternal("42"), + }), + ).toBe(false); +}); + +test("0050: insert fails when any require throws", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new Require(async function First(_vc, _row) { + return tester.respond("First", true); + }), + new Require(async function Second(_vc, _row) { + return tester.respond("Second", Error("wild")); + }), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0060: load succeeds when any rule allows even if another rule throws EntNotReadableError", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [ + new AllowIf(async function First(vc, _row) { + return tester.respond( + "First", + // Not a "wild" exception (since derived from EntAccessError). + new EntNotReadableError( + "other_table", + vc.toString(), + { id: "987" }, + "ent access error", + ), + ); + }), + new AllowIf(async function Second(_vc, _row) { + return tester.respond("Second", true); + }), + ], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + }), + ).toBe(true); +}); + +test("0070: load fails when any rule allows but another rule throws any wild exception", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [ + new AllowIf(async function First(_vc, _row) { + // "Wild" means "not derived from EntAccessError" + return tester.respond("First", Error("wild")); + }), + new AllowIf(async function Second(_vc, _row) { + return tester.respond("Second", true); + }), + ], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + }), + ).toBe(false); +}); + +test("0080: validations fail when no rules defined", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + }), + ).toBe(false); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0090: load fails with nice error message if only one rule", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [ + new AllowIf(async function First(_vc, _row) { + return tester.respond("First", Error("wild")); + }), + ], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + }), + ).toBe(false); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [ + new AllowIf(async function First(_vc, _row) { + return tester.respond("First", false); + }), + ], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + }), + ).toBe(false); +}); + +test("0100: insert fails with nice error message if only one rule", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new Require(async function First(_vc, _row) { + return tester.respond("First", Error("wild")); + }), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new Require(async function First(_vc, _row) { + return tester.respond("First", false); + }), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0110: insert succeeds when DenyIf rule evaluates", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new DenyIf(async function First(_vc, _row) { + return tester.respond("First", false); + }), + new Require(new True()), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(true); +}); + +test("0120: load fails when DenyIf rule throws", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [ + new DenyIf(async function First(vc, _row) { + return tester.respond( + "First", + new EntNotReadableError( + "other_table", + vc.toString(), + { id: "987" }, + "ent access error", + ), + ); + }), + ], + insert: [], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + }), + ).toBe(false); +}); + +test("0130: fail when tenant user id mismatches", async () => { + const tester = new ValidationTester(); + const validation = new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + tenantPrincipalField: "tenant_id", + load: [], + insert: [], + }); + expect( + await tester.matchSnapshot({ + validation, + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateLoad", + vc: vc.toLowerInternal("999"), + }), + ).toBe(false); + expect( + await tester.matchSnapshot({ + validation, + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + vc: vc.toLowerInternal("999"), + }), + ).toBe(false); + expect( + await tester.matchSnapshot({ + validation, + row: { id: "123" } as Row, + method: "validateInsert", + vc: vc.toLowerInternal("999"), + }), + ).toBe(false); + expect( + await tester.matchSnapshot({ + validation, + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateUpdate", + updateInput: {}, + vc: vc.toLowerInternal("999"), + }), + ).toBe(false); + expect( + await tester.matchSnapshot({ + validation, + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateDelete", + vc: vc.toLowerInternal("999"), + }), + ).toBe(false); +}); + +test("0140: load succeeds when some of Or predicates succeed", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new Require( + new Or( + async function First(_vc, _row) { + return tester.respond( + "First", + new EntNotReadableError( + "other_table", + vc.toString(), + { id: "987" }, + "ent access error", + ), + ); + }, + async function Second(_vc, _row) { + return tester.respond("Second", true); + }, + ), + ), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(true); +}); + +test("0150: load fails with nice error when all of Or predicates fail", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new Require( + new Or( + async function First(_vc, _row) { + return tester.respond( + "First", + new EntNotReadableError( + "other_table", + vc.toString(), + { id: "987" }, + { myKey: "ent access error" }, + ), + ); + }, + async function Second(_vc, _row) { + return tester.respond("Second", false); + }, + ), + ), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0160: load crashes when some predicates fail with a wild error", async () => { + const tester = new ValidationTester(); + expect( + await tester.matchSnapshot({ + validation: new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [ + new Require( + new Or( + async function First(_vc, _row) { + return tester.respond("First", Error("wild")); + }, + async function Second(_vc, _row) { + return tester.respond("Second", true); + }, + ), + ), + ], + }), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); diff --git a/src/ent/__tests__/Validation.validate.test.ts b/src/ent/__tests__/Validation.validate.test.ts new file mode 100644 index 0000000..43a6dcf --- /dev/null +++ b/src/ent/__tests__/Validation.validate.test.ts @@ -0,0 +1,315 @@ +import { EntNotFoundError, EntValidationError } from ".."; +import { PgSchema } from "../../pg/PgSchema"; +import { ID } from "../../types"; +import type { + ValidatorStandardSchemaResult, + ValidatorZodSafeParseResult, +} from "../predicates/AbstractIs"; +import { FieldIs } from "../predicates/FieldIs"; +import { RowIs } from "../predicates/RowIs"; +import { True } from "../predicates/True"; +import { Require } from "../rules/Require"; +import type { ValidationRules } from "../Validation"; +import { Validation } from "../Validation"; +import { ValidationTester } from "./test-utils"; + +/* eslint-disable @typescript-eslint/no-unused-vars */ +const companySchema = new PgSchema( + 'ent.validation"validate"company', + { + id: { type: ID, autoInsert: "gen_id()" }, + tenant_id: { type: ID }, + name: { type: String, autoInsert: "''" }, + }, + [], +); + +let tester: ValidationTester; + +beforeEach(() => { + tester = new ValidationTester(); +}); + +test("0010: update fails when field validation fails", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new FieldIs( + "tenant_id", + (value, row) => + tester.respond("FieldIs(tenant_id)", false, [value, row]), + "some one", + ), + new FieldIs( + "tenant_id", + (value, row) => + tester.respond("FieldIs(tenant_id)", false, [value, row]), + "some two", + ), + new FieldIs("name", async (value, row) => + tester.respond( + "FieldIs(tenant_id)", + standardSchemaIssuesAsync("name", "boom"), + [value, row], + ), + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateUpdate", + updateInput: { tenant_id: "42", name: "hi" }, + }), + ).toBe(false); +}); + +test("0020: update succeeds when validation fails, but the field is untouched", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new FieldIs( + "name", + (value, row) => tester.respond("FieldIs(name)", false, [value, row]), + "boom", + ), + new FieldIs("name", async (value, row) => + tester.respond( + "FieldIs(name)", + standardSchemaIssues("name", "boom"), + [value, row], + ), + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateUpdate", + updateInput: { tenant_id: "101" }, + }), + ).toBe(true); +}); + +test("0030: insert succeeds when multiple validators succeed", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new RowIs( + (row) => tester.respond("RowIs", true, row), + "always succeeds", + ), + new RowIs((row) => tester.respond("RowIs", zodSuccess(), row)), + new FieldIs( + "name", + (value, row) => tester.respond("FieldIs(name)", true, [value, row]), + "always succeeds", + ), + new FieldIs("name", async (value, row) => + tester.respond("FieldIs(name)", standardSchemaSuccess(), [ + value, + row, + ]), + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(true); +}); + +test("0040: insert fails when field validation fails", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new FieldIs( + "name", + (value, row) => tester.respond("FieldIs(name)", false, [value, row]), + "boom", + ), + new FieldIs("name", async (value, row) => + tester.respond("FieldIs(name)", zodIssues("name", "boom"), [ + value, + row, + ]), + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0050: update fails when RowIs fails", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new RowIs((row) => tester.respond("RowIs", false, row), "boom"), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateUpdate", + updateInput: {}, + }), + ).toBe(false); +}); + +test("0060: update fails when RowIs fails, and RowIs propagates field name", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new RowIs((row) => + tester.respond("RowIs", zodIssues("name", "boom"), row), + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateUpdate", + updateInput: { name: "hi" }, + }), + ).toBe(false); +}); + +test("0070: update succeeds when RowIs fails, but the field it propagates is untouched", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new RowIs((row) => + tester.respond("RowIs", standardSchemaIssues("name", "boom"), row), + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateUpdate", + updateInput: {}, + }), + ).toBe(true); +}); + +test("0080: insert fails when RowIs fails", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new RowIs((row) => tester.respond("RowIs", false, row), "boom"), + new RowIs((row) => + tester.respond("RowIs", zodIssues("name", "boom"), row), + ), + new RowIs((row) => + tester.respond( + "RowIs", + zodIssues(null, "Expected object, received number"), + row, + ), + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0090: insert throws when FieldIs throws a weird error", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new FieldIs( + "name", + (value, row) => + tester.respond("FieldIs", Error("ouch"), [value, row]), + "boom", + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0095: insert fails on EntNotFoundError in FieldIs", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new FieldIs( + "name", + (value, row) => + tester.respond( + "FieldIs", + new EntNotFoundError("MyEnt", { id: "42" }), + [value, row], + ), + "boom", + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0096: insert fails on manual EntValidationError in RowIs", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new RowIs( + (row) => + tester.respond( + "RowIs", + new EntValidationError("MyEnt", [ + { field: "name", message: "huh" }, + ]), + row, + ), + "boom", + ), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +test("0100: insert throws when RowIs throws a weird error", async () => { + expect( + await tester.matchSnapshot({ + validation: createValidation([ + new RowIs((row) => tester.respond("RowIs", Error("ouch"), row), "boom"), + ]), + row: { id: "123", tenant_id: "42", name: "hi" }, + method: "validateInsert", + }), + ).toBe(false); +}); + +function createValidation( + validate: ValidationRules["validate"], +): Validation { + return new Validation("table", { + inferPrincipal: async (vc) => vc.toGuest(), + load: [], + insert: [new Require(new True())], + validate, + }); +} + +function zodSuccess(): ValidatorZodSafeParseResult { + return { success: true }; +} + +function zodIssues( + field: string | null, + message: string, +): ValidatorZodSafeParseResult { + return { + success: false, + error: { issues: [{ message, path: field ? [field] : [] }] }, + }; +} + +function standardSchemaSuccess(): ValidatorStandardSchemaResult { + return { value: 42 }; +} + +function standardSchemaIssues( + field: string, + message: string, +): ValidatorStandardSchemaResult { + return { issues: [{ message, path: [field] }] }; +} + +async function standardSchemaIssuesAsync( + field: string, + message: string, +): Promise { + return standardSchemaIssues(field, message); +} diff --git a/src/ent/__tests__/__snapshots__/Ent.generic.test.ts.snap b/src/ent/__tests__/__snapshots__/Ent.generic.test.ts.snap new file mode 100644 index 0000000..c137c9d --- /dev/null +++ b/src/ent/__tests__/__snapshots__/Ent.generic.test.ts.snap @@ -0,0 +1,70 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`attempt to use guest VC to load Ents 1`] = ` +"EntTestPost not found: id=guest, because: + ShardError: EntTestPost: cannot detect shard in "loadNullable" query: most likely you're trying to use a guest VC's principal instead of an ID" +`; + +exports[`attempt to use guest VC to load Ents 2`] = ` +"EntTestPost not found: id=guest, because: + ShardError: EntTestPost: cannot detect shard in "loadNullable" query: most likely you're trying to use a guest VC's principal instead of an ID" +`; + +exports[`attempt to use guest VC to load Ents 3`] = ` +"EntTestPost not found: post_id=guest, because: + ShardError: EntTestPost: cannot detect shard in "select" query: most likely you're trying to use a guest VC's principal instead of an ID" +`; + +exports[`attempt to use guest VC to load Ents 4`] = `"EntTestPost: cannot detect shard in "select" query: non-empty "post_id" field must be present at TOP LEVEL of the input, but got { post_id: null }"`; + +exports[`attempt to use guest VC to load Ents 5`] = ` +"EntTestPost not found: id=null, because: + ShardError: EntTestPost: cannot detect shard in "loadNullable" query: you should not pass null or undefined value in "id" field" +`; + +exports[`cannot create posts for different users 1`] = ` +"EntNotInsertableError: EntTestPost: cannot insert in vc:, because: + Rule Require:CanUpdateOutgoingEdge(user_id) returned DENY, because: + EntTestUser: is not updatable/deletable in vc:, because: + Rule Require:OutgoingEdgePointsToVC(id) returned DENY" +`; + +exports[`custom shard 1`] = ` +"EntNotFoundError: EntTestPost not found: $shardOfID="", because: + ShardError: Cannot parse ID "" to detect shard number" +`; + +exports[`load child with no access 1`] = ` +"EntNotReadableError: EntTestComment: is not readable in vc:, because: + Rule AllowIf:CanReadOutgoingEdge(post_id) returned SKIP, because: + EntTestPost: is not readable in vc:, because: + Rule AllowIf:CanReadOutgoingEdge(user_id) returned SKIP, because: + EntTestUser: is not readable in vc:, because: + Rule AllowIf:OutgoingEdgePointsToVC(id) returned SKIP + Rule AllowIf:CanReadOutgoingEdge(company_id) returned SKIP, because: + EntTestCompany: is not readable in vc:, because: + Rule AllowIf:VCIsAllSeeing returned SKIP + Rule AllowIf:IncomingEdgeFromVCExists(EntTestUser[id=vc, company_id=row.id]) returned SKIP" +`; + +exports[`loadBy with no access 1`] = ` +"EntNotReadableError: EntTestUser: is not readable in vc:, because: + Rule AllowIf:OutgoingEdgePointsToVC(id) returned SKIP + Rule AllowIf:CanReadOutgoingEdge(company_id) returned SKIP, because: + EntTestCompany: is not readable in vc:, because: + Rule AllowIf:VCIsAllSeeing returned SKIP + Rule AllowIf:IncomingEdgeFromVCExists(EntTestUser[id=vc, company_id=row.id]) returned SKIP" +`; + +exports[`loadNullable with no access 1`] = ` +"EntNotReadableError: EntTestUser: is not readable in vc:, because: + Rule AllowIf:OutgoingEdgePointsToVC(id) returned SKIP + Rule AllowIf:CanReadOutgoingEdge(company_id) returned SKIP, because: + EntTestCompany: is not readable in vc:, because: + Rule AllowIf:VCIsAllSeeing returned SKIP + Rule AllowIf:IncomingEdgeFromVCExists(EntTestUser[id=vc, company_id=row.id]) returned SKIP" +`; + +exports[`select and count 1`] = `"ShardError: EntTestComment: cannot detect shard in "select" query: non-empty "post_id" field must be present at TOP LEVEL of the input, but got { post_id: [] }"`; + +exports[`select and count 2`] = `"ShardError: EntTestComment: cannot detect shard in "select" query: non-empty "post_id" field must be present at TOP LEVEL of the input, but got { post_id: '' }"`; diff --git a/src/ent/__tests__/__snapshots__/Ent.shards.test.ts.snap b/src/ent/__tests__/__snapshots__/Ent.shards.test.ts.snap new file mode 100644 index 0000000..4904949 --- /dev/null +++ b/src/ent/__tests__/__snapshots__/Ent.shards.test.ts.snap @@ -0,0 +1,16 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`unknown shard failure, multiShardsFromInput, no inverses 1`] = ` +"EntTestHuman not found: universe_id=511119999999, because: + ShardIsNotDiscoverableError: Shard 1111 is not discoverable: no such Shard in the Cluster? some Islands are down? connections limit?" +`; + +exports[`unknown shard failure, singleShardForInsert 1`] = ` +"EntTestHuman not found: parent_id=511117654321, because: + ShardIsNotDiscoverableError: Shard 1111 is not discoverable: no such Shard in the Cluster? some Islands are down? connections limit?" +`; + +exports[`unknown shard failure, singleShardFromID 1`] = ` +"EntTestHuman not found: id=511110000000, because: + ShardIsNotDiscoverableError: Shard 1111 is not discoverable: no such Shard in the Cluster? some Islands are down? connections limit?" +`; diff --git a/src/ent/__tests__/__snapshots__/Inverse.test.ts.snap b/src/ent/__tests__/__snapshots__/Inverse.test.ts.snap new file mode 100644 index 0000000..c6c04f2 --- /dev/null +++ b/src/ent/__tests__/__snapshots__/Inverse.test.ts.snap @@ -0,0 +1,30 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`id2s batched 1`] = ` +" +SELECT id, created_at, type, id1, id2 FROM "inverse""test_inverse" + WHERE (type, id1) IN(VALUES + ((NULL::"inverse""test_inverse").type, (NULL::"inverse""test_inverse").id1), + ('company2users', ''), + ('company2users', ''), + ('company2users', '0'), + ('team2users', '')) +---- +SELECT id, created_at, type, id1, id2 FROM "inverse""test_inverse" + WHERE (type, id1) IN(VALUES + ((NULL::"inverse""test_inverse").type, (NULL::"inverse""test_inverse").id1), + ('?', '?'), ...) +" +`; + +exports[`id2s single 1`] = ` +" +SELECT id, created_at, type, id1, id2 FROM "inverse""test_inverse" + WHERE (type='company2users' + AND id1='') +---- +SELECT id, created_at, type, id1, id2 FROM "inverse""test_inverse" + WHERE (type='?' + AND id1='?') +" +`; diff --git a/src/ent/__tests__/__snapshots__/Triggers.test.ts.snap b/src/ent/__tests__/__snapshots__/Triggers.test.ts.snap new file mode 100644 index 0000000..e8a1a06 --- /dev/null +++ b/src/ent/__tests__/__snapshots__/Triggers.test.ts.snap @@ -0,0 +1,560 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`skip after triggers if operation soft fails: 1: insert happened 1`] = ` +"[ + { + type: 'beforeInsert', + input: { name: 'abc', id: '' } + }, + { + type: 'afterInsert', + input: { name: 'abc', id: '' } + }, + { + type: 'afterMutation', + input: { name: 'abc', id: '' } + } +]" +`; + +exports[`skip after triggers if operation soft fails: 2: insert soft-failed on unique key conflict 1`] = ` +"[ + { + type: 'beforeInsert', + input: { name: 'abc', id: '' } + } +]" +`; + +exports[`skip after triggers if operation soft fails: 3: update soft-failed on non-existing row 1`] = ` +"[ + { + type: 'beforeUpdate', + old: EntTestCountry { id: '', name: 'abc' }, + new: { id: '', name: 'zzz' }, + input: { name: 'zzz' } + } +]" +`; + +exports[`skip after triggers if operation soft fails: 4: delete soft-failed on non-existing row 1`] = ` +"[ + { + type: 'beforeDelete', + old: EntTestCountry { id: '', name: 'abc' } + } +]" +`; + +exports[`triggers: 0: insert happened 1`] = ` +"[ + { + type: 'beforeInsert1', + input: { + user_id: '', + headline: 'xyz', + id: '', + [Symbol($EPHEMERAL)]: null + } + }, + { + type: 'beforeInsert2', + input: { + user_id: '', + headline: 'xyz added-by-beforeInsert1', + id: '', + [Symbol($EPHEMERAL)]: null + } + }, + { + type: 'beforeMutation', + op: 'INSERT', + newOrOldRow: { + user_id: '', + headline: 'xyz added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: null + }, + input: { + user_id: '', + headline: 'xyz added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: null + } + }, + { + type: 'beforeMutation (if name changed or INSERT/DELETE)', + op: 'INSERT', + newOrOldRow: { + user_id: '', + headline: 'xyz added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: null + }, + input: { + user_id: '', + headline: 'xyz added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: null + } + }, + { + type: 'afterInsert', + input: { + user_id: '', + headline: 'xyz added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: null + } + }, + { + type: 'afterMutation', + op: 'INSERT', + newOrOldRow: { + user_id: '', + headline: 'xyz added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: null + } + }, + { + type: 'afterMutation (if name changed or INSERT/DELETE)', + op: 'INSERT', + newOrOldRow: { + user_id: '', + headline: 'xyz added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: null + } + } +]" +`; + +exports[`triggers: 1: insert happened 1`] = ` +"[ + { + type: 'beforeInsert1', + input: { + user_id: '', + headline: 'abc', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 42 + } + }, + { + type: 'beforeInsert2', + input: { + user_id: '', + headline: 'abc added-by-beforeInsert1', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 1042 + } + }, + { + type: 'beforeMutation', + op: 'INSERT', + newOrOldRow: { + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 1042 + }, + input: { + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 1042 + } + }, + { + type: 'beforeMutation (if name changed or INSERT/DELETE)', + op: 'INSERT', + newOrOldRow: { + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 1042 + }, + input: { + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 1042 + } + }, + { + type: 'afterInsert', + input: { + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 1042 + } + }, + { + type: 'afterMutation', + op: 'INSERT', + newOrOldRow: { + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 1042 + } + }, + { + type: 'afterMutation (if name changed or INSERT/DELETE)', + op: 'INSERT', + newOrOldRow: { + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + id: '', + [Symbol($EPHEMERAL)]: 'eph', + [Symbol($EPHEMERAL2)]: 1042 + } + } +]" +`; + +exports[`triggers: 2: update happened 1`] = ` +"[ + { + type: 'beforeUpdate', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + newRow: { + id: '', + user_id: '', + headline: 'xyz-updated', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 101 + }, + input: { + headline: 'xyz-updated', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 101 + } + }, + { + type: 'beforeUpdate (if name changed)', + newRow: { + id: '', + user_id: '', + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + }, + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + input: { + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + } + }, + { + type: 'beforeMutation', + op: 'UPDATE', + newOrOldRow: { + id: '', + user_id: '', + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + }, + input: { + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + } + }, + { + type: 'beforeMutation (if name changed or INSERT/DELETE)', + op: 'UPDATE', + newOrOldRow: { + id: '', + user_id: '', + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + }, + input: { + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + } + }, + { + type: 'afterUpdate', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + newRow: { + id: '', + user_id: '', + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + } + }, + { + type: 'afterUpdate (if name changed)', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + newRow: { + id: '', + user_id: '', + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + } + }, + { + type: 'afterMutation', + op: 'UPDATE', + newOrOldRow: { + id: '', + user_id: '', + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + } + }, + { + type: 'afterMutation (if name changed or INSERT/DELETE)', + op: 'UPDATE', + newOrOldRow: { + id: '', + user_id: '', + headline: 'xyz-updated added-by-beforeUpdate', + name: 'new-name', + [Symbol($EPHEMERAL2)]: 1000101 + } + } +]" +`; + +exports[`triggers: 3: noop-update happened 1`] = ` +"[ + { + type: 'beforeUpdate', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + newRow: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null, + [Symbol($EPHEMERAL)]: 'eph3' + }, + input: { [Symbol($EPHEMERAL)]: 'eph3' } + }, + { + type: 'beforeMutation', + op: 'UPDATE', + newOrOldRow: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate', + name: null, + [Symbol($EPHEMERAL)]: 'eph3' + }, + input: { + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate', + [Symbol($EPHEMERAL)]: 'eph3' + } + }, + { + type: 'afterUpdate', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + newRow: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate', + name: null, + [Symbol($EPHEMERAL)]: 'eph3' + } + }, + { + type: 'afterMutation', + op: 'UPDATE', + newOrOldRow: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate', + name: null, + [Symbol($EPHEMERAL)]: 'eph3' + } + } +]" +`; + +exports[`triggers: 4: updateChanged happened 1`] = ` +"[ + { + type: 'beforeUpdate', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + newRow: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null, + [Symbol($EPHEMERAL)]: 'eph4' + }, + input: { [Symbol($EPHEMERAL)]: 'eph4' } + }, + { + type: 'beforeMutation', + op: 'UPDATE', + newOrOldRow: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate', + name: null, + [Symbol($EPHEMERAL)]: 'eph4' + }, + input: { + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate', + [Symbol($EPHEMERAL)]: 'eph4' + } + }, + { + type: 'afterUpdate', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + newRow: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate', + name: null, + [Symbol($EPHEMERAL)]: 'eph4' + } + }, + { + type: 'afterMutation', + op: 'UPDATE', + newOrOldRow: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2 added-by-beforeUpdate', + name: null, + [Symbol($EPHEMERAL)]: 'eph4' + } + } +]" +`; + +exports[`triggers: 5: delete happened 1`] = ` +"[ + { + type: 'beforeDelete', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + } + }, + { + type: 'beforeMutation', + op: 'DELETE', + newOrOldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + input: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + } + }, + { + type: 'beforeMutation (if name changed or INSERT/DELETE)', + op: 'DELETE', + newOrOldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + }, + input: { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + } + }, + { + type: 'afterDelete', + oldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + } + }, + { + type: 'afterMutation', + op: 'DELETE', + newOrOldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + } + }, + { + type: 'afterMutation (if name changed or INSERT/DELETE)', + op: 'DELETE', + newOrOldRow: EntTestHeadline { + id: '', + user_id: '', + headline: 'abc added-by-beforeInsert1 added-by-beforeInsert2', + name: null + } + } +]" +`; diff --git a/src/ent/__tests__/__snapshots__/Validation.privacy.test.ts.snap b/src/ent/__tests__/__snapshots__/Validation.privacy.test.ts.snap new file mode 100644 index 0000000..39acdb8 --- /dev/null +++ b/src/ent/__tests__/__snapshots__/Validation.privacy.test.ts.snap @@ -0,0 +1,235 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`0000: load succeeds when first rule allows 1`] = ` +"OK +--- (what actually happened) --- +[First returned] true" +`; + +exports[`0010: load succeeds when any rule allows 1`] = ` +"OK +--- (what actually happened) --- +[First returned] false +[Second returned] true" +`; + +exports[`0020: load fails when first rule throws 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +Error: wild +--- (what actually happened) --- +[First threw] Error: wild" +`; + +exports[`0030: insert succeeds when all require allow 1`] = ` +"OK +--- (what actually happened) --- +[First { id: '123', tenant_id: '42', name: 'hi' } returned] true +[Second { id: '123', tenant_id: '42', name: 'hi' } returned] true" +`; + +exports[`0040: insert fails when any require denies 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotInsertableError: table: cannot insert in vc:guest:master, because: + Rule Require:First returned TOLERATE + Rule Require:Second returned DENY +--- (what actually happened) --- +[First returned] true +[Second returned] false" +`; + +exports[`0041: update fails when any require denies 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotUpdatableError: table:123 is not updatable/deletable in vc:42, because: + Rule Require:First returned TOLERATE + Rule Require:Second returned DENY +--- (what actually happened) --- +[First returned] true +[Second returned] false" +`; + +exports[`0042: delete fails when any require denies 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotUpdatableError: table:123 is not updatable/deletable in vc:42, because: + Rule Require:First returned TOLERATE + Rule Require:Second returned DENY +--- (what actually happened) --- +[First returned] true +[Second returned] false" +`; + +exports[`0050: insert fails when any require throws 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +Error: wild +--- (what actually happened) --- +[First returned] true +[Second threw] Error: wild" +`; + +exports[`0060: load succeeds when any rule allows even if another rule throws EntNotReadableError 1`] = ` +"OK +--- (what actually happened) --- +[First threw] EntNotReadableError: other_table:987 is not readable in vc:guest:master, because: + ent access error +[Second returned] true" +`; + +exports[`0070: load fails when any rule allows but another rule throws any wild exception 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +Error: wild +--- (what actually happened) --- +[First threw] Error: wild" +`; + +exports[`0080: validations fail when no rules defined 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotReadableError: table:123 is not readable in vc:guest:master, because: + No "load" rules defined +--- (what actually happened) --- +" +`; + +exports[`0080: validations fail when no rules defined 2`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotInsertableError: table: cannot insert in vc:guest:master, because: + No "insert" rules defined +--- (what actually happened) --- +" +`; + +exports[`0090: load fails with nice error message if only one rule 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +Error: wild +--- (what actually happened) --- +[First threw] Error: wild" +`; + +exports[`0090: load fails with nice error message if only one rule 2`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotReadableError: table:123 is not readable in vc:guest:master, because: + Rule AllowIf:First returned SKIP +--- (what actually happened) --- +[First threw] Error: wild +[First returned] false" +`; + +exports[`0100: insert fails with nice error message if only one rule 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +Error: wild +--- (what actually happened) --- +[First threw] Error: wild" +`; + +exports[`0100: insert fails with nice error message if only one rule 2`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotInsertableError: table: cannot insert in vc:guest:master, because: + Rule Require:First returned DENY +--- (what actually happened) --- +[First threw] Error: wild +[First returned] false" +`; + +exports[`0110: insert succeeds when DenyIf rule evaluates 1`] = ` +"OK +--- (what actually happened) --- +[First returned] false" +`; + +exports[`0120: load fails when DenyIf rule throws 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotReadableError: table:123 is not readable in vc:guest:master, because: + Rule DenyIf:First returned DENY, because: + other_table:987 is not readable in vc:guest:master, because: + ent access error +--- (what actually happened) --- +[First threw] EntNotReadableError: other_table:987 is not readable in vc:guest:master, because: + ent access error" +`; + +exports[`0130: fail when tenant user id mismatches 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotReadableError: table:123 is not readable in vc:999, because: + tenant_id is expected to be "999", but got "42" +--- (what actually happened) --- +" +`; + +exports[`0130: fail when tenant user id mismatches 2`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotInsertableError: table: cannot insert in vc:999, because: + tenant_id is expected to be "999", but got "42" +--- (what actually happened) --- +" +`; + +exports[`0130: fail when tenant user id mismatches 3`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotInsertableError: table: cannot insert in vc:999, because: + tenant_id is expected to be "999", but got undefined +--- (what actually happened) --- +" +`; + +exports[`0130: fail when tenant user id mismatches 4`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotUpdatableError: table:123 is not updatable/deletable in vc:999, because: + tenant_id is expected to be "999", but got "42" +--- (what actually happened) --- +" +`; + +exports[`0130: fail when tenant user id mismatches 5`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotUpdatableError: table:123 is not updatable/deletable in vc:999, because: + tenant_id is expected to be "999", but got "42" +--- (what actually happened) --- +" +`; + +exports[`0140: load succeeds when some of Or predicates succeed 1`] = ` +"OK +--- (what actually happened) --- +[First threw] EntNotReadableError: other_table:987 is not readable in vc:guest, because: + ent access error +[Second returned] true" +`; + +exports[`0150: load fails with nice error when all of Or predicates fail 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +EntNotInsertableError: table: cannot insert in vc:guest:master, because: + Rule Require:Or returned DENY, because: + First: other_table:987 is not readable in vc:guest, because: + { myKey: 'ent access error' } + Second: false +--- (what actually happened) --- +[First threw] EntNotReadableError: other_table:987 is not readable in vc:guest, because: + { myKey: 'ent access error' } +[Second returned] false" +`; + +exports[`0160: load crashes when some predicates fail with a wild error 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +Error: wild +--- (what actually happened) --- +[First threw] Error: wild +[Second returned] true" +`; diff --git a/src/ent/__tests__/__snapshots__/Validation.validate.test.ts.snap b/src/ent/__tests__/__snapshots__/Validation.validate.test.ts.snap new file mode 100644 index 0000000..afc85f3 --- /dev/null +++ b/src/ent/__tests__/__snapshots__/Validation.validate.test.ts.snap @@ -0,0 +1,100 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`0010: update fails when field validation fails 1`] = ` +"Failure +--- (error returned to client) --- +EntValidationError: table: tenant_id: "some one", tenant_id: "some two", name: "boom" +--- (what actually happened) --- +[FieldIs(tenant_id) [ '42', { id: '123', tenant_id: '42', name: 'hi' } ] returned] false +[FieldIs(tenant_id) [ '42', { id: '123', tenant_id: '42', name: 'hi' } ] returned] false +[FieldIs(tenant_id) [ 'hi', { id: '123', tenant_id: '42', name: 'hi' } ] returned] 'Promise'" +`; + +exports[`0020: update succeeds when validation fails, but the field is untouched 1`] = ` +"OK +--- (what actually happened) --- +[FieldIs(name) [ 'hi', { id: '123', tenant_id: '101', name: 'hi' } ] returned] false +[FieldIs(name) [ 'hi', { id: '123', tenant_id: '101', name: 'hi' } ] returned] { issues: [ { message: 'boom', path: [ 'name' ] } ] }" +`; + +exports[`0030: insert succeeds when multiple validators succeed 1`] = ` +"OK +--- (what actually happened) --- +[RowIs { id: '123', tenant_id: '42', name: 'hi' } returned] true +[RowIs { id: '123', tenant_id: '42', name: 'hi' } returned] { success: true } +[FieldIs(name) [ 'hi', { id: '123', tenant_id: '42', name: 'hi' } ] returned] true +[FieldIs(name) [ 'hi', { id: '123', tenant_id: '42', name: 'hi' } ] returned] { value: 42 }" +`; + +exports[`0040: insert fails when field validation fails 1`] = ` +"Failure +--- (error returned to client) --- +EntValidationError: table: name: "boom", name: "boom" +--- (what actually happened) --- +[FieldIs(name) [ 'hi', { id: '123', tenant_id: '42', name: 'hi' } ] returned] false +[FieldIs(name) [ 'hi', { id: '123', tenant_id: '42', name: 'hi' } ] returned] { success: false, error: { issues: [ { message: 'boom', path: [ 'name' ] } ] } }" +`; + +exports[`0050: update fails when RowIs fails 1`] = ` +"Failure +--- (error returned to client) --- +EntValidationError: table: null: "boom" +--- (what actually happened) --- +[RowIs { id: '123', tenant_id: '42', name: 'hi' } returned] false" +`; + +exports[`0060: update fails when RowIs fails, and RowIs propagates field name 1`] = ` +"Failure +--- (error returned to client) --- +EntValidationError: table: name: "boom" +--- (what actually happened) --- +[RowIs { id: '123', tenant_id: '42', name: 'hi' } returned] { success: false, error: { issues: [ { message: 'boom', path: [ 'name' ] } ] } }" +`; + +exports[`0070: update succeeds when RowIs fails, but the field it propagates is untouched 1`] = ` +"OK +--- (what actually happened) --- +[RowIs { id: '123', tenant_id: '42', name: 'hi' } returned] { issues: [ { message: 'boom', path: [ 'name' ] } ] }" +`; + +exports[`0080: insert fails when RowIs fails 1`] = ` +"Failure +--- (error returned to client) --- +EntValidationError: table: null: "boom", name: "boom", null: "Expected object, received number" +--- (what actually happened) --- +[RowIs { id: '123', tenant_id: '42', name: 'hi' } returned] false +[RowIs { id: '123', tenant_id: '42', name: 'hi' } returned] { success: false, error: { issues: [ { message: 'boom', path: [ 'name' ] } ] } } +[RowIs { id: '123', tenant_id: '42', name: 'hi' } returned] { success: false, error: { issues: [ { message: 'Expected object, received number', path: [] } ] } }" +`; + +exports[`0090: insert throws when FieldIs throws a weird error 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +Error: ouch +--- (what actually happened) --- +[FieldIs [ 'hi', { id: '123', tenant_id: '42', name: 'hi' } ] threw] Error: ouch" +`; + +exports[`0095: insert fails on EntNotFoundError in FieldIs 1`] = ` +"Failure +--- (error returned to client) --- +EntValidationError: table: name: "EntNotFoundError" +--- (what actually happened) --- +[FieldIs [ 'hi', { id: '123', tenant_id: '42', name: 'hi' } ] threw] EntNotFoundError: MyEnt not found: id=42" +`; + +exports[`0096: insert fails on manual EntValidationError in RowIs 1`] = ` +"Failure +--- (error returned to client) --- +EntValidationError: table: name: "huh" +--- (what actually happened) --- +[RowIs { id: '123', tenant_id: '42', name: 'hi' } threw] EntValidationError: MyEnt: name: "huh"" +`; + +exports[`0100: insert throws when RowIs throws a weird error 1`] = ` +"Error Thrown Through +--- (error returned to client) --- +Error: ouch +--- (what actually happened) --- +[RowIs { id: '123', tenant_id: '42', name: 'hi' } threw] Error: ouch" +`; diff --git a/src/ent/__tests__/test-utils.ts b/src/ent/__tests__/test-utils.ts new file mode 100644 index 0000000..f1bfd77 --- /dev/null +++ b/src/ent/__tests__/test-utils.ts @@ -0,0 +1,113 @@ +import { inspect } from "util"; +import type { InsertInput, Row, Table, UpdateInput } from "../../types"; +import { EntValidationError } from "../errors/EntValidationError"; +import type { Validation } from "../Validation"; +import { VC } from "../VC"; +import { VCWithQueryCache } from "../VCFlavor"; + +const vcTestGuest = + VC.createGuestPleaseDoNotUseCreationPointsMustBeLimited().withTransitiveMasterFreshness(); + +/** + * Creates a test VC. + */ +export function createVC(noCache?: "no-cache"): VC { + const vc = vcTestGuest + .withFlavor( + noCache ? undefined : new VCWithQueryCache({ maxQueries: 1000 }), + ) + .withDefaultFreshness(); + return vc; +} + +/** + * Normalizes the text before matching the snapshot. + */ +export function expectToMatchSnapshot( + str: string, + snapshotName?: string, +): void { + const exp = expect( + str.replace(/\b(vc:\w+)\(\d+\)/g, "$1").replace(/\d{10,}/g, ""), + ); + snapshotName ? exp.toMatchSnapshot(snapshotName) : exp.toMatchSnapshot(); +} + +/** + * A helper class to log some predicate response (plus the row argument) and + * return it, see how it's used in the code. + * + * - Q: "If we have a class which validates a validation, who'd be validating + * the function which validates the validation?" + * - A: "TS & Jest" + */ +export class ValidationTester { + private log: string[] = []; + + respond( + predName: string, + response: TRes, + row?: unknown, + ): TRes extends Error ? boolean : TRes { + predName += row ? " " + inspect(row) : ""; + if (response instanceof Error) { + this.log.push(`[${predName} threw] ${response}`); + throw response; + } else { + this.log.push( + `[${predName} returned] ` + + (typeof response === "object" + ? inspect( + response instanceof Promise + ? response.constructor.name // hide e.g. Datadog guts + : { ...response }, + { compact: true, depth: 10, breakLength: 10000 }, + ) + : response), + ); + return response as TRes extends Error ? boolean : TRes; + } + } + + async matchSnapshot({ + validation, + row, + method, + updateInput, + vc = vcTestGuest, + }: { + validation: Validation; + row: Row; + vc?: VC; + } & ( + | { + method: "validateLoad" | "validateInsert" | "validateDelete"; + updateInput?: never; + } + | { + method: "validateUpdate"; + updateInput: UpdateInput; + } + )): Promise { + let res = ""; + let allow = false; + try { + if (method === "validateUpdate") { + await validation.validateUpdate(vc, row, updateInput); + } else { + await validation[method](vc, row as InsertInput & Row); + } + + res = "OK"; + allow = true; + } catch (e: unknown) { + const severity = + e instanceof EntValidationError ? "Failure" : "Error Thrown Through"; + res = `${severity}\n--- (error returned to client) ---\n${e}`; + } + + res += "\n--- (what actually happened) ---\n" + this.log.join("\n"); + expect(res.replace(/\b(vc:\w+)\(\d+\)/g, "$1")).toMatchSnapshot(); + return allow; + } +} diff --git a/src/ent/errors/EntAccessError.ts b/src/ent/errors/EntAccessError.ts new file mode 100644 index 0000000..f4a0a95 --- /dev/null +++ b/src/ent/errors/EntAccessError.ts @@ -0,0 +1,70 @@ +import { indent, inspectCompact, appendCause } from "../../internal/misc"; + +/** + * Standard Schema V1 compatible error result. Every EntAccessError can be + * converted to it. See https://standardschema.dev. + */ +export interface StandardSchemaV1FailureResult { + readonly issues: ReadonlyArray<{ + readonly message: string; + readonly path?: readonly string[] | undefined; + }>; +} + +/** + * A base class for errors that trigger the validation framework to process them + * as a DENY/SKIP. Invariants in derived classes: the error message should be + * safe to pass to the client (it must not have any private information; a good + * example is EntValidationError), plus the message alone should be descriptive + * enough to extract information from it. If `cause` is passed, it becomes a + * part of the message, with the above assumptions. + */ +export class EntAccessError extends Error { + public readonly cause: string | Error | null; + + constructor( + public readonly entName: string, + message: string, + cause: unknown = null, + ) { + super( + cause + ? message.replace(/[,.?!:;]+$/s, "") + + ", because:\n" + + indent(causeToString(cause)) + : message, + ); + + Object.defineProperty(this, "name", { + value: this.constructor.name, + writable: true, + enumerable: false, + }); + + if (cause instanceof Error) { + this.cause = cause; + appendCause(this, cause); + } else { + this.cause = cause ? causeToString(cause) : null; + } + } + + toStandardSchemaV1(): StandardSchemaV1FailureResult { + return { + issues: [ + this.entName + ? { message: this.message, path: [this.entName] } + : { message: this.message }, + ], + }; + } +} + +function causeToString(cause: unknown): string { + // - Error is turned into its class name and the message (no stacktrace). + // - String is left as it is. + // - Any other object will be inspected in compact mode. + return cause instanceof Error || typeof cause === "string" + ? String(cause).trimEnd() + : inspectCompact(cause); +} diff --git a/src/ent/errors/EntNotFoundError.ts b/src/ent/errors/EntNotFoundError.ts new file mode 100644 index 0000000..ec8845c --- /dev/null +++ b/src/ent/errors/EntNotFoundError.ts @@ -0,0 +1,32 @@ +import { sanitizeIDForDebugPrinting } from "../../internal/misc"; +import { EntAccessError } from "./EntAccessError"; + +/** + * Error: non-existing ID in the database (failed loadX() call), or non-existing + * Ent (failed loadByX() call). Notice that `where` data is intentionally NOT + * considered as private and may be delivered to the client. + */ +export class EntNotFoundError extends EntAccessError { + constructor( + entName: string, + public readonly where: Record, + cause: unknown = null, + ) { + super(entName, `${entName} not found: ${whereToText(where)}`, cause); + } +} + +function whereToText(where: Record): string { + if (Object.keys(where).length === 1) { + const [k, v] = Object.entries(where)[0]; + return `${k}=${sanitizeIDForDebugPrinting(v)}`; + } + + return ( + "(" + + Object.keys(where).join(",") + + ")=(" + + Object.values(where).map(sanitizeIDForDebugPrinting).join(",") + + ")" + ); +} diff --git a/src/ent/errors/EntNotInsertableError.ts b/src/ent/errors/EntNotInsertableError.ts new file mode 100644 index 0000000..a485932 --- /dev/null +++ b/src/ent/errors/EntNotInsertableError.ts @@ -0,0 +1,15 @@ +import { EntAccessError } from "./EntAccessError"; + +/** + * Error: thrown when an Ent cannot be inserted due to privacy reasons. + */ +export class EntNotInsertableError extends EntAccessError { + constructor( + entName: string, + public readonly vc: string, + public readonly row: object, + cause: unknown = null, + ) { + super(entName, `${entName}: cannot insert in ${vc}`, cause); + } +} diff --git a/src/ent/errors/EntNotReadableError.ts b/src/ent/errors/EntNotReadableError.ts new file mode 100644 index 0000000..8cc704b --- /dev/null +++ b/src/ent/errors/EntNotReadableError.ts @@ -0,0 +1,17 @@ +import type { RowWithID } from "../../types"; +import { ID } from "../../types"; +import { EntAccessError } from "./EntAccessError"; + +/** + * Error: thrown when an Ent cannot be read due to privacy reasons. + */ +export class EntNotReadableError extends EntAccessError { + constructor( + entName: string, + public readonly vc: string, + public readonly row: RowWithID, + cause: unknown = null, + ) { + super(entName, `${entName}:${row[ID]} is not readable in ${vc}`, cause); + } +} diff --git a/src/ent/errors/EntNotUpdatableError.ts b/src/ent/errors/EntNotUpdatableError.ts new file mode 100644 index 0000000..d3b2865 --- /dev/null +++ b/src/ent/errors/EntNotUpdatableError.ts @@ -0,0 +1,21 @@ +import type { RowWithID } from "../../types"; +import { ID } from "../../types"; +import { EntAccessError } from "./EntAccessError"; + +/** + * Error: thrown when an Ent cannot be updated or deleted due to privacy reasons. + */ +export class EntNotUpdatableError extends EntAccessError { + constructor( + entName: string, + public readonly vc: string, + public readonly row: RowWithID, + cause: unknown = null, + ) { + super( + entName, + `${entName}:${row[ID]} is not updatable/deletable in ${vc}`, + cause, + ); + } +} diff --git a/src/ent/errors/EntUniqueKeyError.ts b/src/ent/errors/EntUniqueKeyError.ts new file mode 100644 index 0000000..d167268 --- /dev/null +++ b/src/ent/errors/EntUniqueKeyError.ts @@ -0,0 +1,39 @@ +import { inspectCompact } from "../../internal/misc"; + +/** + * Error: while inserting or updating, DB unique key was violated, + * so the Ent was not mutated. + */ +export class EntUniqueKeyError extends Error { + constructor( + public readonly entName: string, + public readonly input: unknown, + ) { + super( + `${entName} mutation violates unique key constraint: ` + + inspectCompact(input), + ); + + Object.defineProperty(this, "name", { + value: this.constructor.name, + writable: true, + enumerable: false, + }); + } + + /** + * Returns a promise of T on success, and undefined in case unique key + * violation happened during the promise resolution. + */ + static async ignore(promise: Promise): Promise { + try { + return await promise; + } catch (e: unknown) { + if (!(e instanceof this)) { + throw e; + } + + return undefined; + } + } +} diff --git a/src/ent/errors/EntValidationError.ts b/src/ent/errors/EntValidationError.ts new file mode 100644 index 0000000..b7fb75b --- /dev/null +++ b/src/ent/errors/EntValidationError.ts @@ -0,0 +1,49 @@ +import type { StandardSchemaV1FailureResult } from "./EntAccessError"; +import { EntAccessError } from "./EntAccessError"; + +/** + * Error: thrown after all validators are executed, and some of them think that + * the row is invalid. + */ +export class EntValidationError extends EntAccessError { + constructor( + entName: string, + public readonly errors: readonly EntValidationErrorInfo[], + ) { + super( + entName, + // The below string is just for debugging purposes. + `${entName}: ` + + errors + .map((error) => error.field + ": " + JSON.stringify(error.message)) + .join(", "), + ); + } + + /** + * Converts the payload to a Standard Schema V1 compatible error result. See + * https://standardschema.dev. + */ + override toStandardSchemaV1(): StandardSchemaV1FailureResult { + return { + issues: this.errors.map(({ field, message }) => ({ + message, + path: + this.entName || field !== null + ? [ + ...(this.entName ? [this.entName] : []), + ...(field !== null ? [field] : []), + ] + : undefined, + })), + }; + } +} + +/** + * Auxiliary information which every validation predicate should emit. + */ +export interface EntValidationErrorInfo { + field: string | null; // it null, the message relates to the whole row + message: string; +} diff --git a/src/ent/errors/__tests__/EntAccessError.test.ts b/src/ent/errors/__tests__/EntAccessError.test.ts new file mode 100644 index 0000000..a7712df --- /dev/null +++ b/src/ent/errors/__tests__/EntAccessError.test.ts @@ -0,0 +1,23 @@ +import { EntAccessError } from "../EntAccessError"; + +test("toStandardSchemaV1", () => { + expect( + new EntAccessError("EntTest", "my-message").toStandardSchemaV1(), + ).toEqual({ + issues: [{ message: "my-message", path: ["EntTest"] }], + }); + + expect(new EntAccessError("", "my-message").toStandardSchemaV1()).toEqual({ + issues: [{ message: "my-message" }], + }); + + expect( + JSON.stringify( + new EntAccessError( + "EntTest", + "my-message", + "my-cause", + ).toStandardSchemaV1(), + ), + ).toContain("my-cause"); +}); diff --git a/src/ent/errors/__tests__/EntNotFoundError.test.ts b/src/ent/errors/__tests__/EntNotFoundError.test.ts new file mode 100644 index 0000000..b83f889 --- /dev/null +++ b/src/ent/errors/__tests__/EntNotFoundError.test.ts @@ -0,0 +1,12 @@ +import { EntNotFoundError } from "../EntNotFoundError"; + +test("toStandardSchemaV1", () => { + expect( + JSON.stringify( + new EntNotFoundError("EntTest", { + field1: "loadByField1", + field2: "loadByField2", + }).toStandardSchemaV1(), + ), + ).toContain("loadByField1"); // OK to deliver it to the client +}); diff --git a/src/ent/errors/__tests__/EntNotInsertableError.test.ts b/src/ent/errors/__tests__/EntNotInsertableError.test.ts new file mode 100644 index 0000000..0c996bd --- /dev/null +++ b/src/ent/errors/__tests__/EntNotInsertableError.test.ts @@ -0,0 +1,11 @@ +import { EntNotInsertableError } from "../EntNotInsertableError"; + +test("toStandardSchemaV1", () => { + expect( + JSON.stringify( + new EntNotInsertableError("EntTest", "my-vc", { + private: "data", + }).toStandardSchemaV1(), + ), + ).not.toContain("private"); +}); diff --git a/src/ent/errors/__tests__/EntNotReadableError.test.ts b/src/ent/errors/__tests__/EntNotReadableError.test.ts new file mode 100644 index 0000000..fa7c78b --- /dev/null +++ b/src/ent/errors/__tests__/EntNotReadableError.test.ts @@ -0,0 +1,13 @@ +import type { RowWithID } from "../../../types"; +import { EntNotReadableError } from "../EntNotReadableError"; + +test("toStandardSchemaV1", () => { + expect( + JSON.stringify( + new EntNotReadableError("EntTest", "my-vc", { + id: "42", + private: "data", + } as RowWithID).toStandardSchemaV1(), + ), + ).not.toContain("private"); +}); diff --git a/src/ent/errors/__tests__/EntNotUpdatableError.test.ts b/src/ent/errors/__tests__/EntNotUpdatableError.test.ts new file mode 100644 index 0000000..dd48bdd --- /dev/null +++ b/src/ent/errors/__tests__/EntNotUpdatableError.test.ts @@ -0,0 +1,13 @@ +import type { RowWithID } from "../../../types"; +import { EntNotUpdatableError } from "../EntNotUpdatableError"; + +test("toStandardSchemaV1", () => { + expect( + JSON.stringify( + new EntNotUpdatableError("EntTest", "my-vc", { + id: "42", + private: "data", + } as RowWithID).toStandardSchemaV1(), + ), + ).not.toContain("private"); +}); diff --git a/src/ent/errors/__tests__/EntValidationError.test.ts b/src/ent/errors/__tests__/EntValidationError.test.ts new file mode 100644 index 0000000..9ce0141 --- /dev/null +++ b/src/ent/errors/__tests__/EntValidationError.test.ts @@ -0,0 +1,27 @@ +import { EntValidationError } from "../EntValidationError"; + +test("toStandardSchemaV1", () => { + expect( + new EntValidationError("EntTest", [ + { field: "field1", message: "message1" }, + { field: null, message: "message2" }, + ]).toStandardSchemaV1(), + ).toEqual({ + issues: [ + { message: "message1", path: ["EntTest", "field1"] }, + { message: "message2", path: ["EntTest"] }, + ], + }); + + expect( + new EntValidationError("", [ + { field: "field1", message: "message1" }, + { field: null, message: "message2" }, + ]).toStandardSchemaV1(), + ).toEqual({ + issues: [ + { message: "message1", path: ["field1"] }, + { message: "message2" }, + ], + }); +}); diff --git a/src/ent/index.ts b/src/ent/index.ts new file mode 100644 index 0000000..def44f3 --- /dev/null +++ b/src/ent/index.ts @@ -0,0 +1,47 @@ +/** + * @file Automatically generated by barrelsby. + */ + +export * from "./BaseEnt"; +export * from "./Configuration"; +export * from "./IDsCache"; +export * from "./Inverse"; +export * from "./QueryCache"; +export * from "./ShardAffinity"; +export * from "./ShardLocator"; +export * from "./TimelineStorage"; +export * from "./Triggers"; +export * from "./VC"; +export * from "./VCCaches"; +export * from "./VCFlavor"; +export * from "./VCTrace"; +export * from "./Validation"; +export * from "./types"; +export * from "./errors/EntAccessError"; +export * from "./errors/EntNotFoundError"; +export * from "./errors/EntNotInsertableError"; +export * from "./errors/EntNotReadableError"; +export * from "./errors/EntNotUpdatableError"; +export * from "./errors/EntUniqueKeyError"; +export * from "./errors/EntValidationError"; +export * from "./mixins/CacheMixin"; +export * from "./mixins/ConfigMixin"; +export * from "./mixins/HelpersMixin"; +export * from "./mixins/PrimitiveMixin"; +export * from "./predicates/AbstractIs"; +export * from "./predicates/CanDeleteOutgoingEdge"; +export * from "./predicates/CanReadOutgoingEdge"; +export * from "./predicates/CanUpdateOutgoingEdge"; +export * from "./predicates/FieldIs"; +export * from "./predicates/IncomingEdgeFromVCExists"; +export * from "./predicates/Or"; +export * from "./predicates/OutgoingEdgePointsToVC"; +export * from "./predicates/Predicate"; +export * from "./predicates/RowIs"; +export * from "./predicates/True"; +export * from "./predicates/VCHasFlavor"; +export * from "./rules/AllowIf"; +export * from "./rules/DenyIf"; +export * from "./rules/Require"; +export * from "./rules/Rule"; +export * from "./rules/evaluate"; diff --git a/src/ent/mixins/CacheMixin.ts b/src/ent/mixins/CacheMixin.ts new file mode 100644 index 0000000..8f40736 --- /dev/null +++ b/src/ent/mixins/CacheMixin.ts @@ -0,0 +1,163 @@ +import type { Client } from "../../abstract/Client"; +import { hasKey } from "../../internal/misc"; +import { ID } from "../../types"; +import type { + SelectByInput, + CountInput, + ExistsInput, + InsertInput, + LoadByInput, + Order, + Table, + UniqueKey, + Where, +} from "../../types"; +import { QueryCache } from "../QueryCache"; +import type { UpdateOriginalInput } from "../types"; +import type { VC } from "../VC"; +import type { PrimitiveClass, PrimitiveInstance } from "./PrimitiveMixin"; + +const MULTI_ROW_AFFECTING_OPS = [ + "loadByNullable", + "selectBy", + "select", + "count", + "exists", +] as const; + +/** + * Modifies the passed class adding VC-stored cache layer to it. + */ +export function CacheMixin< + TTable extends Table, + TUniqueKey extends UniqueKey, + TClient extends Client, +>( + Base: PrimitiveClass, +): PrimitiveClass { + class CacheMixin extends Base { + declare ["constructor"]: typeof CacheMixin; + + static override async insertIfNotExists( + vc: VC, + input: InsertInput, + ): Promise { + const resPromise = super.insertIfNotExists(vc, input); + vc.cache(QueryCache) + .delete(this, ["loadNullable"], hasKey(ID, input) ? input[ID] : null) + .delete(this, MULTI_ROW_AFFECTING_OPS); + return resPromise; + } + + static override async upsert( + vc: VC, + input: InsertInput, + ): Promise { + const resPromise = super.upsert(vc, input); + vc.cache(QueryCache) + .delete(this, ["loadNullable"], hasKey(ID, input) ? input[ID] : null) + .delete(this, MULTI_ROW_AFFECTING_OPS); + return resPromise; + } + + static override async loadNullable>( + this: new () => TEnt, + vc: VC, + id: string, + ): Promise { + return vc + .cache(QueryCache) + .through(this, "loadNullable", id, async () => + super.loadNullable(vc, id), + ) as Promise; + } + + static override async loadByNullable< + TEnt extends PrimitiveInstance, + >( + this: new () => TEnt, + vc: VC, + input: LoadByInput, + ): Promise { + return vc + .cache(QueryCache) + .through(this, "loadByNullable", JSON.stringify(input), async () => + super.loadByNullable(vc, input), + ) as Promise; + } + + static override async selectBy>( + this: new () => TEnt, + vc: VC, + input: SelectByInput, + ): Promise { + return vc + .cache(QueryCache) + .through(this, "selectBy", JSON.stringify(input), async () => + super.selectBy(vc, input), + ) as Promise; + } + + static override async select>( + this: new () => TEnt, + vc: VC, + where: Where, + limit: number, + order?: Order, + custom?: {}, + ): Promise { + return vc + .cache(QueryCache) + .through( + this, + "select", + JSON.stringify([where, limit, order, custom]), + async () => super.select(vc, where, limit, order, custom), + ) as Promise; + } + + static override async count( + vc: VC, + where: CountInput, + ): Promise { + return vc + .cache(QueryCache) + .through(this, "count", JSON.stringify(where), async () => + super.count(vc, where), + ); + } + + static override async exists( + vc: VC, + where: ExistsInput, + ): Promise { + return vc + .cache(QueryCache) + .through(this, "exists", JSON.stringify(where), async () => + super.exists(vc, where), + ); + } + + override async updateOriginal( + input: UpdateOriginalInput, + ): Promise { + const resPromise = super.updateOriginal(input); + this.vc + .cache(QueryCache) + .delete(this.constructor, ["loadNullable"], this[ID]) + .delete(this.constructor, MULTI_ROW_AFFECTING_OPS); + return resPromise; + } + + override async deleteOriginal(): Promise { + const resPromise = super.deleteOriginal(); + this.vc + .cache(QueryCache) + .delete(this.constructor, ["loadNullable"], this[ID]) + .delete(this.constructor, MULTI_ROW_AFFECTING_OPS); + return resPromise; + } + } + + return CacheMixin; +} diff --git a/src/ent/mixins/ConfigMixin.ts b/src/ent/mixins/ConfigMixin.ts new file mode 100644 index 0000000..5b1b044 --- /dev/null +++ b/src/ent/mixins/ConfigMixin.ts @@ -0,0 +1,236 @@ +import compact from "lodash/compact"; +import type { Client } from "../../abstract/Client"; +import type { Cluster } from "../../abstract/Cluster"; +import type { Schema } from "../../abstract/Schema"; +import { entries } from "../../internal/misc"; +import type { FieldOfIDType, Table, UniqueKey } from "../../types"; +import { ID } from "../../types"; +import { Configuration } from "../Configuration"; +import { Inverse } from "../Inverse"; +import { GLOBAL_SHARD, type ShardAffinity } from "../ShardAffinity"; +import { ShardLocator } from "../ShardLocator"; +import { Triggers } from "../Triggers"; +import { Validation } from "../Validation"; + +export interface ConfigInstance {} + +export interface ConfigClass< + TTable extends Table, + TUniqueKey extends UniqueKey, + TClient extends Client, +> { + /** + * Some Ent parameters need to be configured lazily, on the 1st access, + * because there could be cyclic references between Ent classes (e.g. in their + * privacy rules). So configure() is called on some later stage, at the moment + * of actual Ent operations (like loading, creation etc.). There is no static + * abstract methods in TS yet, so making it non-abstract. + */ + configure(): Configuration; + + /** + * A helper class to work-around TS weakness in return value type inference: + * https://github.com/Microsoft/TypeScript/issues/31273. It could've been just + * a function, but having a class is a little more natural. + */ + readonly Configuration: new ( + cfg: Configuration, + ) => Configuration; + + /** + * A Cluster where this Ent lives. + */ + readonly CLUSTER: Cluster; + + /** + * A schema which represents this Ent. + */ + readonly SCHEMA: Schema; + + /** + * Defines how to find the right Shard during Ent insertion. + */ + readonly SHARD_AFFINITY: ShardAffinity>; + + /** + * Shard locator for this Ent, responsible for resolving IDs into Shard objects. + */ + readonly SHARD_LOCATOR: ShardLocator>; + + /** + * Privacy rules for this Ent class. + */ + readonly VALIDATION: Validation; + + /** + * Triggers for this Ent class. + */ + readonly TRIGGERS: Triggers; + + /** + * Inverse assoc managers for fields. + */ + readonly INVERSES: Array>; + + /** + * TS requires us to have a public constructor to infer instance types in + * various places. We make this constructor throw if it's called. + */ + new (): ConfigInstance; +} + +/** + * Modifies the passed class adding support for Ent configuration (such as: + * Cluster, table schema, privacy rules, triggers etc.). + */ +export function ConfigMixin< + TTable extends Table, + TUniqueKey extends UniqueKey, + TClient extends Client, +>( + Base: new () => {}, + cluster: Cluster, + schema: Schema, +): ConfigClass { + class ConfigMixin extends Base { + static Configuration: new ( + c: Configuration, + ) => Configuration = Configuration; + + static readonly CLUSTER = cluster; + + static readonly SCHEMA = schema; + + static get SHARD_AFFINITY(): ShardAffinity> { + Object.defineProperty(this, "SHARD_AFFINITY", { + value: this.configure().shardAffinity, + writable: false, + }); + return this.SHARD_AFFINITY; + } + + static get SHARD_LOCATOR(): ShardLocator< + TClient, + TTable, + FieldOfIDType + > { + Object.defineProperty(this, "SHARD_LOCATOR", { + value: new ShardLocator({ + cluster, + entName: this.name, + shardAffinity: this.SHARD_AFFINITY, + uniqueKey: schema.uniqueKey, + inverses: this.INVERSES, + }), + writable: false, + }); + return this.SHARD_LOCATOR; + } + + static get VALIDATION(): Validation { + const cfg = this.configure(); + Object.defineProperty(this, "VALIDATION", { + value: new Validation(this.name, { + tenantPrincipalField: cfg.privacyTenantPrincipalField, + inferPrincipal: async (vc, row) => { + const res = + typeof cfg.privacyInferPrincipal === "function" + ? await cfg.privacyInferPrincipal(vc, row) + : cfg.privacyInferPrincipal; + const lowerVC = + typeof res === "string" + ? vc.toLowerInternal(res) + : res === null + ? vc.toGuest() + : res.vc; + if (lowerVC.isOmni()) { + throw Error( + `It is prohibited to return an omni VC "${lowerVC.toString()}" from ${this.name} privacyInferPrincipal callback. Loading VC was: "${vc.toString()}".`, + ); + } else { + return lowerVC; + } + }, + load: cfg.privacyLoad, + insert: cfg.privacyInsert, + update: cfg.privacyUpdate, + delete: cfg.privacyDelete, + validate: cfg.validators, + }), + writable: false, + }); + return this.VALIDATION; + } + + static get TRIGGERS(): Triggers { + const cfg = this.configure(); + Object.defineProperty(this, "TRIGGERS", { + value: new Triggers( + cfg.beforeInsert ?? [], + (cfg.beforeUpdate ?? []).map((trigger) => + trigger instanceof Array ? trigger : [null, trigger], + ), + cfg.beforeDelete ?? [], + (cfg.beforeMutation ?? []).map((trigger) => + trigger instanceof Array ? trigger : [null, trigger], + ), + cfg.afterInsert ?? [], + (cfg.afterUpdate ?? []).map((trigger) => + trigger instanceof Array ? trigger : [null, trigger], + ), + cfg.afterDelete ?? [], + (cfg.afterMutation ?? []).map((trigger) => + trigger instanceof Array ? trigger : [null, trigger], + ), + ), + writable: false, + }); + return this.TRIGGERS; + } + + static get INVERSES(): Array> { + const cfg = this.configure(); + Object.defineProperty(this, "INVERSES", { + value: compact( + entries(cfg.inverses ?? {}).map(([field, { name, type }]) => { + if (this.SHARD_AFFINITY === GLOBAL_SHARD) { + throw Error( + `It's useless to define a ${field} inverse for GLOBAL_SHARD schemas; use just a DB index`, + ); + } + + const spec = schema.table[field]; + if ( + schema.table[field].type !== ID || + spec.autoInsert || + spec.autoUpdate + ) { + throw Error( + `To have inverse specified, the '${field}' must be of type ${ID} and have no autoInsert/autoUpdate`, + ); + } + + return new Inverse({ + cluster, + shardAffinity: this.SHARD_AFFINITY as ShardAffinity, + id2Schema: schema, + id2Field: field, + name, + type, + }); + }), + ), + writable: false, + }); + return this.INVERSES; + } + + declare ["constructor"]: typeof ConfigMixin; + + static configure(): Configuration { + throw Error(`Please define ${this.name}.configure() method`); + } + } + + return ConfigMixin; +} diff --git a/src/ent/mixins/HelpersMixin.ts b/src/ent/mixins/HelpersMixin.ts new file mode 100644 index 0000000..bd913d6 --- /dev/null +++ b/src/ent/mixins/HelpersMixin.ts @@ -0,0 +1,306 @@ +import type { Client } from "../../abstract/Client"; +import type { AddNew, OmitNew } from "../../internal/misc"; +import type { + InsertInput, + LoadByInput, + Row, + Table, + UniqueKey, + UpdateField, + UpdateInput, +} from "../../types"; +import { ID } from "../../types"; +import { EntAccessError } from "../errors/EntAccessError"; +import { EntNotFoundError } from "../errors/EntNotFoundError"; +import { EntUniqueKeyError } from "../errors/EntUniqueKeyError"; +import type { UpdateOriginalInput } from "../types"; +import type { VC } from "../VC"; +import type { PrimitiveClass, PrimitiveInstance } from "./PrimitiveMixin"; + +export interface HelpersInstance< + TTable extends Table, +> extends PrimitiveInstance { + /** + * Same as updateOriginal(), but updates only the fields which are different + * in input and in the current object. + * - This method can works with CAS; see $cas property of the passed object. + * If CAS fails, returns false, the same way as updateOriginal() does. + * - If there was no such Ent in the DB, returns false, the same way as + * updateOriginal() does. + * - If no changed fields are detected, returns null as an indication (it's + * still falsy, but is different from the parent updateOriginal's `false`). + * - Otherwise, when an update happened, returns the list of fields which were + * different and triggered that change (a truthy value). The order of fields + * in the list matches the order of fields in the Ent schema definition. + */ + updateChanged( + input: UpdateOriginalInput, + ): Promise> | false | null>; + + /** + * Same as updateChanged(), but returns the updated Ent (or the original one + * if no fields were updated). + */ + updateChangedReturningX>( + this: TEnt, + input: UpdateInput, + ): Promise; + + /** + * Same as updateOriginal(), but returns the updated Ent (or null of there + * was no such Ent in the database). + */ + updateReturningNullable>( + this: TEnt, + input: UpdateInput, + ): Promise; + + /** + * Same as updateOriginal(), but throws if the object wasn't updated or + * doesn't exist after the update. + */ + updateReturningX>( + this: TEnt, + input: UpdateInput, + ): Promise; +} + +export interface HelpersClass< + TTable extends Table, + TUniqueKey extends UniqueKey, + TClient extends Client, +> extends OmitNew> { + /** + * Same as insertIfNotExists(), but throws if the Ent violates unique key + * constraints. + */ + insert: (vc: VC, input: InsertInput) => Promise; + + /** + * Same as insert(), but returns the created Ent. + */ + insertReturning: >( + this: new () => TEnt, + vc: VC, + input: InsertInput, + ) => Promise; + + /** + * Same, but returns the created/updated Ent. + */ + upsertReturning: >( + this: new () => TEnt, + vc: VC, + input: InsertInput, + ) => Promise; + + /** + * Same as loadNullable(), but if no permissions to read, returns null and + * doesn't throw. It's more a convenience function rather than a concept. + */ + loadIfReadableNullable: >( + this: new () => TEnt, + vc: VC, + id: string, + ) => Promise; + + /** + * Loads an Ent by its ID. Throws if no such Ent is found. + * This method is used VERY often. + */ + loadX: >( + this: new () => TEnt, + vc: VC, + id: string, + ) => Promise; + + /** + * Loads an Ent by its ID. Throws if no such Ent is found. + * This method is used VERY often. + */ + loadByX: >( + this: new () => TEnt, + vc: VC, + input: LoadByInput, + ) => Promise; + + /** + * TS requires us to have a public constructor to infer instance types in + * various places. We make this constructor throw if it's called. + */ + new (): HelpersInstance & Row; +} + +/** + * Modifies the passed class adding convenience methods (like loadX() which + * throws when an Ent can't be loaded instead of returning null as it's done in + * the primitive operations). + */ +export function HelpersMixin< + TTable extends Table, + TUniqueKey extends UniqueKey, + TClient extends Client, +>( + Base: PrimitiveClass, +): HelpersClass { + class HelpersMixin extends Base { + declare ["constructor"]: typeof HelpersMixin; + + static async insert(vc: VC, input: InsertInput): Promise { + const id = await this.insertIfNotExists(vc, input); + if (!id) { + throw new EntUniqueKeyError(this.name, input); + } + + return id; + } + + static async insertReturning( + vc: VC, + input: InsertInput, + ): Promise { + const id = await this.insert(vc, input); + return this.loadX(vc, id); + } + + static async upsertReturning( + vc: VC, + input: InsertInput, + ): Promise { + const id = await this.upsert(vc, input); + return this.loadX(vc, id); + } + + static async loadIfReadableNullable( + vc: VC, + id: string, + ): Promise { + try { + return await this.loadNullable(vc, id); + } catch (e: unknown) { + if (e instanceof EntAccessError) { + return null; + } + + throw e; + } + } + + static async loadX(vc: VC, id: string): Promise { + const ent = await this.loadNullable(vc, id); + if (!ent) { + throw new EntNotFoundError(this.name, { [ID]: id }); + } + + return ent; + } + + static async loadByX( + vc: VC, + input: LoadByInput, + ): Promise { + const ent = await this.loadByNullable(vc, input); + if (!ent) { + throw new EntNotFoundError(this.name, input); + } + + return ent; + } + + async updateChanged( + input: UpdateOriginalInput, + ): Promise> | false | null> { + const changedFields: Array> = []; + const changedInput: UpdateOriginalInput = {}; + + // Iterate over BOTH regular fields AND symbol fields. Notice that for + // symbol fields, we'll always have a "changed" signal since the input Ent + // doesn't have them (they are to be used in triggers only). + for (const keyOrSymbol of Reflect.ownKeys( + this.constructor.SCHEMA.table, + )) { + // ID field is always treated as immutable. + if (keyOrSymbol === ID) { + continue; + } + + const field = keyOrSymbol as Exclude; + const value = input[field]; + const existingValue = (this as Record)[field]; + + // Undefined is treated as "do not touch" signal for the field. + if (value === undefined) { + continue; + } + + // Exact equality means "do not touch". + if (existingValue === value) { + continue; + } + + // Works for most of Node built-in types: Date, Buffer, as well as for + // user-defined custom types. + if ( + value !== null && + typeof value === "object" && + existingValue !== null && + typeof existingValue === "object" && + JSON.stringify(value) === JSON.stringify(existingValue) + ) { + continue; + } + + // There IS a change in this field. Record it. + changedInput[field] = value; + changedFields.push(field); + } + + if (changedFields.length > 0) { + if (input.$literal) { + changedInput.$literal = input.$literal; + } + + if (input.$cas) { + changedInput.$cas = input.$cas; + } + + return (await this.updateOriginal(changedInput)) + ? changedFields + : false; + } + + return null; + } + + async updateChangedReturningX( + input: UpdateInput, + ): Promise { + return (await this.updateChanged(input as UpdateOriginalInput)) + ? this.constructor.loadX(this.vc, this[ID]) + : this; + } + + async updateReturningNullable( + input: UpdateInput, + ): Promise { + const updated = await this.updateOriginal( + input as UpdateOriginalInput, + ); + return updated ? this.constructor.loadNullable(this.vc, this[ID]) : null; + } + + async updateReturningX(input: UpdateInput): Promise { + const res = await this.updateReturningNullable(input); + if (!res) { + throw new EntNotFoundError(this.constructor.name, { [ID]: this[ID] }); + } + + return res; + } + } + + return HelpersMixin as AddNew< + typeof HelpersMixin, + Row + > as HelpersClass; +} diff --git a/src/ent/mixins/PrimitiveMixin.ts b/src/ent/mixins/PrimitiveMixin.ts new file mode 100644 index 0000000..4f80926 --- /dev/null +++ b/src/ent/mixins/PrimitiveMixin.ts @@ -0,0 +1,794 @@ +import { memoize2 } from "fast-typescript-memoize"; +import compact from "lodash/compact"; +import first from "lodash/first"; +import flatten from "lodash/flatten"; +import sum from "lodash/sum"; +import type { Client } from "../../abstract/Client"; +import { ClientError } from "../../abstract/ClientError"; +import type { OmitNew } from "../../internal/misc"; +import { hasKey, join, mapJoin } from "../../internal/misc"; +import type { + CountInput, + ExistsInput, + InsertInput, + LoadByInput, + Order, + Row, + SelectByInput, + Table, + UniqueKey, + UpdateInput, + Where, +} from "../../types"; +import { ID } from "../../types"; +import { EntNotInsertableError } from "../errors/EntNotInsertableError"; +import { IDsCacheReadable, IDsCacheUpdatable } from "../predicates/Predicate"; +import type { TriggerUpdateOrDeleteOldRow } from "../Triggers"; +import type { UpdateOriginalInput } from "../types"; +import type { VC } from "../VC"; +import type { ConfigClass, ConfigInstance } from "./ConfigMixin"; + +export interface PrimitiveInstance< + TTable extends Table, +> extends ConfigInstance { + /** + * VC of this Ent. + */ + readonly vc: VC; + + /** + * For simplicity, every Ent has an ID field name hardcoded to "id". + */ + readonly [ID]: string; + + /** + * Updates the object in the DB, but doesn't update the Ent itself (since it's + * immutable). + * - This method can work with CAS; see $cas property of the passed object. + * - If a special value "skip-if-someone-else-changed-updating-ent-props" is + * passed to $cas, then the list of props for CAS is brought from the input, + * and the values of these props are brought from the Ent itself (i.e. from + * `this`). + * - If a special value, a list of field names, is passed to $cas, then it + * works like described above, but the list of prop names is brought from + * that list of field names. + * - Returns false if there is no such object in the DB, or if CAS check + * didn't succeed. + * - Returns true if the object was found and updated. + */ + updateOriginal(input: UpdateOriginalInput): Promise; + + /** + * Deletes the object in the DB. Returns true if the object was found. Keeps + * the current object untouched (since it's immutable). + */ + deleteOriginal(): Promise; +} + +export type PrimitiveClass< + TTable extends Table, + TUniqueKey extends UniqueKey, + TClient extends Client, +> = OmitNew> & { + /** + * Runs INSERT mutation for the Ent. + * - The Shard is inferred from the input fields using SHARD_AFFINITY. + * - Returns ID of the newly inserted row. + * - Returns null if the Ent violates unique key constraints. + * - If the Ent has some triggers set up, this will be translated into two + * schema operations: idGen() and insert(), and before-triggers will run in + * between having the ID known in advance. + */ + insertIfNotExists: ( + vc: VC, + input: InsertInput, + ) => Promise; + + /** + * Inserts an Ent or updates an existing one if unique key matches. + * - Don't use upsert() too often, because upsert may still delete IDs even if + * the object was updated, not inserted (there is no good ways to solve this + * in some DB engines like relational DBs so far). + * - Upsert can't work if some triggers are defined for the Ent, because we + * don't know Ent ID in advance (whether the upsert succeeds or skips on + * duplication). + */ + upsert: (vc: VC, input: InsertInput) => Promise; + + /** + * Loads an Ent by its ID. Returns null if no such Ent exists. Try to use + * loadX() instead as much as you can. + */ + loadNullable: >( + this: new () => TEnt, + vc: VC, + id: string, + ) => Promise; + + /** + * Loads an Ent by its unique key. Returns null if no such Ent exists. Notice + * that the key must be REALLY unique, otherwise the database may return + * multiple items, and the API will break. Don't try to use this method with + * non-unique keys! + */ + loadByNullable: >( + this: new () => TEnt, + vc: VC, + input: LoadByInput, + ) => Promise; + + /** + * Selects the list of Ents by their unique key prefix. The query can span + * multiple Shards if their locations can be inferred from inverses related to + * the fields mentioned in the query. Ordering of the results is not + * guaranteed. + */ + selectBy: >( + this: new () => TEnt, + vc: VC, + input: SelectByInput, + ) => Promise; + + /** + * Selects the list of Ents by some predicate. + * - The query can span multiple Shards if their locations can be inferred + * from inverses related to the fields mentioned in the query. + * - In multi-Shard case, ordering of results is not guaranteed. + * - In multi-Shard case, it may return more results than requested by limit + * (basically, limit is applied to each Shard individually). The caller has + * then freedom to reorder & slice the results as they wish. + */ + select: >( + this: new () => TEnt, + vc: VC, + where: Where, + limit: number, + order?: Order, + custom?: {}, + ) => Promise; + + /** + * Same as select(), but returns data in chunks. + * - Uses multiple select() queries under the hood. + * - The query can span multiple Shards if their locations can be inferred + * from inverses related to the fields mentioned in the query. + * - Ents in each chunk always belong to the same Shard and are ordered by ID + * (there is no support for custom ordering). Make sure you have the right + * index in the database. + */ + selectChunked: >( + this: new () => TEnt, + vc: VC, + where: Where, + chunkSize: number, + limit: number, + custom?: {}, + ) => AsyncIterableIterator; + + /** + * Returns count of Ents matching a predicate. The query can span multiple + * Shards if their locations can be inferred from inverses related to the + * fields mentioned in the query. + */ + count: (vc: VC, where: CountInput) => Promise; + + /** + * A more optimal approach than count() when we basically just need to know + * whether we have "0 or not 0" rows. + */ + exists: (vc: VC, where: ExistsInput) => Promise; + + /** + * TS requires us to have a public constructor to infer instance types in + * various places. We make this constructor throw if it's called. + * + * KLUDGE here: it should've been PrimitiveInstance & Row. But + * unfortunately if we do so, TS disallows inheritance from PrimitiveClass and + * thus breaks the chain of mixins. So we add Row only at the very + * late stage, in the latest mixin in the chain and not here. + */ + new (): PrimitiveInstance; +}; + +/** + * Modifies the passed class adding support for the minimal number of basic Ent + * operations. Internally, uses Schema abstractions to run them. + */ +export function PrimitiveMixin< + TTable extends Table, + TUniqueKey extends UniqueKey, + TClient extends Client, +>( + Base: ConfigClass, +): PrimitiveClass { + class PrimitiveMixin extends Base { + declare ["constructor"]: typeof PrimitiveMixin; + declare readonly vc: VC; + + readonly [ID]: string; + + constructor() { + super(); + throw Error("Don't create Ents manually, use static loaders"); + } + + static async insertIfNotExists( + vc: VC, + input: InsertInput, + ): Promise { + const [shard] = await join([ + this.SHARD_LOCATOR.singleShardForInsert( + input, + "insert", // falls back to random Shard + ), + vc.heartbeater.heartbeat(), + ]); + + await this.VALIDATION.validateInsert(vc, input); + + if (this.TRIGGERS.hasInsertTriggers() || this.INVERSES.length > 0) { + // We have some triggers or inverses; that means we must generate an ID + // separately to let the before-triggers see it before the actual db + // operation happens. + const [id2, id2IsNewlyGenerated] = hasKey(ID, input) + ? [input[ID], false] + : [ + await shard.run( + this.SCHEMA.idGen(), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ), + true, + ]; + vc.cache(IDsCacheUpdatable).add(this, id2); // to enable privacy checks in beforeInsert triggers + + // Inverses which we're going to create. + const inverseRows = this.INVERSES.map((inverse) => ({ + inverse, + id1: input[inverse.id2Field] as string | null, + id2, + canUndoInverseInsert: id2IsNewlyGenerated, + })); + + let isInserted = false; + let allowUndoInverses = true; + let lastError: unknown = undefined; + try { + // Preliminarily insert inverse rows to inverses table, even before we + // insert the main Ent. This avoids race conditions for cases when + // multiple Clients insert and load the main Ent simultaneously: in + // terms of business logic, there is nothing too bad in having some + // "extra" inverses in the database since they're only "hints" and are + // used to resolve shard CANDIDATES. + await mapJoin(inverseRows, async (inverseRow) => { + const { inverse, id1, id2 } = inverseRow; + if (!(await inverse.beforeInsert(vc, id1, id2))) { + // We must not allow to even try undoing an Inverse creation in + // case we know that we did not create it. (Some Inverses may + // already exist beforehand, e.g. for the cases when Ent ID was + // explicitly passed during its insertion.) + inverseRow.canUndoInverseInsert = false; + } + }); + + // Insert the actual Ent. On DB error, we'll get an exception, and on + // a duplicate key violation (which is a business logic condition), + // we'll get a null returned. + return await this.TRIGGERS.wrapInsert( + async (input) => { + const id = await shard.run( + this.SCHEMA.insert(input), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + (error, _attempt) => { + // Do we know for sure whether the server applied the insert + // or not? Some examples are: "connection reset" or DB proxy + // timeout: in those cases, it's quite possible that the + // insert actually DID succeed internally (so we must NOT undo + // inverses), but we still received an error, so we must NOT + // delete inverses as a cleanup action. + if ( + !(error instanceof ClientError) || + error.kind === "unknown-server-state" + ) { + allowUndoInverses = false; + } + }, + ); + // Remember isInserted flag based on the ID received from the + // direct INSERT operation, prior to the after-triggers kick in. + // This allows to make decisions about inverse-undo in case some + // after-triggers fail. + isInserted = !!id; + return id; + }, + vc, + { ...input, [ID]: id2 }, + ); + } catch (e: unknown) { + lastError = e; + throw e; + } finally { + // There are 3 failure conditions here: + // 1. There was an exception, but we don't know the state of PG server + // (it might or might not have applied the insert). + // 2. There was an exception during the INSERT (in this case, + // isInserted will remain null due to the above initialization), + // and we received the response from PG. + // 3. An INSERT resulted in a no-op due to unique constraints + // violation (in this case, insert() will return null, and we will + // write false to isInserted). + if (!isInserted && allowUndoInverses) { + // We couldn't insert the Ent due to an unique key violation or some + // other DB error for which we know the exact PG server state. Try + // to undo the inverses creation (but if we fail to undo, it's not a + // big deal to have stale inverses in the DB since they are only + // "hints" and affect Shard candidates locating). This logic looks + // scary, but in real life, there will likely be an "inverses fixer" + // service that removes orphaned inverses asynchronously. + await mapJoin( + inverseRows, + async ({ inverse, id1, id2, canUndoInverseInsert }) => { + if (canUndoInverseInsert) { + this.CLUSTER.options.loggers.swallowedErrorLogger({ + where: + `(not an error, just a debug warning) PrimitiveMixin.insertIfNotExists(${this.name}), ` + + `undoing Inverse ${inverse.type} ${id1}->${id2}`, + error: lastError ?? Error("duplicate key on insert"), + elapsed: null, + importance: "low", + }); + await inverse.afterDelete(vc, id1, id2).catch(() => {}); + } + }, + ); + } + } + } else { + // No insert triggers and no inverses: do just a plain insert. + return shard.run( + this.SCHEMA.insert(input), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ); + } + } + + static async upsert(vc: VC, input: InsertInput): Promise { + const [shard] = await join([ + this.SHARD_LOCATOR.singleShardForInsert( + input, + "upsert", // does not fallback to random Shard + ), + vc.heartbeater.heartbeat(), + ]); + + if ( + this.TRIGGERS.hasInsertTriggers() || + this.TRIGGERS.hasUpdateTriggers() + ) { + throw new EntNotInsertableError( + this.name, + vc.toString(), + input, + "upsert cannot work with triggers defined since it doesn't know whether the row was inserted or updated in the database", + ); + } + + if (this.INVERSES.length > 0) { + throw new EntNotInsertableError( + this.name, + vc.toString(), + input, + "upsert cannot work with inverses since it doesn't know the old values of fields in the database", + ); + } + + await this.VALIDATION.validateInsert(vc, input); + + const query = this.SCHEMA.upsert(input); + const id = await shard.run( + query, + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ); + vc.cache(IDsCacheUpdatable).add(this, id); + return id; + } + + static async loadNullable( + vc: VC, + id: string, + ): Promise | null> { + const [shard] = await join([ + this.SHARD_LOCATOR.singleShardFromID(ID, id, "loadNullable"), + vc.heartbeater.heartbeat(), + ]); + if (!shard) { + return null; + } + + const query = this.SCHEMA.load(id); + const row = await shard.run( + query, + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ); + return row ? this.createEnt(vc, row) : null; + } + + static async loadByNullable( + vc: VC, + input: LoadByInput, + ): Promise | null> { + const [shards] = await join([ + this.SHARD_LOCATOR.multiShardsFromInput(vc, input, "loadBy"), + vc.heartbeater.heartbeat(), + ]); + + const rows = compact( + await mapJoin(shards, async (shard) => + shard.run( + this.SCHEMA.loadBy(input), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ), + ), + ); + const row = first(rows); + return row ? this.createEnt(vc, row) : null; + } + + static async selectBy( + vc: VC, + input: SelectByInput, + ): Promise>> { + const [shards] = await join([ + this.SHARD_LOCATOR.multiShardsFromInput(vc, input, "selectBy"), + vc.heartbeater.heartbeat(), + ]); + + const ents = await mapJoin(shards, async (shard) => { + const rows = await shard.run( + this.SCHEMA.selectBy(input), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ); + return mapJoin(rows, async (row) => this.createEnt(vc, row)); + }); + + return flatten(ents); + } + + static async select( + vc: VC, + where: Where, + limit: number, + order?: Order, + custom?: {}, + ): Promise>> { + const [shards] = await join([ + this.SHARD_LOCATOR.multiShardsFromInput(vc, where, "select"), + vc.heartbeater.heartbeat(), + ]); + + const ents = await mapJoin(shards, async (shard) => { + const rows = await shard.run( + this.SCHEMA.select({ where, limit, order, custom }), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ); + return mapJoin(rows, async (row) => this.createEnt(vc, row)); + }); + + return flatten(ents); + } + + static async *selectChunked( + vc: VC, + where: Where, + chunkSize: number, + limit: number, + custom?: {}, + ): AsyncGenerator>, void, unknown> { + const [shards] = await join([ + this.SHARD_LOCATOR.multiShardsFromInput(vc, where, "selectChunked"), + vc.heartbeater.heartbeat(), + ]); + let lastSeenID = "0"; + + while (true) { + if (limit <= 0 || shards.length === 0) { + return; + } + + if (limit < chunkSize) { + chunkSize = limit; + } + + const cursoredWhere = { + ...where, + $and: [{ [ID]: { $gt: lastSeenID } }, ...(where.$and ?? [])], + }; + + await vc.heartbeater.heartbeat(); + + const shard = shards[0]; + const rows = await shard.run( + this.SCHEMA.select({ + where: cursoredWhere, + limit: chunkSize, + order: [{ [ID]: "ASC" }], // IMPORTANT for idCursor + custom, + }), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ); + + if (rows.length > 0) { + const chunk = await mapJoin(rows, async (row) => + this.createEnt(vc, row), + ); + yield chunk; + lastSeenID = chunk[chunk.length - 1][ID]; + limit -= chunk.length; + } + + if (rows.length === 0 || rows.length < chunkSize) { + shards.shift(); + lastSeenID = "0"; + } + } + } + + static async count(vc: VC, where: CountInput): Promise { + const [shards] = await join([ + this.SHARD_LOCATOR.multiShardsFromInput(vc, where, "count"), + vc.heartbeater.heartbeat(), + ]); + + const counts = await mapJoin(shards, async (shard) => + shard.run( + this.SCHEMA.count(where), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ), + ); + + return sum(counts); + } + + static async exists(vc: VC, where: ExistsInput): Promise { + const [shards] = await join([ + this.SHARD_LOCATOR.multiShardsFromInput(vc, where, "exists"), + vc.heartbeater.heartbeat(), + ]); + + const exists = await mapJoin(shards, async (shard) => + shard.run( + this.SCHEMA.exists(where), + vc.toAnnotation(), + vc.timeline(shard, this.SCHEMA.name), + vc.freshness, + ), + ); + + return exists.some((v) => v); + } + + async updateOriginal( + inputIn: UpdateOriginalInput, + ): Promise { + const cas = inputIn.$cas; + const input = ( + cas === "skip-if-someone-else-changed-updating-ent-props" || + cas instanceof Array + ? { + ...inputIn, + $cas: Object.fromEntries( + (cas instanceof Array ? cas : Object.keys(inputIn)) + .filter((k) => !!this.constructor.SCHEMA.table[k]) + .map((k) => [k, this[k as keyof this] ?? null]), + ), + } + : inputIn + ) as UpdateInput; + + const [shard] = await join([ + this.constructor.SHARD_LOCATOR.singleShardFromID( + ID, + this[ID], + "updateOriginal", + ), + this.vc.heartbeater.heartbeat(), + ]); + + await this.constructor.VALIDATION.validateUpdate( + this.vc, + this as Row, + input, + ); + + if (!shard) { + return false; + } + + return this.constructor.TRIGGERS.wrapUpdate( + async (input) => { + const updated = await shard.run( + this.constructor.SCHEMA.update(this[ID], input), + this.vc.toAnnotation(), + this.vc.timeline(shard, this.constructor.SCHEMA.name), + this.vc.freshness, + ); + + if (updated) { + this.vc.cache(IDsCacheUpdatable).add(this.constructor, this[ID]); + await mapJoin( + this.constructor.INVERSES, + async (inverse) => + hasKey(inverse.id2Field, input) && + input[inverse.id2Field] !== undefined && + inverse.afterUpdate( + this.vc, + input[inverse.id2Field] as string | null, + this[ID], + (this as unknown as Record)[ + inverse.id2Field + ], + ), + ); + } + + return updated; + }, + this.vc, + this as TriggerUpdateOrDeleteOldRow, + input, + ); + } + + async deleteOriginal(): Promise { + const [shard] = await join([ + this.constructor.SHARD_LOCATOR.singleShardFromID( + ID, + this[ID], + "deleteOriginal", + ), + this.vc.heartbeater.heartbeat(), + ]); + + if (!this.vc.isOmni()) { + await this.constructor.VALIDATION.validateDelete( + this.vc, + this as Row, + ); + } + + if (!shard) { + return false; + } + + return this.constructor.TRIGGERS.wrapDelete( + async () => { + const deleted = await shard.run( + this.constructor.SCHEMA.delete(this[ID]), + this.vc.toAnnotation(), + this.vc.timeline(shard, this.constructor.SCHEMA.name), + this.vc.freshness, + ); + + if (deleted) { + this.vc.cache(IDsCacheUpdatable).add(this.constructor, this[ID]); + await mapJoin(this.constructor.INVERSES, async (inverse) => + inverse.afterDelete( + this.vc, + (this as unknown as Record)[ + inverse.id2Field + ], + this[ID], + ), + ); + } + + return deleted; + }, + this.vc, + this as TriggerUpdateOrDeleteOldRow, + ); + } + + /** + * Since we disabled the constructor (to not let users call it manually and + * create fake Ents), we simulate its behavior manually. This method is very + * critical to performance since the code normally loads LOTS of Ents. + */ + private static async createEnt( + vc: VC, + row: Row, + ): Promise> { + // If we've already created an Ent for this exact (row, VC, EntClass), + // return it. This covers a very frequent case when the same Ent is loaded + // multiple times concurrently from different places, so the DB query is + // coalesced into one load. We're coalescing the Ent too which saves LOTS + // of CPU (spent in this.VALIDATION otherwise) and also enables memoized + // Ent methods to work much more efficiently. + const creator = memoize2( + row, + $CACHED_ENT, + async (vc: VC, _EntCls: unknown) => { + // Try to reduce permissions and freshness for the injected VC. Also + // turn the omni VC into an user-owning VC (or a guest). For most of + // cases, this call is a no-op (we rarely upgrade/downgrade VCs). + const wasOmniVC = vc.isOmni(); + vc = await this.createLowerVC(vc, row); + + // Cloning is important here. Due to possible deduplication of exactly + // same requests, the same row object can be returned twice, while we + // request it with two different VCs. We don't want to create two Ents + // sharing same row storage if they have different VCs, so we clone. + const ent = Object.assign(Object.create(this.prototype), row); + Object.defineProperty(ent, "vc", { + value: vc, + enumerable: false, // to safely run JSON.stringify() on an Ent + writable: false, + }); + + if (!wasOmniVC) { + await this.VALIDATION.validateLoad(vc, ent); + } + + return ent; + }, + ); + const ent = await creator(vc, this); + + ent.vc.cache(IDsCacheReadable).add(this, ent[ID]); + if (vc !== ent.vc) { + vc.cache(IDsCacheReadable).add(this, ent[ID]); + } + + return ent; + } + + /** + * We never create an Ent with ent.vc = omni; instead, we lower permissions + * to either the Ent's owner (if tenantPrincipalField is used, or if it has a + * field pointing to VC) or to a guest VC. + */ + private static async createLowerVC(vc: VC, row: Row): Promise { + let newRowPrincipal: string | null; + if (vc.isOmni()) { + newRowPrincipal = null; + + if (this.VALIDATION.tenantPrincipalField) { + newRowPrincipal = (row[this.VALIDATION.tenantPrincipalField] ?? + null) as string | null; + } + + if (!newRowPrincipal) { + newRowPrincipal = (await this.VALIDATION.inferPrincipal(vc, row)) + .principal; + } + } else { + newRowPrincipal = vc.principal; + } + + return vc.toLowerInternal(newRowPrincipal); + } + } + + return PrimitiveMixin as PrimitiveClass; +} + +const $CACHED_ENT = Symbol("$CACHED_ENT"); diff --git a/src/ent/mixins/__tests__/CacheMixin.test.ts b/src/ent/mixins/__tests__/CacheMixin.test.ts new file mode 100644 index 0000000..26a79ea --- /dev/null +++ b/src/ent/mixins/__tests__/CacheMixin.test.ts @@ -0,0 +1,147 @@ +import { MASTER } from "../../../abstract/Shard"; +import type { TestPgClient } from "../../../pg/__tests__/test-utils"; +import { + recreateTestTables, + testCluster, +} from "../../../pg/__tests__/test-utils"; +import { escapeIdent } from "../../../pg/helpers/escapeIdent"; +import { PgSchema } from "../../../pg/PgSchema"; +import { createVC } from "../../__tests__/test-utils"; +import { BaseEnt } from "../../BaseEnt"; +import { True } from "../../predicates/True"; +import { AllowIf } from "../../rules/AllowIf"; +import { Require } from "../../rules/Require"; +import { GLOBAL_SHARD } from "../../ShardAffinity"; +import type { VC } from "../../VC"; +import { VCWithQueryCache } from "../../VCFlavor"; + +const entTestSchema = new PgSchema( + 'cache-mixin"test', + { + id: { type: String, autoInsert: "id_gen()" }, + k1: { type: String }, + k2: { type: String }, + }, + ["k1", "k2"], +); + +class EntTest extends BaseEnt(testCluster, entTestSchema) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + k1 text NOT NULL, + k2 text NOT NULL, + UNIQUE (k1, k2) + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [new Require(new True())], + }); + } +} + +const STATIC_ID = "42"; +const ROW = { k1: "test", k2: "test" }; + +let vc: VC; +let master: TestPgClient; + +beforeEach(async () => { + await recreateTestTables([EntTest]); + vc = createVC().withFlavor(new VCWithQueryCache({ maxQueries: 1000 })); + master = await EntTest.CLUSTER.globalShard().client(MASTER); +}); + +test.each([ + [ + "cache is cleaned on insert with id", + async (vc: VC) => EntTest.insert(vc, { id: STATIC_ID, ...ROW }), + ], + [ + "cache is cleaned on upsert with id", + async (vc: VC) => EntTest.upsert(vc, { id: STATIC_ID, ...ROW }), + ], + [ + "cache is cleaned on insert without id", + async (vc: VC) => EntTest.insert(vc, ROW), + ], + [ + "cache is cleaned on upsert without id", + async (vc: VC) => EntTest.upsert(vc, ROW), + ], + [ + "cache is cleaned on update", + async (_vc: VC, existing: EntTest) => + existing.updateOriginal(ROW).then(() => existing.id), + ], +])("%s", async (_name, mutate) => { + const existing = await EntTest.insertReturning(vc, { + ...ROW, + k1: "test-existing", + }); + + // Prewarm cache. + expect(await EntTest.loadNullable(vc, STATIC_ID)).toEqual(null); + expect(await EntTest.loadNullable(vc, existing.id)).not.toMatchObject(ROW); + expect(await EntTest.loadByNullable(vc, ROW)).toEqual(null); + expect(await EntTest.selectBy(vc, { k1: ROW.k1 })).toHaveLength(0); + expect(await EntTest.select(vc, ROW, 1)).toHaveLength(0); + expect(await EntTest.count(vc, ROW)).toEqual(0); + expect(await EntTest.exists(vc, ROW)).toEqual(false); + + const mutatedID = await mutate(vc, existing); + + // Mutation should be reflected in all read calls. + expect(await EntTest.loadNullable(vc, mutatedID)).toMatchObject(ROW); + expect(await EntTest.loadByNullable(vc, ROW)).toMatchObject(ROW); + expect(await EntTest.selectBy(vc, { k1: ROW.k1 })).toHaveLength(1); + expect(await EntTest.select(vc, ROW, 1)).toHaveLength(1); + expect(await EntTest.count(vc, ROW)).toEqual(1); + expect(await EntTest.exists(vc, ROW)).toEqual(true); + + // Direct DB modification must not affect cache. + await master.query({ + query: [ + `DELETE FROM ${escapeIdent(EntTest.SCHEMA.name)} WHERE id=?`, + mutatedID, + ], + isWrite: true, + annotations: [], + op: "", + table: EntTest.SCHEMA.name, + batchFactor: 1, + }); + + expect(await EntTest.loadNullable(vc, mutatedID)).toMatchObject(ROW); + expect(await EntTest.loadByNullable(vc, ROW)).toMatchObject(ROW); + expect(await EntTest.select(vc, ROW, 1)).toHaveLength(1); + expect(await EntTest.count(vc, ROW)).toEqual(1); + expect(await EntTest.exists(vc, ROW)).toEqual(true); +}); + +test("cache is cleaned on delete", async () => { + const existing = await EntTest.insertReturning(vc, ROW); + + // Prewarm cache. + expect(await EntTest.loadNullable(vc, existing.id)).toMatchObject(ROW); + expect(await EntTest.loadByNullable(vc, ROW)).toMatchObject(ROW); + expect(await EntTest.selectBy(vc, { k1: ROW.k1 })).toHaveLength(1); + expect(await EntTest.select(vc, ROW, 1)).toHaveLength(1); + expect(await EntTest.count(vc, ROW)).toEqual(1); + expect(await EntTest.exists(vc, ROW)).toEqual(true); + + await existing.deleteOriginal(); + + // Deletion must be reflected in all read calls. + expect(await EntTest.loadNullable(vc, existing.id)).toEqual(null); + expect(await EntTest.loadByNullable(vc, ROW)).toEqual(null); + expect(await EntTest.selectBy(vc, { k1: ROW.k1 })).toHaveLength(0); + expect(await EntTest.select(vc, ROW, 1)).toHaveLength(0); + expect(await EntTest.count(vc, ROW)).toEqual(0); + expect(await EntTest.exists(vc, ROW)).toEqual(false); +}); diff --git a/src/ent/predicates/AbstractIs.ts b/src/ent/predicates/AbstractIs.ts new file mode 100644 index 0000000..7c64190 --- /dev/null +++ b/src/ent/predicates/AbstractIs.ts @@ -0,0 +1,55 @@ +import type { Predicate } from "./Predicate"; + +/** + * A base interface for all user validation predicates. + */ +export interface AbstractIs extends Predicate { + /** The field this validation predicate is related to (null means that it + * applies to the entire Ent). */ + readonly field: string | null; + /** In case the predicate returns false or doesn't provide error messages by + * throwing EntValidationError, this message will be used. When message is + * null, it means that we expect the validator to return detailed information + * about each field errored (e.g. ValidatorStandardSchemaResult). */ + readonly message: string | null; +} + +/** + * Result of plain validators. + */ +export type ValidatorPlainResult = boolean; + +/** + * Result of Zod safeParse() calls. + */ +export type ValidatorZodSafeParseResult = + | { + readonly success: true; + readonly error?: undefined; + } + | { + readonly success: false; + readonly error: { + readonly issues: ReadonlyArray<{ + readonly message: string; + readonly path: readonly PropertyKey[]; + }>; + }; + }; + +/** + * Result of a Standard Schema validators: https://standardschema.dev/ + */ +export type ValidatorStandardSchemaResult = + | { + readonly value: unknown; + readonly issues?: undefined; + } + | { + readonly issues: Array<{ + readonly message: string; + readonly path?: + | ReadonlyArray + | undefined; + }>; + }; diff --git a/src/ent/predicates/CanDeleteOutgoingEdge.ts b/src/ent/predicates/CanDeleteOutgoingEdge.ts new file mode 100644 index 0000000..6685e15 --- /dev/null +++ b/src/ent/predicates/CanDeleteOutgoingEdge.ts @@ -0,0 +1,45 @@ +import type { EntClass } from "../types"; +import type { VC } from "../VC"; +import type { Predicate } from "./Predicate"; +import { IDsCacheDeletable } from "./Predicate"; + +/** + * Checks that an Ent available via a field can be deleted, or Ent doesn't exist + * (e.g. Ent is orphaned). See CanReadOutgoingEdge comments for more details. + */ +export class CanDeleteOutgoingEdge implements Predicate< + Record +> { + readonly name; + + constructor( + public readonly field: TField, + public readonly toEntClass: EntClass, + ) { + this.name = this.constructor.name + "(" + this.field + ")"; + } + + async check(vc: VC, row: Record): Promise { + const toID = row[this.field]; + if (!toID) { + return false; + } + + const cache = vc.cache(IDsCacheDeletable); + if (cache.has(this.toEntClass, toID)) { + return true; + } + + // Load the target Ent and check that it's deletable. + const toEnt = await this.toEntClass.loadNullable(vc, toID); + if (toEnt === null) { + return true; + } + + await this.toEntClass.VALIDATION.validateDelete(vc, toEnt); + + // Sill here and not thrown? save to the cache. + cache.add(this.toEntClass, toID); + return true; + } +} diff --git a/src/ent/predicates/CanReadOutgoingEdge.ts b/src/ent/predicates/CanReadOutgoingEdge.ts new file mode 100644 index 0000000..c9b4795 --- /dev/null +++ b/src/ent/predicates/CanReadOutgoingEdge.ts @@ -0,0 +1,46 @@ +import type { EntClass } from "../types"; +import type { VC } from "../VC"; +import type { Predicate } from "./Predicate"; +import { IDsCacheReadable } from "./Predicate"; + +/** + * Checks that an ent which a field is pointing to is readable: + * + * EntOur[company_id] ---> EntCompany[id] + * + * This predicate delegates the readability permission check for the current ent + * to another ent with ID equals to the value of our ent's field. + * + * - field = user_id in the above example + * - toEntClass = EntCompany in the above example + */ + +export class CanReadOutgoingEdge implements Predicate< + Record +> { + readonly name; + + constructor( + public readonly field: TField, + public readonly toEntClass: EntClass, + ) { + this.name = this.constructor.name + "(" + this.field + ")"; + } + + async check(vc: VC, row: Record): Promise { + const toID = row[this.field]; + if (!toID) { + return false; + } + + const cache = vc.cache(IDsCacheReadable); + if (cache.has(this.toEntClass, toID)) { + return true; + } + + await this.toEntClass.loadX(vc, toID); + // sill here and not thrown? save to the cache + cache.add(this.toEntClass, toID); + return true; + } +} diff --git a/src/ent/predicates/CanUpdateOutgoingEdge.ts b/src/ent/predicates/CanUpdateOutgoingEdge.ts new file mode 100644 index 0000000..c564f46 --- /dev/null +++ b/src/ent/predicates/CanUpdateOutgoingEdge.ts @@ -0,0 +1,46 @@ +import type { EntClass } from "../types"; +import type { VC } from "../VC"; +import type { Predicate } from "./Predicate"; +import { IDsCacheUpdatable } from "./Predicate"; + +/** + * Checks that an Ent available via a field is updatable. See + * CanReadOutgoingEdge comments for more details. + */ +export class CanUpdateOutgoingEdge implements Predicate< + Record +> { + readonly name; + + constructor( + public readonly field: TField, + public readonly toEntClass: EntClass, + ) { + this.name = this.constructor.name + "(" + this.field + ")"; + } + + async check(vc: VC, row: Record): Promise { + const toID = row[this.field]; + if (!toID) { + return false; + } + + const cache = vc.cache(IDsCacheUpdatable); + if (cache.has(this.toEntClass, toID)) { + return true; + } + + // load the target Ent and check that it's updatable + const toEnt = await this.toEntClass.loadX(vc, toID); + await this.toEntClass.VALIDATION.validateUpdate( + vc, + toEnt, + {}, + true /* privacyOnly */, + ); + + // sill here and not thrown? save to the cache + cache.add(this.toEntClass, toID); + return true; + } +} diff --git a/src/ent/predicates/FieldIs.ts b/src/ent/predicates/FieldIs.ts new file mode 100644 index 0000000..701dc28 --- /dev/null +++ b/src/ent/predicates/FieldIs.ts @@ -0,0 +1,101 @@ +import type { VC } from "../VC"; +import type { + AbstractIs, + ValidatorPlainResult, + ValidatorStandardSchemaResult, + ValidatorZodSafeParseResult, +} from "./AbstractIs"; +import { maybeThrowEntValidationError } from "./internal/maybeThrowEntValidationError"; + +/** + * A field validator function that returns a boolean. + */ +export type FieldIsValidatorPlain< + TField extends string, + TRow extends Partial>, +> = ( + fieldValue: TRow[TField], + row: TRow, + vc: VC, +) => ValidatorPlainResult | Promise; + +/** + * A field validator function that returns a Zod result. + */ +export type FieldIsValidatorZodSafeParse = ( + fieldValue: unknown, + row: TRow, + vc: VC, +) => ValidatorZodSafeParseResult | Promise; + +/** + * A field validator function that returns a Standard Schema validation result. + */ +export type FieldIsValidatorStandardSchemaV1 = ( + fieldValue: unknown, + row: TRow, + vc: VC, +) => ValidatorStandardSchemaResult | Promise; + +/** + * Checks that the validator function returns true for the value in some field. + */ +export class FieldIs< + TField extends string, + TRow extends Partial>, +> implements AbstractIs { + readonly name; + readonly field: TField; + readonly message: string | null; + readonly validator; + + /** + * Manual validator. Implies that we can trust the fieldValue TS type. + */ + constructor( + field: TField, + validator: FieldIsValidatorPlain, + message: string, + ); + + /** + * Rich validator, like Standard Schema (https://standardschema.dev) or Zod. + * No implications are made on the fieldValue type. + */ + constructor( + field: TField, + validator: + | FieldIsValidatorStandardSchemaV1 + | FieldIsValidatorZodSafeParse, + ); + + constructor( + field: TField, + validator: + | FieldIsValidatorPlain + | FieldIsValidatorZodSafeParse + | FieldIsValidatorStandardSchemaV1, + message?: string, + ) { + this.name = this.constructor.name + "(" + field + ")"; + this.field = field; + this.message = message ?? null; + this.validator = validator; + } + + /** + * Returns true if validation succeeds. Returns false if it wants the client + * to use this.message as a validation failure response. Throws an instance of + * EntValidationError when it needs to deliver the detailed error messages + * about multiple fields. + */ + async check(vc: VC, row: TRow): Promise { + const res = await this.validator(row[this.field], row, vc); + return maybeThrowEntValidationError({ + name: this.name, + field: this.field, + res, + allowRichResult: this.message === null, + }); + } +} diff --git a/src/ent/predicates/IncomingEdgeFromVCExists.ts b/src/ent/predicates/IncomingEdgeFromVCExists.ts new file mode 100644 index 0000000..3517a80 --- /dev/null +++ b/src/ent/predicates/IncomingEdgeFromVCExists.ts @@ -0,0 +1,85 @@ +import { localUniqueInt, nullthrows } from "../../internal/misc"; +import type { Row, RowWithID, Table, Where } from "../../types"; +import { ID } from "../../types"; +import type { EntClass } from "../types"; +import type { VC } from "../VC"; +import type { Predicate } from "./Predicate"; +import { IDsCacheCanReadIncomingEdge } from "./Predicate"; + +/** + * An ent may represent not necessarily a node in the graph, but also an edge + * between two nodes. Consider EntMember in the below example: + * + * vc.principal <--- EntMember[user_id, company_id] ---> EntCompany + * + * This predicate verifies that for a e.g. given EntCompany row and a given VC, + * an EntMember row exists (and optionally matches some criterion) in the + * database. + * + * - entEdgeVCField = user_id in the above example + * - entEdgeFKField = company_id in the above example + * - if an EntMember object exists, it must also match entEdgeFilter() + */ +export class IncomingEdgeFromVCExists< + TEdgeTable extends Table, +> implements Predicate { + private readonly instanceID = localUniqueInt(); + readonly name; + + constructor( + public readonly EntEdge: EntClass, + public readonly entEdgeVCField: keyof Row, + public readonly entEdgeFKField: keyof Row, + public readonly entEdgeFilter?: (ent: Row) => boolean, + ) { + this.name = + this.constructor.name + + "(" + + this.EntEdge.name + + "[" + + `${this.entEdgeVCField}=vc, ` + + `${this.entEdgeFKField}=row.${ID}` + + "]" + + ")"; + } + + async check(vc: VC, row: RowWithID): Promise { + const cache = vc.cache(IDsCacheCanReadIncomingEdge); + const cacheKey = nullthrows(row[ID]) + ":" + this.instanceID; + if (cache.has(this.EntEdge, cacheKey)) { + return true; + } + + const where = { + [this.entEdgeFKField]: row[ID], + [this.entEdgeVCField]: vc.principal, + } as Where; + + let allow: boolean; + if (this.entEdgeFilter) { + // We use an omni VC here to avoid cyclic references where the edge ent + // delegates permission checks to the row ent, and row ent loads the edge + // ent to run the edgeEntFilter function. It's safe, because: + // 1. Omni VC is always demoted to the current user's VC (which is vc + // since we filter by vc.principal above) or to a guest VC if it cannot + // find a user_id field in the edge ent (which also never happens). + // 2. The edgeEntFilter function is synchronous, so it can't physically + // access the database anyway. + const ents = await this.EntEdge.select(vc.toOmniDangerous(), where, 1); + const filtered = ents.filter((ent) => + this.entEdgeFilter!(ent as Row), + ); + allow = filtered.length > 0; + } else { + // Exists is not privacy-checked (it doesn't fetch any row to be checked). + allow = await this.EntEdge.exists(vc, where); + } + + if (allow) { + cache.add(this.EntEdge, cacheKey); + return true; + } + + return false; + } +} diff --git a/src/ent/predicates/Or.ts b/src/ent/predicates/Or.ts new file mode 100644 index 0000000..5b7b8a6 --- /dev/null +++ b/src/ent/predicates/Or.ts @@ -0,0 +1,60 @@ +import { mapJoin } from "../../internal/misc"; +import { EntAccessError } from "../errors/EntAccessError"; +import type { VC } from "../VC"; +import type { Predicate } from "./Predicate"; +import { FuncToPredicate } from "./Predicate"; + +/** + * Checks that at least one of the children predicates succeed. + */ +export class Or implements Predicate { + readonly name = this.constructor.name; + readonly predicates: ReadonlyArray>; + + constructor( + ...predicates: ReadonlyArray< + | Predicate + | ((vc: VC, input: TInput) => Promise | boolean) + > + ) { + this.predicates = predicates.map((predicate) => + predicate instanceof Function + ? new FuncToPredicate(predicate) + : predicate, + ); + } + + async check(vc: VC, input: TInput): Promise { + const errorEntNames = new Set(); + const results = await mapJoin(this.predicates, async (predicate) => { + try { + return { + predicate, + res: await predicate.check(vc, input), + }; + } catch (e: unknown) { + if (e instanceof EntAccessError) { + errorEntNames.add(e.entName); + return { predicate, res: e }; + } else { + throw e; + } + } + }); + + if (results.some(({ res }) => res === true)) { + return true; + } + + throw new EntAccessError( + [...errorEntNames].join("|"), + results + .map( + ({ predicate, res }) => + `${predicate.name}: ` + + (typeof res === "boolean" ? res : res.message), + ) + .join("\n"), + ); + } +} diff --git a/src/ent/predicates/OutgoingEdgePointsToVC.ts b/src/ent/predicates/OutgoingEdgePointsToVC.ts new file mode 100644 index 0000000..4fdbc05 --- /dev/null +++ b/src/ent/predicates/OutgoingEdgePointsToVC.ts @@ -0,0 +1,26 @@ +import type { VC } from "../VC"; +import type { Predicate } from "./Predicate"; + +/** + * Checks that the field's value is the same as VC's principal: + * + * EntOur[user_id] ---> vc.principal + */ +export class OutgoingEdgePointsToVC implements Predicate< + Record +> { + readonly name; + + constructor(public readonly field: TField) { + this.name = this.constructor.name + "(" + this.field + ")"; + } + + async check(vc: VC, row: Record): Promise { + const toID = row[this.field]; + if (!toID) { + return false; + } + + return toID === vc.principal; + } +} diff --git a/src/ent/predicates/Predicate.ts b/src/ent/predicates/Predicate.ts new file mode 100644 index 0000000..fb28c76 --- /dev/null +++ b/src/ent/predicates/Predicate.ts @@ -0,0 +1,49 @@ +import { IDsCache } from "../IDsCache"; +import type { VC } from "../VC"; + +/** + * A predicate evaluates against some input (typically a row) and returns true + * or false (or throws which is considering the similar way as returning false). + * + * I.e. Predicate is a "yes/no" logic. If it resolves to "no", then the + * framework may disallow some Ent operation and include predicate object's + * properties (like name or any other info) to the exception. + * + * Also, some predicates try to use caches in vc to make the decision faster + * based on the previously computed results. E.g. CanReadOutgoingEdge predicate + * knows that it already returned true for some ID once, it returns true again + * immediately. This saves us lots of database operations. + */ +export interface Predicate { + readonly name: string; + check(vc: VC, input: TInput): Promise; +} + +/** + * Sometimes, instead of passing a well-known predicate like OutgoingEdgePointsToVC + * or CanUpdateOutgoingEdge, we want to pass just a function which accepts a row + * and returns true or false. This class represents a Predicate which delegates + * its work to such a function. The name of the function becomes the name of the + * predicate. + */ +export class FuncToPredicate implements Predicate { + readonly name; + + constructor( + private func: (vc: VC, input: TInput) => Promise | boolean, + ) { + this.name = this.func.name || "lambda"; + } + + async check(vc: VC, input: TInput): Promise { + return this.func(vc, input); + } +} + +// Ent.ts may populate these caches too when it loads some ent from the DB, +// because often times we then load some dependent Ent. + +export class IDsCacheReadable extends IDsCache {} +export class IDsCacheUpdatable extends IDsCache {} +export class IDsCacheDeletable extends IDsCache {} +export class IDsCacheCanReadIncomingEdge extends IDsCache {} diff --git a/src/ent/predicates/RowIs.ts b/src/ent/predicates/RowIs.ts new file mode 100644 index 0000000..67d26db --- /dev/null +++ b/src/ent/predicates/RowIs.ts @@ -0,0 +1,84 @@ +import type { VC } from "../VC"; +import type { + AbstractIs, + ValidatorPlainResult, + ValidatorStandardSchemaResult, + ValidatorZodSafeParseResult, +} from "./AbstractIs"; +import { maybeThrowEntValidationError } from "./internal/maybeThrowEntValidationError"; + +/** + * A row validator function that returns a boolean. + */ +export type RowIsValidatorPlain = ( + row: TRow, + vc: VC, +) => ValidatorPlainResult | Promise; + +/** + * A row validator function that returns a Zod result. + */ +export type RowIsValidatorZodSafeParse = ( + row: TRow, + vc: VC, +) => ValidatorZodSafeParseResult | Promise; + +/** + * A row validator function that returns a Standard Schema validation result. + */ +export type RowIsValidatorStandardSchemaV1 = ( + row: TRow, + vc: VC, +) => ValidatorStandardSchemaResult | Promise; + +/** + * Checks that the validator function returns true for the entire row. + */ +export class RowIs implements AbstractIs { + readonly name; + readonly field = null; + readonly message: string | null; + readonly validator; + + /** + * Manual validator. + */ + constructor(validator: RowIsValidatorPlain, message: string); + + /** + * Rich validator, like Standard Schema (https://standardschema.dev) or Zod. + */ + constructor( + validator: + | RowIsValidatorZodSafeParse + | RowIsValidatorStandardSchemaV1, + ); + + constructor( + validator: + | RowIsValidatorPlain + | RowIsValidatorZodSafeParse + | RowIsValidatorStandardSchemaV1, + message?: string, + ) { + this.name = this.constructor.name; + this.message = message ?? null; + this.validator = validator; + } + + /** + * Returns true if validation succeeds. Returns false if it wants the client + * to use this.message as a validation failure response. Throws an instance of + * EntValidationError when it needs to deliver the detailed error messages + * about multiple fields. + */ + async check(vc: VC, row: TRow): Promise { + const res = await this.validator(row, vc); + return maybeThrowEntValidationError({ + name: this.name, + field: null, + res, + allowRichResult: this.message === null, + }); + } +} diff --git a/src/ent/predicates/True.ts b/src/ent/predicates/True.ts new file mode 100644 index 0000000..d8dd5a1 --- /dev/null +++ b/src/ent/predicates/True.ts @@ -0,0 +1,13 @@ +import type { VC } from "../VC"; +import type { Predicate } from "./Predicate"; + +/** + * Always passes; used for e.g. globally accessed objects. + */ +export class True implements Predicate { + readonly name = this.constructor.name; + + async check(_vc: VC): Promise { + return true; + } +} diff --git a/src/ent/predicates/VCHasFlavor.ts b/src/ent/predicates/VCHasFlavor.ts new file mode 100644 index 0000000..c1c2f3c --- /dev/null +++ b/src/ent/predicates/VCHasFlavor.ts @@ -0,0 +1,18 @@ +import type { VC } from "../VC"; +import type { VCFlavor } from "../VCFlavor"; +import type { Predicate } from "./Predicate"; + +/** + * Checks that the VC has some flavor. + */ +export class VCHasFlavor implements Predicate { + readonly name; + + constructor(private Flavor: new (...args: never[]) => VCFlavor) { + this.name = this.constructor.name + ":" + this.Flavor.name; + } + + async check(vc: VC): Promise { + return !!vc.flavor(this.Flavor); + } +} diff --git a/src/ent/predicates/__tests__/CanReadOutgoingEdge.test.ts b/src/ent/predicates/__tests__/CanReadOutgoingEdge.test.ts new file mode 100644 index 0000000..469b0e7 --- /dev/null +++ b/src/ent/predicates/__tests__/CanReadOutgoingEdge.test.ts @@ -0,0 +1,182 @@ +import { + recreateTestTables, + testCluster, +} from "../../../pg/__tests__/test-utils"; +import { PgSchema } from "../../../pg/PgSchema"; +import { createVC } from "../../__tests__/test-utils"; +import { BaseEnt } from "../../BaseEnt"; +import { EntNotReadableError } from "../../errors/EntNotReadableError"; +import { AllowIf } from "../../rules/AllowIf"; +import { GLOBAL_SHARD } from "../../ShardAffinity"; +import { CanReadOutgoingEdge } from "../CanReadOutgoingEdge"; +import { OutgoingEdgePointsToVC } from "../OutgoingEdgePointsToVC"; +import { True } from "../True"; + +/** + * CanReadOutgoingEdge uses global ID cache to determine if an Ent is readable. + * If the same ID exists for a different Ent with different privacy rules, then + * VC could elevate access before fixing the bug. + * + * Setup: + * * TestUser + * * TestObject (only accessible to owners) + * * TestObjectShallow (accessible to anyone + * * TestObjectSecret (only accessible to TestObject owners) + * + * By reading TestObjectShallow first, the attacker could then read + * TestObjectSecret. Now, they cannot, since IDsCache uses the Ent class + * objectId() to prevent keys collision. + */ +class EntTestUser extends BaseEnt( + testCluster, + new PgSchema( + 'ent.can-read-outgoing-edge"user', + { + id: { type: String, autoInsert: "id_gen()" }, + }, + [], + ), +) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: async (_vc, row) => row.id, + privacyLoad: [new AllowIf(new OutgoingEdgePointsToVC("id"))], + privacyInsert: [], + }); + } +} + +class EntTestObject extends BaseEnt( + testCluster, + new PgSchema( + 'ent.can-read-outgoing-edge"object', + { + id: { type: String, autoInsert: "id_gen()" }, + owner_id: { type: String }, + }, + [], + ), +) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + owner_id bigint NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [ + new AllowIf(new CanReadOutgoingEdge("owner_id", EntTestUser)), + ], + privacyInsert: [], + }); + } +} + +class EntTestObjectShallow extends BaseEnt( + testCluster, + new PgSchema( + 'ent.can-read-outgoing-edge"object', + { + id: { type: String, autoInsert: "id_gen()" }, + owner_id: { type: String }, + }, + [], + ), +) { + static readonly CREATE = []; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [new AllowIf(new True())], + privacyInsert: [], // Insert is not permitted + }); + } +} + +class EntTestObjectSecret extends BaseEnt( + testCluster, + new PgSchema( + 'ent.can-read-outgoing-edge"object_secret', + { + id: { type: String, autoInsert: "id_gen()" }, + object_id: { type: String }, + secret: { type: String }, + }, + [], + ), +) { + static readonly CREATE = [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + object_id bigint NOT NULL UNIQUE, + secret text NOT NULL + )`, + ]; + + static override configure() { + return new this.Configuration({ + shardAffinity: GLOBAL_SHARD, + privacyInferPrincipal: null, + privacyLoad: [ + new AllowIf(new CanReadOutgoingEdge("object_id", EntTestObject)), + ], + privacyInsert: [], + }); + } +} + +beforeEach(async () => { + await recreateTestTables([EntTestUser, EntTestObject, EntTestObjectSecret]); +}); + +test("Accessing the full ent fails", async () => { + const vc = createVC(); + const owner = await EntTestUser.insertReturning(vc.toOmniDangerous(), {}); + const objectID = await EntTestObject.insert(vc.toOmniDangerous(), { + owner_id: owner.id, + }); + await EntTestObjectSecret.insert(vc.toOmniDangerous(), { + object_id: objectID, + secret: "swordfish", + }); + + const attacker = await EntTestUser.insertReturning(vc.toOmniDangerous(), {}); + await expect( + EntTestObjectSecret.select(attacker.vc, { object_id: objectID }, 1), + ).rejects.toThrow(EntNotReadableError); +}); + +test("Accessing the shallow ent first and then accessing the full ent fails", async () => { + const vc = createVC(); + const owner = await EntTestUser.insertReturning(vc.toOmniDangerous(), {}); + const objectID = await EntTestObject.insert(vc.toOmniDangerous(), { + owner_id: owner.id, + }); + await EntTestObjectSecret.insert(vc.toOmniDangerous(), { + object_id: objectID, + secret: "swordfish", + }); + + const attacker = await EntTestUser.insertReturning(vc.toOmniDangerous(), {}); + await expect( + EntTestObjectShallow.loadX(attacker.vc, objectID), + ).resolves.toMatchObject({ + owner_id: owner.id, + }); + await expect( + EntTestObjectSecret.select(attacker.vc, { object_id: objectID }, 1), + ).rejects.toThrow(EntNotReadableError); +}); diff --git a/src/ent/predicates/internal/maybeThrowEntValidationError.ts b/src/ent/predicates/internal/maybeThrowEntValidationError.ts new file mode 100644 index 0000000..a197f1c --- /dev/null +++ b/src/ent/predicates/internal/maybeThrowEntValidationError.ts @@ -0,0 +1,48 @@ +import { EntValidationError } from "../../errors/EntValidationError"; +import type { + ValidatorPlainResult, + ValidatorStandardSchemaResult, + ValidatorZodSafeParseResult, +} from "../AbstractIs"; + +export function maybeThrowEntValidationError({ + name, + field, + res, + allowRichResult, +}: { + name: string; + field: string | null; + res: + | ValidatorPlainResult + | ValidatorStandardSchemaResult + | ValidatorZodSafeParseResult; + allowRichResult: boolean; +}): boolean { + if (typeof res === "boolean") { + return res; + } + + const issues = + "issues" in res ? res.issues : "error" in res ? res.error?.issues : []; + if (issues?.length) { + throw new EntValidationError( + name, + issues.map(({ message, path }) => ({ + field: + path + ?.map((part) => (typeof part === "object" ? part.key : part)) + .join(".") || field, + message, + })), + ); + } + + if (allowRichResult) { + return true; + } + + throw Error( + "BUG: validator must return true, false or one of Validator*Result values", + ); +} diff --git a/src/ent/rules/AllowIf.ts b/src/ent/rules/AllowIf.ts new file mode 100644 index 0000000..3ac1342 --- /dev/null +++ b/src/ent/rules/AllowIf.ts @@ -0,0 +1,30 @@ +import { EntAccessError } from "../errors/EntAccessError"; +import type { VC } from "../VC"; +import type { RuleResult } from "./Rule"; +import { Rule } from "./Rule"; + +/** + * Returns ALLOW if the predicate succeeds, otherwise SKIP. + * - Used mostly for read permission checks. + * - This rule may still throw an exception if the exception is a wild one (not + * derived from EntAccessError). + */ +export class AllowIf extends Rule { + declare readonly _TAG: "AllowIf"; + + async evaluate(vc: VC, input: TInput): Promise { + try { + return (await this.predicate.check(vc, input)) + ? { decision: "ALLOW", rule: this, cause: null } + : { decision: "SKIP", rule: this, cause: null }; + } catch (error: unknown) { + if (error instanceof EntAccessError) { + // We carry a cause for this SKIP decision too if it was due to an + // access-related error. + return { decision: "SKIP", rule: this, cause: error }; + } + + throw error; + } + } +} diff --git a/src/ent/rules/DenyIf.ts b/src/ent/rules/DenyIf.ts new file mode 100644 index 0000000..70bdec9 --- /dev/null +++ b/src/ent/rules/DenyIf.ts @@ -0,0 +1,32 @@ +import { EntAccessError } from "../errors/EntAccessError"; +import type { VC } from "../VC"; +import type { RuleResult } from "./Rule"; +import { Rule } from "./Rule"; + +/** + * Returns DENY if the predicate succeeds, otherwise SKIP. + * - Used mostly to early block some read/write access. + * - EntAccessError exception will be treated as a DENY signal (so it will abort + * processing immediately). + * - This rule may still throw an exception if the exception is a wild one (not + * derived from EntAccessError). + */ +export class DenyIf extends Rule { + declare readonly _TAG: "DenyIf"; + + async evaluate(vc: VC, input: TInput): Promise { + try { + return (await this.predicate.check(vc, input)) + ? { decision: "DENY", rule: this, cause: null } + : { decision: "SKIP", rule: this, cause: null }; + } catch (error: unknown) { + if (error instanceof EntAccessError) { + // We carry a cause for this DENY decision too if it was due to an + // access-related error. + return { decision: "DENY", rule: this, cause: error }; + } + + throw error; + } + } +} diff --git a/src/ent/rules/Require.ts b/src/ent/rules/Require.ts new file mode 100644 index 0000000..cc8e188 --- /dev/null +++ b/src/ent/rules/Require.ts @@ -0,0 +1,19 @@ +import type { VC } from "../VC"; +import type { RuleResult } from "./Rule"; +import { Rule } from "./Rule"; + +/** + * Returns TOLERATE if the predicate succeeds, otherwise DENY. + * - Used mostly for write permission checks. + * - This rule may still throw an exception if it's a wild one (i.e. not derived + * from EntAccessError). + */ +export class Require extends Rule { + declare readonly _TAG: "Require"; + + async evaluate(vc: VC, input: TInput): Promise { + return (await this.predicate.check(vc, input)) + ? { decision: "TOLERATE", rule: this, cause: null } + : { decision: "DENY", rule: this, cause: null }; + } +} diff --git a/src/ent/rules/Rule.ts b/src/ent/rules/Rule.ts new file mode 100644 index 0000000..1b962b1 --- /dev/null +++ b/src/ent/rules/Rule.ts @@ -0,0 +1,54 @@ +import type { EntAccessError } from "../errors/EntAccessError"; +import type { Predicate } from "../predicates/Predicate"; +import { FuncToPredicate } from "../predicates/Predicate"; +import type { VC } from "../VC"; + +/** + * Each Rule evaluates to some Decision (or throws). + */ +export type RuleDecision = "ALLOW" | "TOLERATE" | "SKIP" | "DENY"; + +/** + * A full debug info about some Rule decision (which Rule produced this + * decision, what was thrown etc.). + */ +export interface RuleResult { + decision: RuleDecision; + rule: Rule; + cause: EntAccessError | null; +} + +/** + * A base class which can e.g. accept not only a predicate, but also a plain JS + * lambda function as a predicate. Also has a logic of "glueing" the rule name + * with the predicate name. + * + * Each Rule must either: + * - throw (or return DENY) if it disallows access immediately, + * - return ALLOW if the access is granted (so no other rules will run), + * - return TOLERATE if it's okay with the row, but wants others' votes too, + * - or return SKIP to fully delegate the decision to the next rule. + * + * See more comments in rules.ts. + * + * Each rule carries a predicate which it calls and then decides, how to + * interpret the result. + */ +export abstract class Rule { + readonly predicate: Predicate; + readonly name: string; + + abstract evaluate(vc: VC, input: TInput): Promise; + + constructor( + predicate: + | Predicate + | ((vc: VC, input: TInput) => Promise | boolean), + ) { + this.predicate = + predicate instanceof Function + ? new FuncToPredicate(predicate) + : predicate; + this.name = this.constructor.name + ":" + this.predicate.name; + } +} diff --git a/src/ent/rules/evaluate.ts b/src/ent/rules/evaluate.ts new file mode 100644 index 0000000..d85b3e7 --- /dev/null +++ b/src/ent/rules/evaluate.ts @@ -0,0 +1,148 @@ +import { indent, inspectCompact, mapJoin } from "../../internal/misc"; +import { EntAccessError } from "../errors/EntAccessError"; +import type { VC } from "../VC"; +import type { Rule, RuleResult } from "./Rule"; + +/** + * This is a hearth of permissions checking, a machine which evaluates the rules + * chain from top to bottom (one after another) and makes the decision based on + * the following logic: + * - ALLOW immediately allows the chain, the rest of the rules are not checked. + * It's an eager allowance. + * - DENY immediately denies the chain, the rest of the rules are not checked. + * It's an eager denial. + * - TOLERATE delegates the decision to the next rules; if it's the last + * decision in the chain, then allows the chain. I.e. it's like an allowance, + * but only if everyone else is tolerant. + * - SKIP also delegates the decision to the next rules, but if it's the last + * rule in the chain (i.e. nothing to skip to anymore), denies the chain. I.e. + * it's "I don't vote here, please ask others". + * - An empty chain is always denied. + * + * Having TOLERATE decision may sound superfluous, but unfortunately it's not. + * The TOLERATE enables usage of the same machinery for both read-like checks + * (where we typically want ANY of the rules to be okay with the row) and for + * write-like checks (where we typically want ALL rules to be okay with the + * row). Having the same logic for everything simplifies the code. + * + * If parallel argument is true, all the rules are run at once in concurrent + * promises before the machine starts. This doesn't affect the final result, + * just speeds up processing if we know that there is a high chance that most of + * the rules will likely return TOLERATE and we'll anyway need to evaluate all + * of them (e.g. most of the rules are Require, like in write operations). As + * opposed, for read operation, there is a high chance for the first rule (which + * is often AllowIf) to succeed, so we evaluate the rules sequentially, not in + * parallel (to minimize the number of DB queries). + * + * Example of a chain (the order of rules always matters!): + * - new Require(new OutgoingEdgePointsToVC("user_id")) + * - new Require(new CanReadOutgoingEdge("post_id", EntPost)) + * + * Example of a chain: + * - new AllowIf(new OutgoingEdgePointsToVC("user_id")) + * - new AllowIf(new CanReadOutgoingEdge("post_id", EntPost)) + * + * Example of a chain: + * - new DenyIf(new UserIsPendingApproval()) + * - new AllowIf(new OutgoingEdgePointsToVC("user_id")) + */ +export async function evaluate( + vc: VC, + input: TInput, + rules: Array>, + fashion: "parallel" | "sequential", +): Promise<{ + allow: boolean; + results: RuleResult[]; + cause: string; +}> { + const results = + fashion === "parallel" + ? await mapJoin(rules, async (rule) => ruleEvaluate(rule, vc, input)) + : []; + let lastResult: RuleResult | null = null; + for (let i = 0; i < rules.length; i++) { + if (!results[i]) { + results[i] = await ruleEvaluate(rules[i], vc, input); + } + + lastResult = results[i]; + switch (lastResult.decision) { + case "ALLOW": + return { + allow: true, + results, + cause: resultsToCause(results), + }; + case "DENY": + return { + allow: false, + results, + cause: resultsToCause(results), + }; + case "TOLERATE": + case "SKIP": + continue; + default: + throw Error("BUG: weird RuleResult " + inspectCompact(lastResult)); + } + } + + const cause = resultsToCause(results); + + if (!lastResult) { + return { allow: false, results, cause }; + } + + if (lastResult.decision === "SKIP") { + return { allow: false, results, cause }; + } + + if (lastResult.decision === "TOLERATE") { + return { allow: true, results, cause }; + } + + throw Error("BUG: weird last rule result: " + inspectCompact(lastResult)); +} + +/** + * Evaluates one rule turning all EntAccessError exceptions (if they happen) + * into DENY decision (and annotating them with the rule name which caused the + * exception). All "wild" (non-EntAccessError) are thrown through. + */ +async function ruleEvaluate( + rule: Rule, + vc: VC, + input: TInput, +): Promise { + try { + return await rule.evaluate(vc, input); + } catch (error: unknown) { + if (error instanceof EntAccessError) { + // This includes e.g. derived EntValidationError class. + return { + decision: "DENY", + rule, + cause: error, + }; + } + + throw error; + } +} + +/** + * A helper function which returns a debugging text for a list of rule + * evaluation results. + */ +function resultsToCause(results: RuleResult[]): string { + return results.length === 0 + ? "No rules defined" + : results + .map( + ({ rule, decision, cause }) => + `Rule ${rule.name} returned ${decision}` + + (cause ? ", because:\n" + indent(cause.message) : ""), + ) + .join("\n"); +} diff --git a/src/ent/types.ts b/src/ent/types.ts new file mode 100644 index 0000000..bc02dd1 --- /dev/null +++ b/src/ent/types.ts @@ -0,0 +1,89 @@ +import type { Client } from "../abstract/Client"; +import type { Schema } from "../abstract/Schema"; +import type { DesperateAny } from "../internal/misc"; +import type { + CountInput, + ExistsInput, + ID, + InsertInput, + Literal, + LoadByInput, + Order, + Table, + UniqueKey, + UpdateField, + UpdateInput, + Value, + Where, +} from "../types"; +import type { ShardLocator } from "./ShardLocator"; +import type { Validation } from "./Validation"; +import type { VC } from "./VC"; + +/** + * A very shallow interface of Ent class (as a collection of static methods). + * Used in some places where we need the very minimum from the Ent. + * @interface + */ +export type EntClass = { + readonly SCHEMA: Schema; + readonly VALIDATION: Validation; + readonly SHARD_LOCATOR: ShardLocator; + readonly name: string; // class constructor name + + new (): Ent; + loadX(vc: VC, id: string): Promise>; + loadNullable(vc: VC, id: string): Promise | null>; + loadIfReadableNullable(vc: VC, id: string): Promise | null>; + count(vc: VC, where: CountInput): Promise; + exists(vc: VC, where: ExistsInput): Promise; + select( + vc: VC, + where: Where, + limit: number, + order?: Order, + ): Promise>>; + selectChunked( + vc: VC, + where: Where, + chunkSize: number, + limit: number, + custom?: {}, + ): AsyncIterableIterator>>; + loadByX( + vc: VC, + keys: LoadByInput>, + ): Promise>; + loadByNullable( + vc: VC, + input: LoadByInput>, + ): Promise | null>; + insert(vc: VC, input: InsertInput): Promise; + upsert(vc: VC, input: InsertInput): Promise; +}; + +/** + * A very shallow interface of one Ent. + * @interface + */ +export type Ent = { + readonly [ID]: string; + readonly vc: VC; + deleteOriginal(): Promise; + updateOriginal(input: UpdateOriginalInput): Promise; +}; + +/** + * The input of updateOriginal() method. It supports some additional syntax + * sugar for $cas property, so to work-around TS weakness of Omit<> & type + * inference, we redefine this type from scratch. + */ +export type UpdateOriginalInput = { + [K in UpdateField]?: Value; +} & { + $literal?: Literal; + $cas?: + | "skip-if-someone-else-changed-updating-ent-props" + | ReadonlyArray> + | UpdateInput["$cas"]; +}; diff --git a/src/helpers/index.ts b/src/helpers/index.ts new file mode 100644 index 0000000..fe91e1e --- /dev/null +++ b/src/helpers/index.ts @@ -0,0 +1,6 @@ +/** + * @file Automatically generated by barrelsby. + */ + +export * from "./isBigintStr"; +export * from "./testSpecTypeIntegrity"; diff --git a/src/helpers/isBigintStr.ts b/src/helpers/isBigintStr.ts new file mode 100644 index 0000000..bbca1c9 --- /dev/null +++ b/src/helpers/isBigintStr.ts @@ -0,0 +1,13 @@ +const MAX_BIGINT = "9223372036854775807"; +const MAX_BIGINT_RE = new RegExp("^\\d{1," + MAX_BIGINT.length + "}$"); + +/** + * It's hard to support PG bigint type in JS, so people use strings instead. + * This function checks that a string can be passed to PG as a bigint. + */ +export function isBigintStr(str: string): boolean { + return ( + !!str.match(MAX_BIGINT_RE) && + (str.length < MAX_BIGINT.length || str <= MAX_BIGINT) + ); +} diff --git a/src/helpers/testSpecTypeIntegrity.ts b/src/helpers/testSpecTypeIntegrity.ts new file mode 100644 index 0000000..6a0a0cb --- /dev/null +++ b/src/helpers/testSpecTypeIntegrity.ts @@ -0,0 +1,46 @@ +import assert, { AssertionError } from "assert"; + +/** + * A tool to verify integrity of custom field types. It is meant to be called + * from Jest expect(). The helper runs dbValueToJs, stringify and parse methods + * on the type and makes sure that parse() is the opposite of stringify(). The + * returned object can then be compared against a Jest snapshot. + */ +export function testSpecTypeIntegrity( + SpecType: { + dbValueToJs: (dbValue: TDBValue) => TJsValue; + stringify: (jsValue: TJsValue) => string; + parse: (str: string) => TJsValue; + }, + dbValue: TDBValue, +): { + jsValueDecoded: TJsValue; + stringifiedBack: string; +} { + const jsValue = SpecType.dbValueToJs(dbValue); + const str = SpecType.stringify(jsValue); + const parsed = SpecType.parse(str); + + const error1 = new AssertionError({ + actual: str, + expected: SpecType.stringify(parsed), + }); + assert( + JSON.stringify(error1.actual) === JSON.stringify(error1.expected), + error1, + ); + + const error2 = new AssertionError({ + actual: jsValue, + expected: parsed, + }); + assert( + JSON.stringify(error2.actual) === JSON.stringify(error2.expected), + error2, + ); + + return { + jsValueDecoded: jsValue, + stringifiedBack: typeof str === "string" ? str.replace(/\n/g, "^n") : str, + }; +} diff --git a/src/internal/CachedRefreshedValue.ts b/src/internal/CachedRefreshedValue.ts new file mode 100644 index 0000000..e2e02cc --- /dev/null +++ b/src/internal/CachedRefreshedValue.ts @@ -0,0 +1,203 @@ +import { Memoize } from "fast-typescript-memoize"; +import type { DeferredPromise } from "p-defer"; +import pDefer from "p-defer"; +import type { MaybeCallable } from "./misc"; +import { maybeCall, runInVoid } from "./misc"; + +export interface CachedRefreshedValueOptions { + /** Delay between calling resolver. */ + delayMs: MaybeCallable; + /** Log a timeout Error if a resolver takes more than X ms to complete. */ + warningTimeoutMs: MaybeCallable; + /** The handler deps.handler() is called every deps.delayMs (typically, + * frequently); if it returns a different value than previously (using "===" + * comparison), then waiting for the next delayMs is interrupted prematurely, + * and the value gets refreshed. This allows to frequently recheck for some + * configuration changes and act accordingly. */ + deps: { + delayMs: MaybeCallable; + handler: () => unknown | Promise; + }; + /** The name of the resolverFn function. Used in error messages. */ + resolverName?: string; + /** A resolver function that returns the value. It's assumed that this + * function would eventually either resolve or throw. */ + resolverFn: () => Promise; + /** An error handler. */ + onError: (error: unknown, elapsed: number) => void; + /** A custom delay implementation. */ + delay: (ms: number) => Promise; +} + +/** + * Utility class to provide a caching layer for a resolverFn with the following + * assumptions: + * - The value is stable and does not change frequently. + * - The resolverFn can throw or take more time to resolve (e.g. outage). In + * that case, the cached value is still valid, unless a fresh value is + * requested with refreshAndWait(). + * + * The implementation is as follows: + * - Once value is accessed, we schedule an endless loop of calling resolver to + * get latest value. + * - The result is cached, so next calls will return it immediately in most of + * the cases. + * - Once every delayMs we call resolverFn to get latest value. All calls during + * this time will get previous value (if available). + */ +export class CachedRefreshedValue { + /** Latest value pulled from the cache. */ + private latestValue: TValue | null = null; + /** Deferred promise containing the next value. Fulfilled promises are + * replaced right away. */ + private nextValue: DeferredPromise = pDefer(); + /** Each time before resolverFn() is called, this value is increased. */ + private resolverFnCallCount: number = 0; + /** Whether the instance is destroyed or not. Used to prevent memory leaks in + * unit tests. */ + private destroyedError: Error | null = null; + /** A callback to skip the current delay() call. */ + private skipDelay: (() => void) | null = null; + + /** + * Initializes the instance. + */ + constructor(public readonly options: CachedRefreshedValueOptions) {} + + /** + * Returns latest cached value. If the value has been calculated at least + * once, then it is guaranteed that in the worst case, it will be returned + * immediately. I.e. this method never blocks once at least one calculation + * succeeded in the past. (But it may block during the very first call.) + */ + async cached(): Promise { + runInVoid(this.refreshLoop()); + return this.latestValue ?? this.nextValue.promise; + } + + /** + * Triggers the call to resolverFn() ASAP (i.e. sooner than the next interval + * specified in delayMs) and waits for the next complete SUCCESSFUL cache + * refresh. If the method is called during the period of time when the + * resolverFn() is already running, then it waits till it finishes, and then + * waits again till the next resolverFn() call finishes, so we're sure that + * it's a strong barrier. + */ + async refreshAndWait(): Promise { + runInVoid(this.refreshLoop()); + // To unfreeze, we want a completely new resolverFn() call to finish within + // refreshLoop(). I.e. the call to resolverFn() must start strictly AFTER we + // entered refreshAndWait(); thus, `while` loop below may spin twice. + const startCallCount = this.resolverFnCallCount; + while (this.resolverFnCallCount <= startCallCount) { + // Skip waiting between loops. + this.skipDelay?.(); + // After await resolves here, it's guaranteed that this.nextValue will be + // reassigned with a new pDefer, and this.latestValue will be updated (see + // refreshLoop() body). + await this.nextValue.promise; + } + } + + /** + * Destroys the instance. Stops refreshing the value and any call to it will + * result in an error. + */ + destroy(): void { + this.destroyedError = Error( + `${this.constructor.name}: This instance is destroyed`, + ); + } + + @Memoize() + private async refreshLoop(): Promise { + while (!this.destroyedError) { + const warningDelayMs = maybeCall(this.options.warningTimeoutMs); + const depsDelayMs = maybeCall(this.options.deps.delayMs); + const startTime = performance.now(); + + const warningTimeout = setTimeout( + () => + this.onError( + Error( + `${this.constructor.name}.refreshLoop: Warning: ` + + `${this.options.resolverName ?? "resolverFn"}() did not complete in ${warningDelayMs} ms!`, + ), + Math.round(performance.now() - startTime), + ), + warningDelayMs, + ).unref(); + + let depsPrev: unknown = undefined; + try { + this.resolverFnCallCount++; + depsPrev = await this.options.deps.handler(); + this.latestValue = await this.options.resolverFn(); + const oldNextValue = this.nextValue; + this.nextValue = pDefer(); + oldNextValue.resolve(this.latestValue); + } catch (e: unknown) { + this.onError(e, Math.round(performance.now() - startTime)); + } finally { + clearTimeout(warningTimeout); + } + + // Wait for delayMs. If this.skipDelay() is called, the code unfreezes + // immediately. Also, deps are rechecked every depsDelayMs, and if they + // change, the code unfreezes too. + const delayDefer = pDefer(); + this.skipDelay = () => delayDefer.resolve(); + + let depsTimeoutBody: null | (() => void) = () => + runInVoid(async () => { + try { + const depsCurr = await this.options.deps.handler(); + if (depsCurr !== depsPrev) { + delayDefer.resolve(); + } + } catch (e: unknown) { + this.onError(e, Math.round(performance.now() - startTime)); + } finally { + if (depsTimeoutBody) { + depsTimeout = setTimeout(depsTimeoutBody, depsDelayMs).unref(); + } + } + }); + + let depsTimeout = + depsPrev !== undefined + ? setTimeout(depsTimeoutBody, depsDelayMs).unref() + : undefined; + try { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.options + .delay(maybeCall(this.options.delayMs)) + .finally(() => delayDefer.resolve()); + await delayDefer.promise; + } finally { + depsTimeoutBody = null; + clearTimeout(depsTimeout); + } + } + + // Mark current instance as destroyed. + this.latestValue = null; + runInVoid( + this.nextValue.promise.catch(() => { + // Stops unhandled promise rejection errors. + }), + ); + this.nextValue.reject(this.destroyedError); + } + + /** + * A never throwing version of options.onError(). + */ + private onError(...args: Parameters<(typeof this.options)["onError"]>): void { + try { + this.options.onError(...args); + } catch { + // noop + } + } +} diff --git a/src/internal/DefaultMap.ts b/src/internal/DefaultMap.ts new file mode 100644 index 0000000..9052d28 --- /dev/null +++ b/src/internal/DefaultMap.ts @@ -0,0 +1,11 @@ +export class DefaultMap extends Map { + getOrAdd(k: K, DefConstructor: new () => V): V { + if (!this.has(k)) { + const def = new DefConstructor(); + this.set(k, def); + return def; + } + + return this.get(k)!; + } +} diff --git a/src/internal/Ref.ts b/src/internal/Ref.ts new file mode 100644 index 0000000..ebc60a0 --- /dev/null +++ b/src/internal/Ref.ts @@ -0,0 +1,6 @@ +/** + * See PgClient#withShard() for reasons of having this class. + */ +export class Ref { + constructor(public current: T) {} +} diff --git a/src/internal/Registry.ts b/src/internal/Registry.ts new file mode 100644 index 0000000..c67dc5a --- /dev/null +++ b/src/internal/Registry.ts @@ -0,0 +1,50 @@ +import { mapJoin } from "./misc"; + +/** + * Represents a container of TObj's that can be created in the container from + * the TInit data. If the object corresponding to a particular init data already + * exists, it's returned instead of being created. + */ +export class Registry { + private map = new Map(); + + constructor( + private options: { + /** A handler which computes an unique key from the init data provided. */ + key: (init: TInit) => string; + /** A handler which creates a new object from its init data. */ + create: (init: TInit) => TObj; + /** This handler is called when an object is deleted from the registry. */ + end?: (obj: TObj, key: string, init: TInit) => Promise; + }, + ) {} + + /** + * Computes the key for the init data and returns the object corresponding to + * that key if it already exists in the registry. Otherwise, creates a new + * object, adds it to the registry and returns it. + */ + getOrCreate(init: TInit): [obj: TObj, key: string] { + const key = this.options.key(init); + let obj = this.map.get(key)?.obj; + if (!obj) { + obj = this.options.create(init); + this.map.set(key, { obj, key, init }); + } + + return [obj, key]; + } + + /** + * Deletes all objects from the registry except those whose keys are in the + * keepKeys set. For each object, calls an optional end() handler. + */ + async deleteExcept(keepKeys: Set): Promise { + await mapJoin([...this.map], async ([key, { obj, init }]) => { + if (!keepKeys.has(key)) { + this.map.delete(key); + await this.options.end?.(obj, key, init); + } + }); + } +} diff --git a/src/internal/WeakTicker.ts b/src/internal/WeakTicker.ts new file mode 100644 index 0000000..c6b3ea0 --- /dev/null +++ b/src/internal/WeakTicker.ts @@ -0,0 +1,89 @@ +export interface WeakTickerTarget { + onTick(tickNo: number, tickMs: number): "keep" | "unschedule"; +} + +/** + * A perf efficient approximate scheduler which doesn't retain the scheduled + * objects in memory, so they remain subject for GC. + */ +export class WeakTicker { + private slots = new Map< + number, + { + refs: Set>; + nextTickNos: WeakMap; + interval: NodeJS.Timeout; + } + >(); + + /** + * Schedules a call to target.onTick() to be called periodically, every tickMs + * approximately. + * - The target scheduled will NOT be retained from GC. This is the main + * difference with setInterval() and the reason why we accept an object, not + * a closure. + * - The 1st call to target.onTick() will happen between 0 and tickMs from + * now: this is the second difference from setInterval(). Then, next calls + * will follow. The current tick number is passed to onTick(). + * - If the same target is scheduled again, its tick number will be reset to + * 0, as if it's scheduled the very 1st time. The 2nd scheduling is cheap + * (just 2 map lookups), so an object can be rescheduled-over as many times + * as needed. + * - If target.onTick() returns "unschedule", the target will be unscheduled. + */ + schedule(target: WeakTickerTarget, tickMs: number): void { + // We DO NOT use any closures here! Otherwise, target would be retained in + // that closures, and it won't be garbage collected. + let slot = this.slots.get(tickMs); + if (!slot) { + slot = { + refs: new Set(), + nextTickNos: new WeakMap(), + interval: setInterval(this.onTick.bind(this, tickMs), tickMs).unref(), + }; + this.slots.set(tickMs, slot); + } + + if (!slot.nextTickNos.has(target)) { + const ref = new WeakRef(target); + slot.refs.add(ref); + } + + slot.nextTickNos.set(target, 0); + } + + /** + * Returns true if there are no targets scheduled at the moment. + */ + isEmpty(): boolean { + return this.slots.size === 0; + } + + /** + * Called by internal setInterval(). + */ + private onTick(tickMs: number): void { + const slot = this.slots.get(tickMs)!; + for (const ref of slot.refs) { + const target = ref.deref(); + if (target) { + const nextTickNo = slot.nextTickNos.get(target)!; + if (target.onTick(nextTickNo, tickMs) === "unschedule") { + slot.refs.delete(ref); + slot.nextTickNos.delete(target); + } + + slot.nextTickNos.set(target, nextTickNo + 1); + } else { + slot.refs.delete(ref); + // Target was garbage collected, which means it is already auto-removed + // from slot.nextTickNos WeakMap too. + } + } + + if (slot.refs.size === 0) { + clearInterval(slot.interval); + this.slots.delete(tickMs); + } + } +} diff --git a/src/internal/__tests__/CachedRefreshedValue.test.ts b/src/internal/__tests__/CachedRefreshedValue.test.ts new file mode 100644 index 0000000..9889db3 --- /dev/null +++ b/src/internal/__tests__/CachedRefreshedValue.test.ts @@ -0,0 +1,331 @@ +import delay from "delay"; +import pDefer from "p-defer"; +import waitForExpect from "wait-for-expect"; +import { CachedRefreshedValue } from "../CachedRefreshedValue"; + +jest.useFakeTimers({ advanceTimers: true }); + +const OPTIONS = { + delayMs: 10, + deps: { + delayMs: 100, + handler: () => "", + }, + warningTimeoutMs: 1000, + onError: () => {}, + delay: async (ms: number) => delay(ms), +}; + +// Ensures that the instance is destroyed after each test. +let cache: CachedRefreshedValue; +afterEach(() => { + cache?.destroy(); +}); + +test("first value", async () => { + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: jest + .fn() + .mockReturnValueOnce("first") + .mockReturnValueOnce("second") + .mockReturnValueOnce("third"), + }); + expect(await cache.cached()).toBe("first"); +}); + +test("latest value", async () => { + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: jest + .fn() + .mockReturnValueOnce("first") + .mockReturnValueOnce("second") + .mockReturnValueOnce("third") + .mockReturnValue("latest value"), + }); + for (let i = 0; i < 3; i++) { + await cache.cached(); + await delay(30); + } + + expect(await cache.cached()).toBe("latest value"); +}); + +test("error swallowing", async () => { + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: jest + .fn() + .mockRejectedValueOnce(Error("Error")) + .mockReturnValue("after error"), + }); + expect(await cache.cached()).toBe("after error"); +}); + +test("error handler", async () => { + const err = Error("Error"); + const onError = jest.fn(); + cache = new CachedRefreshedValue({ + ...OPTIONS, + onError, + resolverFn: jest.fn().mockRejectedValueOnce(err).mockReturnValue("first"), + }); + await cache.cached(); + expect(onError).toBeCalledWith(err, expect.any(Number)); +}); + +test("eventual consistency", async () => { + const slowFn = jest.fn(); + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: jest + .fn() + .mockImplementationOnce(async () => { + await delay(2000); // longer than delayMs + slowFn(); + return "timed out value"; + }) + .mockReturnValue("first"), + }); + await cache.cached(); + await delay(1000); + expect(await cache.cached()).toBe("first"); + expect(slowFn).toBeCalled(); +}); + +test("timeout warning", async () => { + const deferred = pDefer(); + const onError = jest.fn(); + cache = new CachedRefreshedValue({ + ...OPTIONS, + onError, + warningTimeoutMs: 100, + resolverFn: async () => deferred.promise, + }); + const promise = cache.cached(); + await delay(200); + deferred.resolve("first"); + await promise; + expect(onError.mock.lastCall[0]).toMatchInlineSnapshot( + "[Error: CachedRefreshedValue.refreshLoop: Warning: resolverFn() did not complete in 100 ms!]", + ); +}); + +test("custom delay handler", async () => { + const delayFn = jest.fn().mockImplementation(async (ms: number) => delay(ms)); + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: jest.fn().mockReturnValue("foo"), + delay: delayFn, + delayMs: 10, + }); + await cache.cached(); + expect(delayFn).toBeCalledWith(10); +}); + +test("throwing in onError during timeout", async () => { + const deferred = pDefer(); + const onError = jest.fn().mockImplementation(() => { + throw Error("Error in onError"); + }); + cache = new CachedRefreshedValue({ + ...OPTIONS, + onError, + warningTimeoutMs: 100, + resolverFn: async () => deferred.promise, + }); + const promise = cache.cached(); + await delay(200); + deferred.resolve("first"); + await promise; + expect(onError.mock.lastCall[0]).toMatchInlineSnapshot( + "[Error: CachedRefreshedValue.refreshLoop: Warning: resolverFn() did not complete in 100 ms!]", + ); +}); + +test("throwing in onError during error", async () => { + const onError = jest.fn().mockImplementation(() => { + throw Error("Error in onError"); + }); + cache = new CachedRefreshedValue({ + ...OPTIONS, + onError, + resolverFn: jest + .fn() + .mockRejectedValueOnce(Error("Errored in resolverFn")) + .mockResolvedValueOnce("value"), + }); + await cache.cached(); + expect(onError.mock.lastCall[0]).toMatchInlineSnapshot( + "[Error: Errored in resolverFn]", + ); +}); + +test("refreshAndWait() multiple times", async () => { + let i = 0; + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: jest.fn().mockImplementation(async () => { + await delay(500); + return `delayed ${i++}`; + }), + }); + await cache.refreshAndWait(); + expect(await cache.cached()).toBe("delayed 1"); + await cache.refreshAndWait(); + expect(await cache.cached()).toBe("delayed 2"); + await cache.refreshAndWait(); + expect(await cache.cached()).toBe("delayed 3"); +}); + +test("destroy before cached()", async () => { + cache = new CachedRefreshedValue({ ...OPTIONS, resolverFn: jest.fn() }); + cache.destroy(); + await expect(cache.cached()).rejects.toMatchInlineSnapshot( + "[Error: CachedRefreshedValue: This instance is destroyed]", + ); +}); + +test("destroy before refreshAndWait()", async () => { + cache = new CachedRefreshedValue({ ...OPTIONS, resolverFn: jest.fn() }); + cache.destroy(); + await expect(cache.refreshAndWait()).rejects.toMatchInlineSnapshot( + "[Error: CachedRefreshedValue: This instance is destroyed]", + ); +}); + +test("destroy after cached()", async () => { + cache = new CachedRefreshedValue({ ...OPTIONS, resolverFn: jest.fn() }); + await cache.cached(); + cache.destroy(); + await expect(cache.cached()).rejects.toMatchInlineSnapshot( + "[Error: CachedRefreshedValue: This instance is destroyed]", + ); +}); + +test("destroy after refreshAndWait()", async () => { + cache = new CachedRefreshedValue({ ...OPTIONS, resolverFn: jest.fn() }); + await cache.refreshAndWait(); + cache.destroy(); + await expect(cache.refreshAndWait()).rejects.toMatchInlineSnapshot( + "[Error: CachedRefreshedValue: This instance is destroyed]", + ); +}); + +test("destroy stops resolverFn() calls", async () => { + const resolverFn = jest.fn().mockResolvedValue("foo"); + cache = new CachedRefreshedValue({ ...OPTIONS, resolverFn }); + await cache.cached(); + cache.destroy(); + await delay(OPTIONS.delayMs * 5); + expect(resolverFn).toBeCalledTimes(1); +}); + +test("refreshAndWait() returns fresh value", async () => { + let resolveDeferred = pDefer(); + let delayDeferred = pDefer(); + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: async () => { + const val = await resolveDeferred.promise; + resolveDeferred = pDefer(); + return val; + }, + delay: async () => { + await delayDeferred.promise; + delayDeferred = pDefer(); + }, + }); + + resolveDeferred.resolve("init"); + await cache.cached(); + + await delay(1000); + + const freshValue = cache.refreshAndWait().then(async () => cache.cached()); + delayDeferred.resolve(); + resolveDeferred.resolve("fresh"); + expect(await freshValue).toBe("fresh"); +}); + +test("refreshAndWait() skips in-flight value", async () => { + let resolveCalled = pDefer(); + let resolveDeferred = pDefer(); + let delayDeferred = pDefer(); + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: async () => { + resolveCalled.resolve(); + resolveCalled = pDefer(); + const val = await resolveDeferred.promise; + resolveDeferred = pDefer(); + return val; + }, + delay: async () => { + await delayDeferred.promise; + delayDeferred = pDefer(); + }, + }); + + resolveDeferred.resolve("init"); + await cache.cached(); + + delayDeferred.resolve(); + await resolveCalled.promise; + const freshValue = cache.refreshAndWait().then(async () => cache.cached()); + + resolveDeferred.resolve("in-flight"); + delayDeferred.resolve(); + + await resolveCalled.promise; + resolveDeferred.resolve("fresh"); + + expect(await freshValue).toBe("fresh"); +}); + +test("refreshAndWait() skips delay", async () => { + cache = new CachedRefreshedValue({ + ...OPTIONS, + resolverFn: jest + .fn() + .mockResolvedValueOnce("one") + .mockResolvedValueOnce("two") + .mockResolvedValueOnce("three") + .mockResolvedValueOnce("everything else"), + delay: async () => + // Never resolves! + delay(1_000_000), + }); + + await cache.cached(); + + await cache.refreshAndWait(); + expect(await cache.cached()).toBe("two"); + + await cache.refreshAndWait(); + expect(await cache.cached()).toBe("three"); +}, 10_000 /* timeout */); + +test("changes in deps are respected", async () => { + let depsValue = "some"; + cache = new CachedRefreshedValue({ + ...OPTIONS, + deps: { + delayMs: 10, + handler: () => depsValue, + }, + resolverFn: jest + .fn() + .mockResolvedValueOnce("one") + .mockResolvedValueOnce("two"), + delay: async () => + // Never resolves! + delay(1_000_000), + }); + + expect(await cache.cached()).toEqual("one"); + + depsValue = "other"; + await waitForExpect(async () => expect(await cache.cached()).toEqual("two")); +}); diff --git a/src/internal/__tests__/Registry.test.ts b/src/internal/__tests__/Registry.test.ts new file mode 100644 index 0000000..8b32d60 --- /dev/null +++ b/src/internal/__tests__/Registry.test.ts @@ -0,0 +1,25 @@ +import { Registry } from "../Registry"; + +test("creates and deletes objects", async () => { + const registry = new Registry({ + key: (number) => number.toString(), + create: (number) => ({ id: number }), + end: async (obj) => (obj.id = null), + }); + + const [obj1, key1] = registry.getOrCreate(1); + const [obj1same, key1same] = registry.getOrCreate(1); + expect(obj1).toBe(obj1same); + expect(key1).toBe(key1same); + + const [obj2, key2] = registry.getOrCreate(2); + expect(obj1).not.toBe(obj2); + + await registry.deleteExcept(new Set([key2])); + expect(obj1.id).toBeNull(); + const [obj2same] = registry.getOrCreate(2); + expect(obj2).toBe(obj2same); + + const [obj1new] = registry.getOrCreate(1); + expect(obj1new).not.toBe(obj1); +}); diff --git a/src/internal/__tests__/WeakTicker.test.ts b/src/internal/__tests__/WeakTicker.test.ts new file mode 100644 index 0000000..6c37d64 --- /dev/null +++ b/src/internal/__tests__/WeakTicker.test.ts @@ -0,0 +1,50 @@ +import delay from "delay"; +import waitForExpect from "wait-for-expect"; +import { WeakTicker } from "../WeakTicker"; + +class Target { + constructor( + private ticksOut: number[], + public maxTicks: number, + ) {} + + onTick(tickNo: number): "keep" | "unschedule" { + this.ticksOut.push(tickNo); + return tickNo < this.maxTicks - 1 ? "keep" : "unschedule"; + } +} + +test("schedules a tick", async () => { + const weakTicker = new WeakTicker(); + const ticks: number[] = []; + const target = new Target(ticks, 2); + weakTicker.schedule(target, 10); + await waitForExpect(() => expect(ticks).toEqual([0, 1])); + expect(weakTicker.isEmpty()).toBeTruthy(); +}); + +test("rescheduling clears tick number", async () => { + const weakTicker = new WeakTicker(); + const ticks: number[] = []; + const target = new Target(ticks, Number.MAX_SAFE_INTEGER); + weakTicker.schedule(target, 10); + await waitForExpect(() => expect(ticks.length).toBeGreaterThan(5)); + weakTicker.schedule(target, 10); + await waitForExpect(() => + expect(ticks.filter((v) => v === 0)).toEqual([0, 0]), + ); +}); + +(typeof global.gc === "function" ? test : test.skip)( + "does not retain target", + async () => { + const weakTicker = new WeakTicker(); + const ticks: number[] = []; + let target: Target | null = new Target(ticks, Number.MAX_SAFE_INTEGER); + weakTicker.schedule(target, 10); + target = null; + await delay(10); + global.gc!(); + await waitForExpect(() => expect(weakTicker.isEmpty()).toBeTruthy()); + }, +); diff --git a/src/internal/__tests__/__snapshots__/misc.test.ts.snap b/src/internal/__tests__/__snapshots__/misc.test.ts.snap new file mode 100644 index 0000000..33f8c2d --- /dev/null +++ b/src/internal/__tests__/__snapshots__/misc.test.ts.snap @@ -0,0 +1,39 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`appendCause - merge stacks 1`] = ` +"PgError: there is no unique or exclusion constraint matching the ON CONFLICT specification (pg_error) + at TestPgClient.query (/Users/user/source/sd/packages/ent-framework/src/pg/PgClient.ts:662:15) + at processTicksAndRejections (node:internal/process/task_queues:95:5) + at PgRunnerUpsert.clientQuery (/Users/user/source/sd/packages/ent-framework/src/pg/PgRunner.ts:72:18) + at PgRunnerUpsert.runBatch (/Users/user/source/sd/packages/ent-framework/src/pg/PgQueryUpsert.ts:113:18) + at Batcher.flushQueue (/Users/user/source/sd/packages/ent-framework/src/abstract/Batcher.ts:54:19) + at /Users/user/source/sd/packages/ent-framework/src/abstract/Batcher.ts:151:15 + at processTicksAndRejections (node:internal/process/task_queues:95:5) + at /Users/user/source/sd/packages/ent-framework/src/abstract/Shard.ts:78:21 + at Cluster.runOnShard (/Users/user/source/sd/packages/ent-framework/src/abstract/Cluster.ts:461:16) + at Function.upsert (/Users/user/source/sd/packages/ent-framework/src/ent/mixins/PrimitiveMixin.ts:397:18) + at async Promise.all (index 0) + at join (/Users/user/source/sd/packages/ent-framework/src/internal/misc.ts:142:24) + at Object. (/Users/user/source/sd/packages/ent-framework/src/ent/__tests__/Ent.errors.test.ts:48:5) +Cause: error: there is no unique or exclusion constraint matching the ON CONFLICT specification + at Parser.parseErrorMessage (/Users/user/source/sd/node_modules/pg-protocol/src/parser.ts:369:69) + at Parser.handlePacket (/Users/user/source/sd/node_modules/pg-protocol/src/parser.ts:188:21) + at Parser.parse (/Users/user/source/sd/node_modules/pg-protocol/src/parser.ts:103:30) + at Socket. (/Users/user/source/sd/node_modules/pg-protocol/src/index.ts:7:48) + at Socket.emit (node:events:519:28) + at addChunk (node:internal/streams/readable:559:12) + at readableAddChunkPushByteMode (node:internal/streams/readable:510:3) + at Socket.Readable.push (node:internal/streams/readable:390:5) + at TCP.onStreamRead (node:internal/stream_base_commons:191:23)" +`; + +exports[`minifyStack 1`] = ` +"at Ent.ts:525 (Function.loadNullable) +at Ent.ts:563 (Function.loadX) +at EntSome.ts:45 (EntSome.asset2) +at Memoize.ts:75 (EntSome.) +at AsanaDriver.ts:256 +at promise.ts:55 +at promise.ts:55 (Object.mapJoin) +at AsanaDriver.ts:255 (loadProjects)" +`; diff --git a/src/internal/__tests__/deepEqual.test.ts b/src/internal/__tests__/deepEqual.test.ts new file mode 100644 index 0000000..49f06d7 --- /dev/null +++ b/src/internal/__tests__/deepEqual.test.ts @@ -0,0 +1,437 @@ +import { deepEqual } from "../deepEqual"; + +test("equal numbers", () => expect(deepEqual(1, 1)).toBe(true)); + +test("not equal numbers", () => expect(deepEqual(1, 2)).toBe(false)); + +test("number and array are not equal", () => + expect(deepEqual(1, [])).toBe(false)); + +test("0 and null are not equal", () => expect(deepEqual(0, null)).toBe(false)); + +test("equal strings", () => expect(deepEqual("a", "a")).toBe(true)); + +test("not equal strings", () => expect(deepEqual("a", "b")).toBe(false)); + +test("empty string and null are not equal", () => + expect(deepEqual("", null)).toBe(false)); + +test("null is equal to null", () => expect(deepEqual(null, null)).toBe(true)); + +test("equal booleans (true)", () => expect(deepEqual(true, true)).toBe(true)); + +test("equal booleans (false)", () => + expect(deepEqual(false, false)).toBe(true)); + +test("not equal booleans", () => expect(deepEqual(true, false)).toBe(false)); + +test("1 and true are not equal", () => expect(deepEqual(1, true)).toBe(false)); + +test("0 and false are not equal", () => + expect(deepEqual(0, false)).toBe(false)); + +test("NaN and NaN are equal", () => expect(deepEqual(NaN, NaN)).toBe(true)); + +test("0 and -0 are equal", () => expect(deepEqual(0, -0)).toBe(true)); + +test("Infinity and Infinity are equal", () => + expect(deepEqual(Infinity, Infinity)).toBe(true)); + +test("Infinity and -Infinity are not equal", () => + expect(deepEqual(Infinity, -Infinity)).toBe(false)); + +test("empty objects are equal", () => expect(deepEqual({}, {})).toBe(true)); + +test('equal objects (same properties "order")', () => + expect(deepEqual({ a: 1, b: "2" }, { a: 1, b: "2" })).toBe(true)); + +test('equal objects (different properties "order")', () => + expect(deepEqual({ a: 1, b: "2" }, { b: "2", a: 1 })).toBe(true)); + +test("not equal objects (extra property)", () => + expect(deepEqual({ a: 1, b: "2" }, { a: 1, b: "2", c: [] })).toBe(false)); + +test("not equal objects (different property values)", () => + expect(deepEqual({ a: 1, b: "2", c: 3 }, { a: 1, b: "2", c: 4 })).toBe( + false, + )); + +test("not equal objects (different properties)", () => + expect(deepEqual({ a: 1, b: "2", c: 3 }, { a: 1, b: "2", d: 3 })).toBe( + false, + )); + +test("equal objects (same sub-properties)", () => + expect(deepEqual({ a: [{ b: "c" }] }, { a: [{ b: "c" }] })).toBe(true)); + +test("not equal objects (different sub-property value)", () => + expect(deepEqual({ a: [{ b: "c" }] }, { a: [{ b: "d" }] })).toBe(false)); + +test("not equal objects (different sub-property)", () => + expect(deepEqual({ a: [{ b: "c" }] }, { a: [{ c: "c" }] })).toBe(false)); + +test("empty array and empty object are not equal", () => + expect(deepEqual({}, [])).toBe(false)); + +test("object with extra undefined properties are equal #1", () => + expect(deepEqual({}, { foo: undefined })).toBe(true)); + +test("object with extra undefined properties are equal #2", () => + expect(deepEqual({ foo: undefined }, {})).toBe(true)); + +test("object with extra undefined properties are equal #3", () => + expect(deepEqual({ foo: undefined }, { bar: undefined })).toBe(true)); + +test("nulls are equal", () => expect(deepEqual(null, null)).toBe(true)); + +test("null and undefined are not equal", () => + expect(deepEqual(null, undefined)).toBe(false)); + +test("null and empty object are not equal", () => + expect(deepEqual(null, {})).toBe(false)); + +test("undefined and empty object are not equal", () => + expect(deepEqual(undefined, {})).toBe(false)); + +test("objects with different `toString` functions returning same values are equal", () => + expect( + deepEqual( + { toString: () => "Hello world!" }, + { toString: () => "Hello world!" }, + ), + ).toBe(true)); + +test("objects with `toString` functions returning different values are not equal", () => + expect( + deepEqual({ toString: () => "Hello world!" }, { toString: () => "Hi!" }), + ).toBe(false)); + +test("two empty arrays are equal", () => expect(deepEqual([], [])).toBe(true)); + +test("equal arrays", () => expect(deepEqual([1, 2, 3], [1, 2, 3])).toBe(true)); + +test("not equal arrays (different item)", () => + expect(deepEqual([1, 2, 3], [1, 2, 4])).toBe(false)); + +test("not equal arrays (different length)", () => + expect(deepEqual([1, 2, 3], [1, 2])).toBe(false)); + +test("equal arrays of objects", () => + expect(deepEqual([{ a: "a" }, { b: "b" }], [{ a: "a" }, { b: "b" }])).toBe( + true, + )); + +test("not equal arrays of objects", () => + expect(deepEqual([{ a: "a" }, { b: "b" }], [{ a: "a" }, { b: "c" }])).toBe( + false, + )); + +test("pseudo array and equivalent array are not equal", () => + expect(deepEqual({ "0": 0, "1": 1, length: 2 }, [0, 1])).toBe(false)); + +test("equal date objects", () => + expect( + deepEqual( + new Date("2017-06-16T21:36:48.362Z"), + new Date("2017-06-16T21:36:48.362Z"), + ), + ).toBe(true)); + +test("not equal date objects", () => + expect( + deepEqual( + new Date("2017-06-16T21:36:48.362Z"), + new Date("2017-01-01T00:00:00.000Z"), + ), + ).toBe(false)); + +test("date and string are not equal", () => + expect( + deepEqual(new Date("2017-06-16T21:36:48.362Z"), "2017-06-16T21:36:48.362Z"), + ).toBe(false)); + +test("date and object are not equal", () => + expect(deepEqual(new Date("2017-06-16T21:36:48.362Z"), {})).toBe(false)); + +test("equal RegExp objects", () => expect(deepEqual(/foo/, /foo/)).toBe(true)); + +test("not equal RegExp objects (different pattern)", () => + expect(deepEqual(/foo/, /bar/)).toBe(false)); + +test("not equal RegExp objects (different flags)", () => + expect(deepEqual(/foo/, /foo/i)).toBe(false)); + +test("RegExp and string are not equal", () => + expect(deepEqual(/foo/, "foo")).toBe(false)); + +test("RegExp and object are not equal", () => + expect(deepEqual(/foo/, {})).toBe(false)); + +test("same function is equal", () => + expect(deepEqual(func1, func1)).toBe(true)); + +test("different functions are not equal", () => + expect(deepEqual(func1, func2)).toBe(false)); + +test("big object", () => + expect( + deepEqual( + { + prop1: "value1", + prop2: "value2", + prop3: "value3", + prop4: { + subProp1: "sub value1", + subProp2: { + subSubProp1: "sub sub value1", + subSubProp2: [1, 2, { prop2: 1, prop: 2 }, 4, 5], + }, + }, + prop5: 1000, + prop6: new Date(2016, 2, 10), + }, + { + prop5: 1000, + prop3: "value3", + prop1: "value1", + prop2: "value2", + prop6: new Date("2016/03/10"), + prop4: { + subProp2: { + subSubProp1: "sub sub value1", + subSubProp2: [1, 2, { prop2: 1, prop: 2 }, 4, 5], + }, + subProp1: "sub value1", + }, + }, + ), + ).toBe(true)); + +test("equal numbers", () => expect(deepEqual(1, 1)).toBe(true)); + +test("equal bigints", () => expect(deepEqual(BigInt(1), BigInt(1))).toBe(true)); + +test("not equal bigints", () => + expect(deepEqual(BigInt(1), BigInt(2))).toBe(false)); + +test("empty maps are equal", () => + expect(deepEqual(new Map(), new Map())).toBe(true)); + +test("empty maps of different class are not equal", () => + expect(deepEqual(new Map(), new MyMap())).toBe(false)); + +test('equal maps (same key "order")', () => + expect(deepEqual(map({ a: 1, b: "2" }), map({ a: 1, b: "2" }))).toBe(true)); + +test('not equal maps (same key "order" - instances of different classes)', () => + expect(deepEqual(map({ a: 1, b: "2" }), myMap({ a: 1, b: "2" }))).toBe( + false, + )); + +test('equal maps (different key "order")', () => + expect(deepEqual(map({ a: 1, b: "2" }), map({ b: "2", a: 1 }))).toBe(true)); + +test('equal maps (different key "order" - instances of the same subclass)', () => + expect(deepEqual(myMap({ a: 1, b: "2" }), myMap({ b: "2", a: 1 }))).toBe( + true, + )); + +test("not equal maps (extra key)", () => + expect(deepEqual(map({ a: 1, b: "2" }), map({ a: 1, b: "2", c: [] }))).toBe( + false, + )); + +test("not equal maps (different key value)", () => + expect( + deepEqual(map({ a: 1, b: "2", c: 3 }), map({ a: 1, b: "2", c: 4 })), + ).toBe(false)); + +test("not equal maps (different keys)", () => + expect( + deepEqual(map({ a: 1, b: "2", c: 3 }), map({ a: 1, b: "2", d: 3 })), + ).toBe(false)); + +test("equal maps (same sub-keys)", () => + expect( + deepEqual(map({ a: [map({ b: "c" })] }), map({ a: [map({ b: "c" })] })), + ).toBe(true)); + +test("not equal maps (different sub-key value)", () => + expect( + deepEqual(map({ a: [map({ b: "c" })] }), map({ a: [map({ b: "d" })] })), + ).toBe(false)); + +test("not equal maps (different sub-key)", () => + expect( + deepEqual(map({ a: [map({ b: "c" })] }), map({ a: [map({ c: "c" })] })), + ).toBe(false)); + +test("empty map and empty object are not equal", () => + expect(deepEqual({}, new Map())).toBe(false)); + +test("map with extra undefined key is not equal #1", () => + expect(deepEqual(map({}), map({ foo: undefined }))).toBe(false)); + +test("map with extra undefined key is not equal #2", () => + expect(deepEqual(map({ foo: undefined }), map({}))).toBe(false)); + +test("maps with extra undefined keys are not equal #3", () => + expect(deepEqual(map({ foo: undefined }), map({ bar: undefined }))).toBe( + false, + )); + +test("null and empty map are not equal", () => + expect(deepEqual(null, new Map())).toBe(false)); + +test("undefined and empty map are not equal", () => + expect(deepEqual(undefined, new Map())).toBe(false)); + +test("map and a pseudo map are not equal", () => + expect( + deepEqual(map({}), { + constructor: Map, + size: 0, + has: () => true, + get: () => 1, + }), + ).toBe(false)); + +test("empty sets are equal", () => + expect(deepEqual(new Set(), new Set())).toBe(true)); + +test("empty sets of different class are not equal", () => + expect(deepEqual(new Set(), new MySet())).toBe(false)); + +test('equal sets (same value "order")', () => + expect(deepEqual(set(["a", "b"]), set(["a", "b"]))).toBe(true)); + +test('not equal sets (same value "order" - instances of different classes)', () => + expect(deepEqual(set(["a", "b"]), mySet(["a", "b"]))).toBe(false)); + +test('equal sets (different value "order")', () => + expect(deepEqual(set(["a", "b"]), set(["b", "a"]))).toBe(true)); + +test('equal sets (different value "order" - instances of the same subclass)', () => + expect(deepEqual(mySet(["a", "b"]), mySet(["b", "a"]))).toBe(true)); + +test("not equal sets (extra value)", () => + expect(deepEqual(set(["a", "b"]), set(["a", "b", "c"]))).toBe(false)); + +test("not equal sets (different values)", () => + expect(deepEqual(set(["a", "b", "c"]), set(["a", "b", "d"]))).toBe(false)); + +test("not equal sets (different instances of objects)", () => + expect(deepEqual(set(["a", {}]), set(["a", {}]))).toBe(false)); + +test("equal sets (same instances of objects)", () => + expect(deepEqual(set(["a", emptyObj]), set(["a", emptyObj]))).toBe(true)); + +test("empty set and empty object are not equal", () => + expect(deepEqual({}, new Set())).toBe(false)); + +test("empty set and empty array are not equal", () => + expect(deepEqual([], new Set())).toBe(false)); + +test("set with extra undefined value is not equal #1", () => + expect(deepEqual(set([]), set([undefined]))).toBe(false)); + +test("set with extra undefined value is not equal #2", () => + expect(deepEqual(set([undefined]), set([]))).toBe(false)); + +test("set and pseudo set are not equal", () => + expect( + deepEqual(new Set(), { + constructor: Set, + size: 0, + has: () => true, + }), + ).toBe(false)); + +test("two empty arrays of the same class are equal", () => + expect(deepEqual(new Int32Array([]), new Int32Array([]))).toBe(true)); + +test("two empty arrays of the different class are not equal", () => + expect(deepEqual(new Int32Array([]), new Int16Array([]))).toBe(false)); + +test("equal arrays", () => + expect(deepEqual(new Int32Array([1, 2, 3]), new Int32Array([1, 2, 3]))).toBe( + true, + )); + +test("equal BigUint64Array arrays", () => + expect( + deepEqual( + BigUint64Array.from([BigInt(1), BigInt(2), BigInt(3)]), + BigUint64Array.from([BigInt(1), BigInt(2), BigInt(3)]), + ), + ).toBe(true)); + +test("not equal BigUint64Array arrays", () => + expect( + deepEqual( + BigUint64Array.from([BigInt(1), BigInt(2), BigInt(3)]), + BigUint64Array.from([BigInt(1), BigInt(2), BigInt(4)]), + ), + ).toBe(false)); + +test("not equal arrays (same items, different class)", () => + expect(deepEqual(new Int32Array([1, 2, 3]), new Int16Array([1, 2, 3]))).toBe( + false, + )); + +test("not equal arrays (different item)", () => + expect(deepEqual(new Int32Array([1, 2, 3]), new Int32Array([1, 2, 4]))).toBe( + false, + )); + +test("not equal arrays (different length)", () => + expect(deepEqual(new Int32Array([1, 2, 3]), new Int32Array([1, 2]))).toBe( + false, + )); + +test("pseudo array and equivalent typed array are equal", () => + expect( + deepEqual( + { "0": 1, "1": 2, length: 2, constructor: Int32Array }, + new Int32Array([1, 2]), + ), + ).toBe(true)); + +function func1(): void {} + +function func2(): void {} + +class MyMap extends Map {} + +class MySet extends Set {} + +function map( + obj: Record, + Class?: typeof MyMap, +): Map { + const a = new (Class || Map)(); + for (const key in obj) { + a.set(key, obj[key]); + } + + return a; +} + +function myMap(obj: Record): MyMap { + return map(obj, MyMap); +} + +function set(arr: unknown[], Class?: typeof MySet): Set { + const a = new (Class || Set)(); + for (const value of arr) { + a.add(value); + } + + return a; +} + +function mySet(arr: unknown[]): Set { + return set(arr, MySet); +} + +const emptyObj = {}; diff --git a/src/internal/__tests__/misc.test.ts b/src/internal/__tests__/misc.test.ts new file mode 100644 index 0000000..de35991 --- /dev/null +++ b/src/internal/__tests__/misc.test.ts @@ -0,0 +1,84 @@ +import { addSentenceSuffixes, appendCause, minifyStack } from "../misc"; + +test("minifyStack", () => { + expect( + minifyStack( + `Error: + at VC.toAnnotation (/Users/user/sd/packages/slapdash-server/src/lib/ent/ent/VC.ts:353:24) + at Function.loadNullable (/Users/user/sd/packages/slapdash-server/src/lib/ent/ent/Ent.ts:525:45) + at Function.loadX (/Users/user/sd/packages/slapdash-server/src/lib/ent/ent/Ent.ts:563:30) + at EntSome.asset2 (/Users/user/sd/packages/slapdash-server/src/ents/EntSome.ts:45:21) + at EntSome. (/Users/user/sd/packages/slapdash-shared/src/Memoize.ts:75:40) + at /Users/user/sd/packages/slapdash-server/src/drivers/asana/AsanaDriver.ts:256:32 + at /Users/user/sd/packages/slapdash-shared/src/promise.ts:55:39 + at Array.map () + at Object.mapJoin (/Users/user/sd/packages/slapdash-shared/src/promise.ts:55:23) + at loadProjects (/Users/user/sd/packages/slapdash-server/src/drivers/asana/AsanaDriver.ts:255:11)`, + 1, + ), + ).toMatchSnapshot(); +}); + +test("addSentenceSuffixes", () => { + expect(addSentenceSuffixes("a", " b", "c")).toEqual("a bc"); + expect(addSentenceSuffixes("a.", " [b]", "c")).toEqual("a [b]c"); + expect(addSentenceSuffixes("a?", "\nb", "c")).toEqual("a?\nbc"); + expect(addSentenceSuffixes("a?", "b", "c")).toEqual("abc"); +}); + +const CAUSE_STACK = `error: there is no unique or exclusion constraint matching the ON CONFLICT specification + at Parser.parseErrorMessage (/Users/user/source/sd/node_modules/pg-protocol/src/parser.ts:369:69) + at Parser.handlePacket (/Users/user/source/sd/node_modules/pg-protocol/src/parser.ts:188:21) + at Parser.parse (/Users/user/source/sd/node_modules/pg-protocol/src/parser.ts:103:30) + at Socket. (/Users/user/source/sd/node_modules/pg-protocol/src/index.ts:7:48) + at Socket.emit (node:events:519:28) + at addChunk (node:internal/streams/readable:559:12) + at readableAddChunkPushByteMode (node:internal/streams/readable:510:3) + at Socket.Readable.push (node:internal/streams/readable:390:5) + at TCP.onStreamRead (node:internal/stream_base_commons:191:23)`; +const ERR_STACK = `PgError: there is no unique or exclusion constraint matching the ON CONFLICT specification (pg_error) + at TestPgClient.query (/Users/user/source/sd/packages/ent-framework/src/pg/PgClient.ts:662:15) + at processTicksAndRejections (node:internal/process/task_queues:95:5) + at PgRunnerUpsert.clientQuery (/Users/user/source/sd/packages/ent-framework/src/pg/PgRunner.ts:72:18) + at PgRunnerUpsert.runBatch (/Users/user/source/sd/packages/ent-framework/src/pg/PgQueryUpsert.ts:113:18) + at Batcher.flushQueue (/Users/user/source/sd/packages/ent-framework/src/abstract/Batcher.ts:54:19) + at /Users/user/source/sd/packages/ent-framework/src/abstract/Batcher.ts:151:15 + at processTicksAndRejections (node:internal/process/task_queues:95:5) + at /Users/user/source/sd/packages/ent-framework/src/abstract/Shard.ts:78:21 + at Cluster.runOnShard (/Users/user/source/sd/packages/ent-framework/src/abstract/Cluster.ts:461:16) + at Function.upsert (/Users/user/source/sd/packages/ent-framework/src/ent/mixins/PrimitiveMixin.ts:397:18) + at async Promise.all (index 0) + at join (/Users/user/source/sd/packages/ent-framework/src/internal/misc.ts:142:24) + at Object. (/Users/user/source/sd/packages/ent-framework/src/ent/__tests__/Ent.errors.test.ts:48:5)`; + +test("appendCause - merge stacks", () => { + const err = new Error("test"); + err.stack = ERR_STACK; + const cause = new Error("cause"); + cause.stack = CAUSE_STACK; + appendCause(err, cause); + expect(err.stack).toMatchSnapshot(); +}); + +test("appendCause - don't append twice stacks", () => { + const err = new Error("test"); + const ERR_STACK_2 = `${ERR_STACK}\nCause: ${CAUSE_STACK}`; + err.stack = ERR_STACK_2; + const cause = new Error("cause"); + cause.stack = CAUSE_STACK; + appendCause(err, cause); + expect(err.stack).toMatch(ERR_STACK_2); +}); + +test("appendCause - keep original", () => { + const err = new Error("test"); + for (const cause of [ + { something: "12345" }, + { stack: undefined }, + null, + undefined, + ]) { + appendCause(err, cause); + expect(err.stack).toMatch(err.stack!); + } +}); diff --git a/src/internal/__tests__/objectId.test.ts b/src/internal/__tests__/objectId.test.ts new file mode 100644 index 0000000..4ad46b4 --- /dev/null +++ b/src/internal/__tests__/objectId.test.ts @@ -0,0 +1,11 @@ +import { objectId } from "../objectId"; + +test("different", () => { + expect(objectId({ a: 42 })).not.toEqual(objectId({})); + expect(objectId({ a: 42 })).not.toEqual(objectId({ a: 42 })); +}); + +test("same", () => { + const obj = { a: 42 }; + expect(objectId(obj)).toEqual(objectId(obj)); +}); diff --git a/src/internal/deepEqual.ts b/src/internal/deepEqual.ts new file mode 100644 index 0000000..f17da50 --- /dev/null +++ b/src/internal/deepEqual.ts @@ -0,0 +1,145 @@ +/** + * Inspired by https://github.com/epoberezkin/fast-deep-equal + * + * The original implementation treats objects with absent keys and the keys with + * undefined values as different, but these objects should be considered the same + * to perform proper comparison in Ent Framework. + * + * THE ORIGINAL COPYRIGHT NOTICE: + * -------------------------------------------------------------------------------- + * MIT License + * + * Copyright (c) 2017 Evgeny Poberezkin + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/** + * Deep compares a and b ignoring the order of keys in objects and keys with + * undefined values. + */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) { + return true; + } + + if (a && b && typeof a === "object" && typeof b === "object") { + if (a.constructor !== b.constructor) { + return false; + } + + let i; + if (Array.isArray(a) && Array.isArray(b)) { + const length = a.length; + if (length !== b.length) { + return false; + } + + for (i = length; i-- !== 0; ) { + if (!deepEqual(a[i], b[i])) { + return false; + } + } + + return true; + } + + if (a instanceof Map && b instanceof Map) { + if (a.size !== b.size) { + return false; + } + + for (i of a.entries()) { + if (!b.has(i[0])) { + return false; + } + } + + for (i of a.entries()) { + if (!deepEqual(i[1], b.get(i[0]))) { + return false; + } + } + + return true; + } + + if (a instanceof Set && b instanceof Set) { + if (a.size !== b.size) { + return false; + } + + for (i of a.entries()) { + if (!b.has(i[0])) { + return false; + } + } + + return true; + } + + if (ArrayBuffer.isView(a) && ArrayBuffer.isView(b)) { + const length = a.byteLength; + if (length !== b.byteLength) { + return false; + } + + return Buffer.from(a.buffer, 0).equals( + Buffer.from(b.buffer) as Uint8Array, + ); + } + + if (a instanceof ArrayBuffer && b instanceof ArrayBuffer) { + const length = a.byteLength; + if (length !== b.byteLength) { + return false; + } + + return Buffer.from(a, 0).equals(Buffer.from(b) as Uint8Array); + } + + if (a instanceof RegExp && b instanceof RegExp) { + return a.source === b.source && a.flags === b.flags; + } + + if (a.valueOf !== Object.prototype.valueOf) { + return a.valueOf() === b.valueOf(); + } + + if (a.toString !== Object.prototype.toString) { + return a.toString() === b.toString(); + } + + const keys = new Set([...Object.keys(a), ...Object.keys(b)]); + for (const key of keys) { + if ( + !deepEqual( + (a as Record)[key], + (b as Record)[key], + ) + ) { + return false; + } + } + + return true; + } + + // True if both NaN, false otherwise. + return a !== a && b !== b; +} diff --git a/src/internal/misc.ts b/src/internal/misc.ts new file mode 100644 index 0000000..3f519f5 --- /dev/null +++ b/src/internal/misc.ts @@ -0,0 +1,447 @@ +import { createHash } from "crypto"; +import { inspect } from "util"; +import compact from "lodash/compact"; +import objectHashModule from "object-hash"; + +/** + * In some cases (e.g. when actively working with callbacks), TS is still weak + * enough, so we are not always able to use generics/unknown/never types. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type DesperateAny = any; + +/** + * Removes constructor signature from a type. + * https://github.com/microsoft/TypeScript/issues/40110#issuecomment-747142570 + */ +export type OmitNew unknown> = Pick< + T, + keyof T +>; + +/** + * Adds a type alternative to constructor signature's return value. This is + * useful when we e.g. turn an instance of some Ent class into an Instance & Row + * type where Row is dynamically inferred from the schema. + */ +export type AddNew< + TClass extends new (...args: never[]) => unknown, + TRet, +> = OmitNew & { new (): InstanceType & TRet }; + +/** + * Flattens the interface to make it more readable in IntelliSense. Can be used + * when someone modifies (picks, omits, etc.) a huge type. + */ +export type Flatten = {} & { [P in keyof T]: T[P] }; + +/** + * Cancels "readonly" specifier on object's properties. + */ +export type Writeable = { -readonly [P in keyof T]: T[P] }; + +/** + * Returns a union type of all tuple strict prefixes: + * ["a", "b", "c"] -> ["a", "b"] | ["a"] + */ +export type TuplePrefixes = T extends [unknown] + ? [] + : T extends [infer First, ...infer Rest] + ? [First, ...TuplePrefixes] | [First] + : []; + +/** + * Picks only partial (optional) keys of an object. + */ +export type PickPartial = { + [K in keyof T as undefined extends T[K] ? K : never]: T[K]; +}; + +/** + * Denotes an option which can be dynamically configured at runtime. + */ +export type MaybeCallable = T | (() => T); + +/** + * Similar to MaybeCallable, but allows for async functions. + */ +export type MaybeAsyncCallable = T | (() => T) | (() => Promise); + +/** + * Some Node APIs throw not an instance of Error object, but something looking + * like an Error. So we can't do "instanceof Error" check in all cases, we can + * only compare the shape of a variable received in a `catch (e: unknown)` block + * and hope for best. + */ +export type MaybeError = + | ({ code?: string; message?: string; stack?: string } & Partial) + | null + | undefined; + +/** + * Turns a list of Promises to a list of Promise resolution results. + */ +export async function join( + promises: TList, +): Promise<{ -readonly [P in keyof TList]: Awaited }>; + +/** + * Turns an object where some values are Promises to an object with values as + * Promise resolution results. + */ +export async function join>>( + promises: TRec, +): Promise<{ -readonly [K in keyof TRec]: Awaited }>; + +/** + * A safe replacement for Promise-all built-in method. + * + * Works the same way as Promise-all, but additionally guarantees that ALL OTHER + * promises have settled in case one of them rejects. This is needed to ensure + * that we never have unexpected "dangling" promises continuing running in + * nowhere in case one of the promises rejects early (the behavior of + * Promise.all is to reject eagerly and let the rest of stuff running whilst the + * caller code unfreezes). + * + * The behavior of join() is similar to Promise.allSettled(), but it throws the + * 1st exception occurred; this is what's expected in most of the cases, and + * this is how promises are implemented in e.g. Hack. + * + * The benefits of ensuring everything is settled: + * + * 1. We never have surprising entries in our logs (e.g. imagine a request + * aborted long time ago, and then some "dangling" promises continue running + * and issue queries as if nothing happened). + * 2. Predictable control flow: if we run `await join()`, we know that no side + * effects from the spawned promises will appear after this await throws or + * returns. + * + * "Join" is a term from parallel programming (e.g. "join threads"), it’s pretty + * concrete and means that after the call, multiple parallel execution flows + * “join” into one. It's a word to describe having "one" from "many". + * + * What’s interesting is that, besides Promise-all leaks execution flows, it + * still doesn’t trigger unhandledRejection for them in case one of them throws + * later, it just swallows all other exceptions. + * + * I.e. Promise-all means "run all in parallel, if one throws - throw + * immediately and let the others continue running in nowhere; if some of THAT + * others throws, swallow their exceptions". + * + * And join() means "run all in parallel, if one throws - wait until everyone + * finishes, and then throw the 1st exception; if some of others throw, swallow + * their exceptions". + * + * See also https://en.wikipedia.org/wiki/Fork%E2%80%93join_model + */ +export async function join(promises: unknown[] | object): Promise { + const promisesArray = + promises instanceof Array ? promises : Object.values(promises); + let firstError: unknown = undefined; + let errorCount = 0; + const resultsArray = await Promise["all"]( + promisesArray.map(async (promise) => + Promise.resolve(promise).catch((err: unknown) => { + if (errorCount === 0) { + firstError = err; + } + + errorCount++; + return undefined; + }), + ), + ); + if (errorCount > 0) { + throw firstError; + } + + return promises instanceof Array + ? resultsArray + : Object.fromEntries( + Object.keys(promises).map((key, i) => [key, resultsArray[i]]), + ); +} + +/** + * A shortcut for `await join(arr.map(async ...))`. + */ +export async function mapJoin( + arr: readonly TElem[] | Promise, + func: (e: TElem, idx: number) => PromiseLike | TRet, +): Promise { + return join((await arr).map((e, idx) => func(e, idx))); +} + +/** + * Returns a random value between 1 and 1+jitter. + */ +export function jitter(jitter: number): number { + return 1 + jitter * Math.random(); +} + +/** + * Appends a stack-trace from causeErr error into err object. Useful for + * lightweight exceptions wrapping. + */ +export function appendCause< + TError extends Error, + TCause extends { stack?: unknown } | null | undefined, +>(err: TError, causeErr: TCause): TError { + if (typeof causeErr?.stack !== "string") { + return err; + } + + if (causeErr.stack && !err.stack?.includes(causeErr.stack)) { + err.stack = + (err.stack ? err.stack.trimEnd() + "\n" : "") + + "Cause: " + + causeErr.stack; + } + + return err; +} + +/** + * Inserts the caller stack into the error stack if it is not already there. + */ +export function appendCaller(e: unknown): unknown { + if ( + e && + typeof e === "object" && + "stack" in e && + typeof e.stack === "string" + ) { + const oldStack = e.stack; + Error.captureStackTrace(e, appendCaller); + const newStack = e.stack + ?.trimEnd() + .replaceAll(/(?<=\(index )\d+(?=\))/g, "*"); + if (newStack && !oldStack.includes(newStack)) { + e.stack = newStack + "\nCause: " + oldStack; + } else { + e.stack = oldStack; + } + } + + return e; +} + +/** + * Tries to minify a stacktrace by removing common parts of the paths. See unit + * test with snapshot for examples. + */ +export function minifyStack(stack: string, framesToPop: number): string { + return stack + .replace(/^\w+:[ ]*\n/s, "") // remove "Error:" prefix + .trim() + .split("\n") + .slice(framesToPop) + .join("\n") + .replace(/^\s+/gm, "") + .replace(/^[^\n]+\(\)\n/gm, "") + .replace(/(:\d+):\d+(?=[\n)])/gs, "$1") + .replace(/^(at )\/.+\//gm, "$1") + .replace(/^(at [^\n]+\()\/.+\//gm, "$1") + .replace(/^(at )([^\n]+?) \((.+)\)/gm, "$1$3 ($2)"); +} + +/** + * A simple sequence generator which never returns the same value twice within + * the same process. It's NOT random, NOT for cryptography, NOT stored (so + * starts from scratch on a process restart) and is NOT shared with other + * processes. + */ +export function localUniqueInt(): number { + return sequenceValue++; +} + +let sequenceValue = 1; + +/** + * The quickest string hasher. Don't use for crypto purposes! + * https://medium.com/@chris_72272/what-is-the-fastest-node-js-hashing-algorithm-c15c1a0e164e + */ +export function stringHash(s: string): string { + return createHash("sha1").update(s).digest("hex"); +} + +/** + * Used to calculate stable hashes of e.g. unique keys. + */ +export function objectHash(obj: object): Buffer { + return objectHashModule(obj, { + algorithm: "sha1", + encoding: "buffer", + }); +} + +/** + * Similar to objectHash(), but uses JSON.stringify() under the hood, assuming + * that it's faster than objectHash(). Also, doesn't throw when the object + * contains bigint values (as opposed to JSON.stringify()). + */ +export function jsonHash(obj: unknown): string { + return stringHash( + JSON.stringify(obj, (_, value) => + typeof value === "bigint" ? value.toString() : value, + ), + ); +} + +/** + * Indents each line of the text with 2 spaces. + */ +export function indent(message: string): string { + return message.replace(/^/gm, " "); +} + +/** + * Adds text suffixes to the sentence (typically, to an error message). + */ +export function addSentenceSuffixes( + sentence: string, + ...suffixes: Array +): string { + const compacted = compact(suffixes); + if (compacted.length === 0) { + return sentence; + } + + const suffix = compacted + .filter((suffix) => !sentence.endsWith(suffix)) + .join(""); + return suffix.startsWith("\n") + ? sentence + suffix + : sentence.trimEnd().replace(/[.!?]+$/s, "") + suffix; +} + +/** + * Returns the 1st line of the message. + */ +export function firstLine(message: T): T { + return ( + typeof message === "string" ? message.replace(/\n.*/s, "") : message + ) as T; +} + +/** + * A shorthand for inspect() in compact/no-break mode. + */ +export function inspectCompact(obj: unknown): string { + return inspect(obj, { compact: true, breakLength: Infinity }).replace( + /^([[])\s+|\s+([\]])$/gs, + (_, $1, $2) => $1 || $2, + ); +} + +/** + * Prepares something which is claimed to be an ID for debug printing in e.g. + * exception messages. We replace all non-ASCII characters to their \u + * representations. + */ +export function sanitizeIDForDebugPrinting(idIn: unknown): string { + const MAX_LEN = 32; + const id = "" + idIn; + const value = + id + .substring(0, MAX_LEN) + // We want to use control characters in this regex. + // eslint-disable-next-line no-control-regex + .replace(/[^\x1F-\x7F]/g, (v) => "\\u" + v.charCodeAt(0)) + + (id.length > MAX_LEN ? "..." : ""); + return value === "" ? '""' : value; +} + +/** + * Throws if the value passed is null or undefined. + */ +export function nullthrows( + x?: T | null, + message?: (() => string | Error) | string | Error, +): T { + if (x !== null && x !== undefined) { + return x; + } + + if (typeof message === "function") { + message = message(); + } + + const error = + message instanceof Error + ? message + : Error(message ?? `Got unexpected ${x} in nullthrows()`); + Error.captureStackTrace(error, nullthrows); + throw error; +} + +/** + * Two modes: + * 1. If an async (or sync) function is passed, spawns it in background and + * doesn't await for its termination. + * 2. If a Promise is passed, lets it continue executing, doesn't await on it. + * + * Useful when we want to launch a function "in the air", "hanging in nowhere", + * and make no-misused-promises and no-floating-promises rules happy with it. An + * example is some legacy callback-based API (e.g. chrome extension API) where + * we want to pass an async function. + * + * It's like an analog of "async on intent" comment in the code. + */ +export function runInVoid( + funcOrPromise: (() => Promise | void) | Promise | void, +): void { + if (funcOrPromise instanceof Function) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + funcOrPromise(); + } else { + // do nothing, our Promise is already hanging in nowhere + } +} + +/** + * A typesafe-way to invariant the object's key presence and being + * non-undefined. It is not always working for union types: sometimes it asserts + * the value of the key to be "any". It also doesn't remove "undefined" from the + * type of the value. + */ +export function hasKey( + k: K, + o: unknown, +): o is { [_ in K]: DesperateAny } { + return ( + !!o && + (typeof o === "object" || typeof o === "function") && + k in o && + (o as Record)[k] !== undefined + ); +} + +/** + * Same as Object.entries(), but returns strongly-typed entries. + */ +export function entries( + obj: Partial>, +): Array<[K, V]> { + return Object.entries(obj) as Array<[K, V]>; +} + +/** + * If the passed value is a function, calls it; otherwise, returns it intact. + */ +export function maybeCall(valueOrFn: MaybeCallable): T { + return typeof valueOrFn === "function" || valueOrFn instanceof Function + ? (valueOrFn as Function)() + : valueOrFn; +} + +/** + * Same as maybeCall(), but for MaybeAsyncCallable. + */ +export async function maybeAsyncCall( + valueOrFn: MaybeAsyncCallable, +): Promise { + return typeof valueOrFn === "function" || valueOrFn instanceof Function + ? (valueOrFn as Function)() + : valueOrFn; +} diff --git a/src/internal/objectId.ts b/src/internal/objectId.ts new file mode 100644 index 0000000..abce630 --- /dev/null +++ b/src/internal/objectId.ts @@ -0,0 +1,13 @@ +const $OBJECT_ID = Symbol("$OBJECT_ID"); +let seq = 0; + +/** + * Mimics the behavior of Python's `id()` function. The idea is that often times + * we can't use e.g. obj.constructor.name, because it got mangled by e.g. + * UglifyJS during bundling. But to build cache keys, we still need some + * per-object identifier. + */ +export function objectId(objIn: object): number { + const obj = objIn as { [$OBJECT_ID]?: number }; + return (obj[$OBJECT_ID] ??= seq++); +} diff --git a/src/pg/PgClient.ts b/src/pg/PgClient.ts new file mode 100644 index 0000000..57bf4ee --- /dev/null +++ b/src/pg/PgClient.ts @@ -0,0 +1,850 @@ +import defaults from "lodash/defaults"; +import range from "lodash/range"; +import pg from "pg"; +import type { + ClientConnectionIssue, + ClientOptions, + ClientPingInput, + ClientRole, +} from "../abstract/Client"; +import { Client } from "../abstract/Client"; +import type { ClientErrorPostAction } from "../abstract/ClientError"; +import { ClientError } from "../abstract/ClientError"; +import { + OP_PING, + OP_SHARD_NOS, + OP_TIMELINE_POS_REFRESH, +} from "../abstract/internal/misc"; +import type { SwallowedErrorLoggerProps } from "../abstract/Loggers"; +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { TimelineManager } from "../abstract/TimelineManager"; +import type { MaybeCallable, MaybeError, PickPartial } from "../internal/misc"; +import { + addSentenceSuffixes, + jitter, + mapJoin, + maybeCall, + runInVoid, +} from "../internal/misc"; +import { Ref } from "../internal/Ref"; +import type { Hints, Literal } from "../types"; +import { escapeLiteral } from "./helpers/escapeLiteral"; +import { buildHintQueries } from "./internal/buildHintQueries"; +import { CLIENT_ERROR_PREDICATES } from "./internal/misc"; +import { parseLsn } from "./internal/parseLsn"; +import { PgError } from "./PgError"; + +/** + * Options for PgClient constructor. + */ +export interface PgClientOptions< + TPool extends pg.Pool = pg.Pool, +> extends ClientOptions { + /** Node-Postgres config. We can't make it MaybeCallable unfortunately, + * because it's used to initialize Node-Postgres Pool. */ + config: pg.PoolConfig & { min?: number | undefined }; + /** Should create an instance of Pool class compatible with node-postgres + * Pool. By default, node-postgres Pool is used. */ + createPool?: (config: pg.PoolConfig) => TPool; + /** Close the connection after the query if it was opened long time ago. */ + maxConnLifetimeMs?: MaybeCallable; + /** Jitter for maxConnLifetimeMs. */ + maxConnLifetimeJitter?: MaybeCallable; + /** Add not more than this number of connections in each prewarm interval. New + * connections are expensive to establish (especially when SSL is enabled). */ + prewarmIntervalStep?: MaybeCallable; + /** How often to send bursts of prewarm queries to all Clients to keep the + * minimal number of open connections. The default value is half of the + * default node-postgres'es idleTimeoutMillis=10s. Together with 1..1.5x + * jitter (default prewarmIntervalJitter=0.5), it is still slightly below + * idleTimeoutMillis, and thus, doesn't let Ent Framework close the + * connections prematurely. */ + prewarmIntervalMs?: MaybeCallable; + /** Jitter for prewarmIntervalMs. */ + prewarmIntervalJitter?: MaybeCallable; + /** What prewarm query to send. */ + prewarmQuery?: MaybeCallable; + /** If true, also sends prewarm queries and keeps the min number of + * connections in all sub-pools. See pool() method for details. */ + prewarmSubPools?: boolean; + /** PG "SET key=value" hints to run before each query. Often times we use it + * to pass statement_timeout option since e.g. PGBouncer doesn't support + * per-connection statement timeout in transaction pooling mode: it throws + * "unsupported startup parameter" error. I.e. we may want to emit "SET + * statement_timeout TO ..." before each query in multi-query mode. */ + hints?: MaybeCallable | null; + /** After how many milliseconds we give up waiting for the replica to catch up + * with the master. When role="replica", then this option is the only way to + * "unlatch" the reads from the master node after a write. */ + maxReplicationLagMs?: MaybeCallable; + /** Sometimes, the role of this Client is known statically, e.g. when pointing + * to AWS Aurora writer and reader endpoints. If "master" or "replica" are + * provided, then no attempt is made to use functions like + * pg_current_wal_insert_lsn() etc. (they are barely supported in e.g. AWS + * Aurora). Instead, for "replica" role, it is treated as "always lagging up + * until maxReplicationLagMs after the last write". If role="unknown", then + * auto-detection and automatic lag tracking is performed using + * pg_current_wal_insert_lsn() and other built-in PostgreSQL functions. */ + role?: ClientRole; + /** Up to how often we call TimelineManager#triggerRefresh(). */ + replicaTimelinePosRefreshMs?: MaybeCallable; +} + +/** + * An opened low-level PostgreSQL connection. + */ +export interface PgClientConn + extends pg.PoolClient { + /** Pool instance that created this connection. */ + pool: TPool; + /** An additional property to the vanilla client: auto-incrementing ID of the + * connection for logging purposes. */ + id: number; + /** An additional property to the vanilla client: number of queries sent + * within this connection. */ + queriesSent: number; + /** An additional property to the vanilla client: when do we want to + * hard-close that connection. */ + closeAt: number | null; +} + +/** + * A named low-level Pool config used to create sub-pools. Sub-pool derives + * configuration from the default PgClientOptions#config, but allow overrides. + * See PgClient#pool() method for details. + */ +export interface PgClientSubPoolConfig extends Partial { + name: string; +} + +/** + * An abstract PostgreSQL Client. Includes connection pooling logic. + * + * Since the class is cloneable internally (using the prototype substitution + * technique, see withShard()), the contract of this class is that ALL its + * derived classes may only have readonly immediate properties. Use Ref helper + * if you need some mutable properties. + */ +export class PgClient extends Client { + /** Default values for the constructor options. */ + static override readonly DEFAULT_OPTIONS: Required< + PickPartial> + > = { + ...super.DEFAULT_OPTIONS, + createPool: (config) => new pg.Pool(config), + maxConnLifetimeMs: 0, + maxConnLifetimeJitter: 0.5, + prewarmIntervalStep: 1, + prewarmIntervalMs: 5000, + prewarmIntervalJitter: 0.5, + prewarmQuery: 'SELECT 1 AS "prewarmQuery"', + prewarmSubPools: false, + hints: null, + role: "unknown", + maxReplicationLagMs: 60000, + replicaTimelinePosRefreshMs: 1000, + }; + + /** PG named connection pools to use. The default pool has `null` key.*/ + private readonly pools = new Map(); + + /** Prewarming periodic timer (if scheduled). */ + private readonly prewarmTimeout = new Ref(null); + + /** Whether the pool has been ended and is not usable anymore. */ + private readonly ended = new Ref(false); + + /** This value is set after each request to reflect the actual role of the + * client. The idea is that master/replica role may change online, without + * reconnecting the Client, so we need to refresh it after each request and be + * ready for a fallback. The expectation is that the initial value is + * populated during the very first shardNos() call. */ + private readonly reportedRoleAfterLastQuery: Ref; + + /** This value is non-null if there was an unsuccessful connection attempt + * (i.e. the PG is down), and there were no successful queries since then. */ + private readonly reportedConnectionIssue = + new Ref(null); + + /** PgClient configuration options. */ + override readonly options: Required>; + + /** Name of the shard associated to this Client. */ + readonly shardName: string = "public"; + + /** An active TimelineManager for this particular Client. */ + readonly timelineManager: TimelineManager; + + /** + * Calls swallowedErrorLogger() doing some preliminary amendment. + */ + protected override logSwallowedError(props: SwallowedErrorLoggerProps): void { + if (!this.ended.current) { + super.logSwallowedError(props); + } + } + + /** + * Initializes an instance of PgClient. + */ + constructor(options: PgClientOptions) { + super(options); + this.options = defaults( + {}, + options, + (this as Client).options, + { + maxReplicationLagMs: + options.role !== "unknown" + ? 2000 // e.g. AWS Aurora, assuming it always "catches up" fast + : undefined, + }, + PgClient.DEFAULT_OPTIONS, + ); + + this.reportedRoleAfterLastQuery = new Ref(this.options.role); + + this.timelineManager = new TimelineManager( + this.options.maxReplicationLagMs, + this.options.replicaTimelinePosRefreshMs, + async () => { + const startTime = performance.now(); + try { + await this.query({ + query: [`SELECT '${OP_TIMELINE_POS_REFRESH}'`], + isWrite: false, + annotations: [], + op: OP_TIMELINE_POS_REFRESH, + table: "pg_catalog", + }); + } catch (error: unknown) { + this.logSwallowedError({ + where: OP_TIMELINE_POS_REFRESH, + error, + elapsed: Math.round(performance.now() - startTime), + importance: "normal", + }); + } + }, + ); + } + + /** + * Represents the full destination address this Client is working with. + * Depending on the implementation, it may include hostname, port number, + * database name, shard name etc. It is required that the address is stable + * enough to be able to cache some destination database related metadata (e.g. + * shardNos) based on that address. + */ + override address(): string { + const { host, port, database } = this.options.config; + return ( + host + + (port ? `:${port}` : "") + + (database ? `/${database}` : "") + + "#" + + this.shardName + ); + } + + /** + * Gracefully closes all the connections of this Client to let the caller + * destroy it. The pending queries are awaited to finish before returning. The + * Client becomes unusable right after calling this method (even before the + * connections are drained): you should not send queries to it. + */ + override async end(): Promise { + if (this.ended.current) { + return; + } + + this.ended.current = true; + clearTimeout(this.prewarmTimeout.current ?? undefined); + this.prewarmTimeout.current = null; + await mapJoin([...this.pools.values()], async (pool) => pool.end()); + } + + /** + * Returns true if the Client is ended and can't be used anymore. + */ + override isEnded(): boolean { + return this.ended.current; + } + + /** + * Returns all Shard numbers discoverable via the connection to the Client's + * database. + */ + override async shardNos(): Promise { + const shardNamer = this.options.shardNamer; + + // An installation without sharding enabled. + if (!shardNamer) { + return [0]; + } + + // e.g. sh0000, sh0123 and not e.g. sh1 or sh12345678 + const rows = await this.query>>({ + query: [maybeCall(shardNamer.options.discoverQuery)], + isWrite: false, + annotations: [], + op: OP_SHARD_NOS, + table: "pg_catalog", + }); + return rows + .map((row) => Object.values(row)[0]) + .map((name) => (name ? shardNamer.shardNoByName(name) : null)) + .filter((no): no is number => no !== null) + .sort((a, b) => a - b); + } + + /** + * Sends a read or write test query to the server. Tells the server to sit and + * wait for at least the provided number of milliseconds. + */ + override async ping({ + execTimeMs, + isWrite, + annotation, + }: ClientPingInput): Promise { + await this.query>>({ + query: [ + "DO $$ BEGIN PERFORM pg_sleep(?); IF pg_is_in_recovery() AND ? THEN RAISE read_only_sql_transaction; END IF; END $$", + execTimeMs / 1000, + isWrite, + ], + isWrite, + annotations: [annotation], + op: OP_PING, + table: "pg_catalog", + }); + } + + /** + * Creates a new Client which is namespaced to the provided Shard number. The + * new Client will share the same connection pool with the parent's Client. + */ + override withShard(no: number): this { + return Object.assign(Object.create(this.constructor.prototype), { + ...this, + shardName: this.options.shardNamer + ? this.options.shardNamer.shardNameByNo(no) + : this.shardName, + // Notice that we can ONLY have readonly properties in this and all + // derived classes to make it work. If we need some mutable props shared + // across all of the clones, we need to wrap them in a Ref (and make the + // Ref object itself readonly). That's a pretty fragile contract though. + }); + } + + /** + * Returns the Client's role reported after the last successful query. Master + * and replica roles may switch online unpredictably, without reconnecting, so + * we only know the role after a query. + */ + override role(): ClientRole { + return this.reportedRoleAfterLastQuery.current; + } + + /** + * Returns a non-nullable value if the Client couldn't connect to the server + * (or it could, but the load balancer reported the remote server as not + * working), so it should ideally be removed from the list of active replicas + * until e.g. the next discovery query to it (or any query) succeeds. + */ + override connectionIssue(): ClientConnectionIssue | null { + return this.reportedConnectionIssue.current; + } + + /** + * A convenience method to put connections prewarming logic to. The idea is to + * keep the needed number of open connections and also, in each connection, + * minimize the time which the very 1st query will take (e.g. pre-cache + * full-text dictionaries). + */ + override prewarm(): void { + if (this.prewarmTimeout.current) { + // Already scheduled a prewarm, so skipping. + return; + } + + const subPools = this.options.prewarmSubPools + ? [...this.pools.entries()] + .filter(([name]) => name !== null) + .map(([_, pool]) => pool) + : []; + + for (const pool of [this.pool(), ...subPools]) { + const config = pool.options as PgClientOptions["config"]; + if (!config.min) { + continue; + } + + const min = Math.min( + config.min, + config.max ?? Infinity, + pool.totalCount + (maybeCall(this.options.prewarmIntervalStep) || 1), + ); + const toPrewarm = min - pool.waitingCount; + if (toPrewarm > 0) { + const startTime = performance.now(); + range(toPrewarm).forEach(() => + runInVoid( + pool + .query(maybeCall(this.options.prewarmQuery)) + .catch((error: unknown) => + this.logSwallowedError({ + where: `${this.constructor.name}.prewarm`, + error, + elapsed: Math.round(performance.now() - startTime), + importance: "normal", + }), + ), + ), + ); + } + } + + this.prewarmTimeout.current = setTimeout( + () => { + this.prewarmTimeout.current = null; + this.prewarm(); + }, + Math.round( + maybeCall(this.options.prewarmIntervalMs) * + jitter(maybeCall(this.options.prewarmIntervalJitter)), + ), + ).unref(); + } + + /** + * Returns a default pool (when subPoolConfig is not passed), or a "sub-pool" + * (a named low-level Pool implementation compatible to node-postgres). The + * method is specific to the current class and is not a part of + * database-agnostic Client API. + * - Sub-pools are lazily created and memoized by the provided name. They may + * differ by config options (like statement_timeout or max connections). + * - Sub-pools inherit the properties from default PgClientOptions.config. + * - It is implied (but not enforced) that all sub-pools use the same physical + * database, because otherwise it makes not a lot of sense. + */ + pool(subPoolConfig?: PgClientSubPoolConfig): TPool { + let pool = this.pools.get(subPoolConfig?.name ?? null); + if (pool) { + return pool; + } + + pool = this.options + .createPool( + defaults({}, subPoolConfig, this.options.config, { + allowExitOnIdle: true, + }), + ) + .on("connect", (poolConn) => { + // Called only once, after the connection is 1st created. + const conn = poolConn as PgClientConn; + + // Initialize additional properties merged into the default PoolClient. + const maxConnLifetimeMs = + maybeCall(this.options.maxConnLifetimeMs) * + jitter(maybeCall(this.options.maxConnLifetimeJitter)); + conn.pool = pool!; + conn.id = connId++; + conn.queriesSent = 0; + conn.closeAt = + maxConnLifetimeMs > 0 + ? Date.now() + Math.round(maxConnLifetimeMs) + : null; + + // Sets a "default error" handler to not let errors leak to e.g. Jest + // and the outside world as "unhandled error". Appending an additional + // error handler to EventEmitter doesn't affect the existing error + // handlers anyhow, so should be safe. + conn.on("error", () => {}); + this.options.loggers?.clientConnectedLogger?.({ + node: this.options, + }); + }) + .on("error", (error) => + // Having this hook prevents node from crashing. + this.logSwallowedError({ + where: 'Pool.on("error")', + error, + elapsed: null, + importance: "low", + }), + ); + + this.pools.set(subPoolConfig?.name ?? null, pool); + return pool; + } + + /** + * Called when the Client needs a connection in the default pool (when + * subPoolConfig is not passed), or in a sub-pool (see pool() method) to run a + * query against. Implies than the caller MUST call release() method on the + * returned object. The difference from pool().connect() is that when calling + * release() on a result of acquireConn(), it additionally closes the + * connection automatically if was OPENED (not queried!) more than + * maxConnLifetimeMs ago (node-postgres Pool doesn't have this feature) The + * method is specific to the current class and is not a part of + * database-agnostic Client API. + */ + async acquireConn( + subPoolConfig?: PgClientSubPoolConfig, + ): Promise> { + const pool = this.pool(subPoolConfig); + const conn = (await pool.connect()) as PgClientConn; + + const connReleaseOrig = conn.release.bind(conn); + conn.release = (arg) => { + // Manage maxConnLifetimeMs manually since it's not supported by the + // vanilla node-postgres. + const needClose = !!(conn.closeAt && Date.now() > conn.closeAt); + return connReleaseOrig(arg !== undefined ? arg : needClose); + }; + + return conn; + } + + /** + * Sends a query (internally, a multi-query) through the default Pool (if + * subPoolConfig is not passed), or through a named sub-pool (see pool() + * method). After the query finishes, we should expect that role() returns the + * actual master/replica role. The method is specific to the current class and + * is not a part of database-agnostic Client API. + */ + async query({ + query: queryLiteral, + hints, + isWrite, + annotations, + op, + table, + batchFactor, + subPoolConfig, + }: { + query: Literal; + hints?: Hints; + isWrite: boolean; + annotations: QueryAnnotation[]; + op: string; + table: string; + batchFactor?: number; + subPoolConfig?: PgClientSubPoolConfig; + }): Promise { + const { queries, queriesRollback, debugQueryWithHints, resultPos } = + this.buildMultiQuery( + hints, + queryLiteral, + this.options.role === "unknown" + ? // For master, we read its WAL LSN (pg_current_wal_insert_lsn) after + // each query (notice that, when run on a replica, + // pg_current_wal_insert_lsn() throws, so we call it only if + // pg_is_in_recovery() returns false). For replica, we read its WAL + // LSN (pg_last_wal_replay_lsn). + "SELECT CASE WHEN pg_is_in_recovery() THEN NULL ELSE pg_current_wal_insert_lsn() END AS pg_current_wal_insert_lsn, pg_last_wal_replay_lsn()" + : undefined, + isWrite, + ); + + const startTime = performance.now(); + let queryTime: number | undefined = undefined; + let conn: PgClientConn | undefined = undefined; + let res: TRow[] | undefined = undefined; + let e: MaybeError<{ severity?: unknown }> = undefined; + let postAction: ClientErrorPostAction = "fail"; + + try { + if (this.isEnded()) { + throw new ClientError( + Error(`Cannot use ${this.constructor.name} since it's ended`), + this.options.name, + "choose-another-client", + "data-on-server-is-unchanged", + "client_is_ended", + ); + } + + conn = await this.acquireConn(subPoolConfig); + conn.queriesSent++; + + queryTime = Math.round(performance.now() - startTime); + const resMulti = await this.sendMultiQuery( + conn, + queries, + queriesRollback, + ); + this.reportedConnectionIssue.current = null; + + res = resMulti[resultPos].rows; + + if (this.options.role === "unknown") { + const lsns = resMulti[resMulti.length - 1].rows[0] as { + pg_current_wal_insert_lsn: string | null; + pg_last_wal_replay_lsn: string | null; + }; + if (lsns.pg_current_wal_insert_lsn !== null) { + this.reportedRoleAfterLastQuery.current = "master"; + this.timelineManager.setCurrentPos( + parseLsn(lsns.pg_current_wal_insert_lsn)!, + ); + } else if (lsns.pg_last_wal_replay_lsn !== null) { + this.reportedRoleAfterLastQuery.current = "replica"; + this.timelineManager.setCurrentPos( + parseLsn(lsns.pg_last_wal_replay_lsn), + ); + } else { + throw Error( + "BUG: both pg_current_wal_insert_lsn() and pg_last_wal_replay_lsn() returned null", + ); + } + } else if (this.options.role === "master") { + this.reportedRoleAfterLastQuery.current = "master"; + // In this mode, master pos is always =1 constant. + this.timelineManager.setCurrentPos(BigInt(1), true); + } else { + this.reportedRoleAfterLastQuery.current = "replica"; + // In this mode, replica pos is always =0 constant (i.e. always behind + // the master), and we solely rely on maxReplicationLagMs timeline data + // expiration in Timeline object. + this.timelineManager.setCurrentPos(BigInt(0), true); + } + + return res; + } catch (cause: unknown) { + e = cause as MaybeError<{ severity?: unknown }>; + + if (e instanceof ClientError) { + throw e; + } + + // Infer ClientError which affects Client choosing logic. + for (const predicate of CLIENT_ERROR_PREDICATES) { + const res = predicate({ + code: "" + e?.code, + message: "" + e?.message, + }); + if (res) { + if (!isWrite) { + // For read queries, we know for sure that the data wasn't changed. + res.kind = "data-on-server-is-unchanged"; + } + + postAction = + this.role() === "master" + ? res.postAction.ifMaster + : res.postAction.ifReplica; + + if (res.postAction.reportConnectionIssue) { + // Mark the current Client as non-healthy, so the retry logic will + // likely choose another one if available. + this.reportedConnectionIssue.current = { + timestamp: new Date(), + cause, + postAction, + kind: res.kind, + comment: res.comment, + }; + } + + throw new ClientError( + e, + this.options.name, + postAction, + res.kind, + res.abbreviation, + res.comment + + (res.kind === "unknown-server-state" + ? " The write might have been committed on the PG server though." + : ""), + ); + } + } + + // Only wrap the errors which PG sent to us explicitly. Those errors mean + // that there was some aborted transaction, so it's safe to retry. + if (e?.severity) { + throw new PgError(e, this.options.name, debugQueryWithHints, table); + } + + // Some other error which should not trigger query retries or + // Shards/Islands rediscovery. + throw e; + } finally { + conn?.release(); + const pool = conn?.pool ?? this.pool(subPoolConfig); + const now = performance.now(); + this.options.loggers?.clientQueryLogger?.({ + annotations, + op, + shard: this.shardName, + table, + batchFactor: batchFactor ?? 1, + msg: debugQueryWithHints, + output: res ? res : undefined, + elapsed: { + total: Math.round(now - startTime), + acquire: + queryTime !== undefined ? queryTime : Math.round(now - startTime), + }, + connStats: { + id: conn ? "" + (conn.id ?? 0) : "?", + queriesSent: conn?.queriesSent ?? 0, + }, + poolStats: { + totalConns: pool.totalCount, + idleConns: pool.idleCount, + queuedReqs: pool.waitingCount, + }, + error: + e === undefined + ? undefined + : addSentenceSuffixes( + `${e}`, + e?.code ? ` (${e.code})` : undefined, + ` [${postAction}]`, + ), + role: this.role(), + backend: this.options.name, + address: this.address(), + }); + } + } + + /** + * Prepares a PG Client multi-query from the query literal and hints. + */ + private buildMultiQuery( + hints: Hints | undefined, + literal: Literal, + epilogue: string | undefined, + isWrite: boolean, + ): { + queries: string[]; + queriesRollback: string[]; + debugQueryWithHints: string; + resultPos: number; + } { + const queriesPrologue: string[] = []; + const queriesEpilogue: string[] = []; + const queriesRollback: string[] = []; + + const [rawPrepend, hintQueriesDefault, hintQueries] = buildHintQueries( + this.options.hints ? maybeCall(this.options.hints) : undefined, + hints, + ); + + const rawQuery = escapeLiteral(literal).trimEnd(); + if (rawQuery === "") { + throw Error("Empty query passed to query()"); + } + + const query = rawPrepend + rawQuery; + + // Prepend per-query hints to the prologue (if any); they will be logged. + queriesPrologue.unshift(...hintQueries); + + // The query which is logged to the logging infra. For more brief messages, + // we don't log internal hints (this.hints) and search_path; see below. + const debugQueryWithHints = + `/*${this.shardName}*/` + [...queriesPrologue, query].join("; ").trim(); + + // Prepend internal per-Client hints to the prologue. + queriesPrologue.unshift(...hintQueriesDefault); + + // We must always have "public" in search_path, because extensions are by + // default installed in "public" schema. Some extensions may expose + // operators (e.g. "citext" exposes comparison operators) which must be + // available in all Shards by default, so they should live in "public". + // (There is a way to install an extension to a particular schema, but a) + // there can be only one such schema, and b) there are problems running + // pg_dump when migrating this Shard to another machine since pg_dump + // doesn't emit CREATE EXTENSION statement when filtering by schema name). + queriesPrologue.unshift( + `SET LOCAL search_path TO ${this.shardName}, public`, + ); + + if (epilogue) { + queriesEpilogue.push(epilogue); + } + + // Why wrapping with BEGIN...COMMIT for write queries? See here: + // https://www.postgresql.org/message-id/20220803.163217.1789690807623885906.horikyota.ntt%40gmail.com + if (isWrite && queriesEpilogue.length > 0) { + queriesPrologue.unshift("BEGIN"); + queriesRollback.unshift("ROLLBACK"); + queriesEpilogue.unshift("COMMIT"); + } + + return { + queries: [...queriesPrologue, query, ...queriesEpilogue], + queriesRollback, + debugQueryWithHints, + resultPos: queriesPrologue.length, + }; + } + + /** + * Sends a multi-query to PG Client. + * + * A good and simple explanation of the protocol is here: + * https://www.postgresql.org/docs/13/protocol-flow.html. In short, we can't + * use prepared-statement-based operations even theoretically, because this + * mode doesn't support multi-queries. Also notice that TS typing is doomed + * for multi-queries: + * https://github.com/DefinitelyTyped/DefinitelyTyped/pull/33297 + */ + private async sendMultiQuery( + conn: PgClientConn, + queries: string[], + queriesRollback: string[], + ): Promise { + const queriesStr = `/*${this.shardName}*/${queries.join("; ")}`; + + // For multi-query, query() actually returns an array of pg.QueryResult, but + // it's not reflected in its TS typing, so patching this. + const resMulti = (await conn.query(queriesStr).catch(async (e: unknown) => { + // We must run a ROLLBACK if we used BEGIN in the queries, because + // otherwise the connection is released to the pool in "aborted + // transaction" state (see the protocol link above). + queriesRollback.length > 0 && + (await conn.query(queriesRollback.join("; ")).catch(() => {})); + throw e; + })) as unknown as Array>; + + if (resMulti.length !== queries.length) { + throw Error( + `Multi-query (with semicolons) is not allowed as an input to query(); got ${queriesStr}`, + ); + } + + return resMulti; + } +} + +/** + * For backward compatibility, exposing the old name as well. + * @deprecated Use PgClient instead. + * @ignore + */ +export const PgClientPool = PgClient; + +/** + * For backward compatibility, exposing the old name as well. + * @deprecated Use PgClient instead. + * @ignore + */ +export type PgClientPool = PgClient; + +/** + * For backward compatibility, exposing the old name as well. + * @deprecated Use PgClientOptions instead. + * @ignore + */ +export type PgClientPoolOptions = PgClientOptions; + +/** + * Auto-incrementing connection number (for debugging purposes). + */ +let connId = 1; diff --git a/src/pg/PgError.ts b/src/pg/PgError.ts new file mode 100644 index 0000000..1802a2a --- /dev/null +++ b/src/pg/PgError.ts @@ -0,0 +1,25 @@ +import { ClientError } from "../abstract/ClientError"; + +export class PgError extends ClientError { + constructor( + cause: null | undefined | {}, + where: string, + public readonly sql: string, + public readonly table: string, + ) { + super(cause, where, "fail", "data-on-server-is-unchanged", "pg_error"); + + Object.defineProperty(this, "sql", { + value: sql, + writable: false, + enumerable: false, + }); + } + + isFKError(fkName?: string): boolean { + return ( + this.message.includes("foreign key constraint") && + (!fkName || this.message.includes(fkName)) + ); + } +} diff --git a/src/pg/PgQueryCount.ts b/src/pg/PgQueryCount.ts new file mode 100644 index 0000000..7d6f821 --- /dev/null +++ b/src/pg/PgQueryCount.ts @@ -0,0 +1,80 @@ +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import type { Schema } from "../abstract/Schema"; +import type { CountInput, Table } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryCount extends QueryBase< + TTable, + CountInput, + number, + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerCount; +} + +class PgRunnerCount extends PgRunner< + TTable, + CountInput, + number +> { + static override readonly IS_WRITE = false; + private builder; + + readonly op = "COUNT"; + readonly maxBatchSize = 100; + readonly default = 0; // We just need something here. + + constructor(schema: Schema, client: PgClient) { + super(schema, client); + this.builder = this.createWhereBuilder({ + prefix: this.fmt("SELECT COUNT(1) AS count FROM %T "), + suffix: this.fmt(""), + }); + } + + override key(input: CountInput): string { + // Coalesce equal queries. + return JSON.stringify(input); + } + + async runSingle( + input: CountInput, + annotations: QueryAnnotation[], + ): Promise { + const sql = + this.builder.prefix + this.builder.func(input) + this.builder.suffix; + const res = await this.clientQuery<{ count: string }>(sql, annotations, 1); + return parseInt(res[0].count); + } + + async runBatch( + inputs: Map>, + annotations: QueryAnnotation[], + ): Promise> { + // SELECT COUNT(1) FROM ... WHERE ... + // UNION ALL + // SELECT COUNT(1) FROM ... WHERE ... + const sql = [...inputs.values()] + .map( + (input) => + this.builder.prefix + this.builder.func(input) + this.builder.suffix, + ) + .join("\n UNION ALL\n"); + const rows = await this.clientQuery<{ i: string; count: string }>( + sql, + annotations, + inputs.size, + ); + const outputs = new Map(); + let i = 0; + for (const key of inputs.keys()) { + outputs.set(key, parseInt(rows[i].count)); + i++; + } + + return outputs; + } +} diff --git a/src/pg/PgQueryDelete.ts b/src/pg/PgQueryDelete.ts new file mode 100644 index 0000000..fb4c436 --- /dev/null +++ b/src/pg/PgQueryDelete.ts @@ -0,0 +1,74 @@ +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import type { Schema } from "../abstract/Schema"; +import type { Table } from "../types"; +import { ID } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryDelete extends QueryBase< + TTable, + string, + boolean, + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerDelete; +} + +class PgRunnerDelete extends PgRunner< + TTable, + string, + boolean +> { + static override readonly IS_WRITE = true; + private builder; + + readonly op = "DELETE"; + readonly maxBatchSize = 100; + readonly default = false; // If no delete happened, returns false. + + constructor(schema: Schema, client: PgClient) { + super(schema, client); + this.builder = { + prefix: this.fmt("DELETE FROM %T WHERE "), + func: this.createOneOfBuilder(ID), + suffix: this.fmt(` RETURNING %PK AS ${ID}`), + }; + } + + override key(input: string): string { + return input; + } + + async runSingle( + input: string, + annotations: QueryAnnotation[], + ): Promise { + const sql = + this.builder.prefix + this.builder.func([input]) + this.builder.suffix; + const rows = await this.clientQuery<{ [ID]: string }>(sql, annotations, 1); + return rows.length > 0 ? true : false; + } + + async runBatch( + inputs: Map, + annotations: QueryAnnotation[], + ): Promise> { + const sql = + this.builder.prefix + + this.builder.func(inputs.values()) + + this.builder.suffix; + const rows = await this.clientQuery<{ [ID]: string }>( + sql, + annotations, + inputs.size, + ); + const outputs = new Map(); + for (const row of rows) { + outputs.set(row[ID], true); + } + + return outputs; + } +} diff --git a/src/pg/PgQueryDeleteWhere.ts b/src/pg/PgQueryDeleteWhere.ts new file mode 100644 index 0000000..2f3a97e --- /dev/null +++ b/src/pg/PgQueryDeleteWhere.ts @@ -0,0 +1,68 @@ +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import type { Schema } from "../abstract/Schema"; +import { stringHash } from "../internal/misc"; +import type { DeleteWhereInput, Table } from "../types"; +import { ID } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryDeleteWhere extends QueryBase< + TTable, + DeleteWhereInput, + string[], + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerDeleteWhere; +} + +class PgRunnerDeleteWhere extends PgRunner< + TTable, + DeleteWhereInput, + string[] +> { + static override readonly IS_WRITE = true; + private builder; + + readonly op = "DELETE_WHERE"; + readonly maxBatchSize = 1; + readonly default = []; + + // This runner doesn't support batching. + runBatch = undefined; + + constructor(schema: Schema, client: PgClient) { + super(schema, client); + this.builder = this.createWhereBuilder({ + prefix: this.fmt("DELETE FROM %T "), + suffix: this.fmt(` RETURNING %PK AS ${ID}`), + }); + } + + override key(input: DeleteWhereInput): string { + // Coalesce equal delete queries. + const json = JSON.stringify(input); + return stringHash(json); + } + + async runSingle( + input: DeleteWhereInput, + annotations: QueryAnnotation[], + ): Promise { + if (!(input[ID] instanceof Array)) { + throw Error( + `Field ${ID} must be an array of IDs in ${this.op} query (for safety)`, + ); + } + + const sql = + this.builder.prefix + this.builder.func(input) + this.builder.suffix; + const rows = await this.clientQuery<{ [ID]: string }>( + sql, + annotations, + input[ID].length, + ); + return rows.map((row) => row[ID]); + } +} diff --git a/src/pg/PgQueryExists.ts b/src/pg/PgQueryExists.ts new file mode 100644 index 0000000..d4af53b --- /dev/null +++ b/src/pg/PgQueryExists.ts @@ -0,0 +1,87 @@ +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import type { Schema } from "../abstract/Schema"; +import type { ExistsInput, Table } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryExists extends QueryBase< + TTable, + ExistsInput, + boolean, + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerExists; +} + +class PgRunnerExists extends PgRunner< + TTable, + ExistsInput, + boolean +> { + static override readonly IS_WRITE = false; + private builder; + + readonly op = "EXISTS"; + readonly maxBatchSize = 100; + readonly default = false; // We just need something here. + + constructor(schema: Schema, client: PgClient) { + super(schema, client); + this.builder = this.createWhereBuilder({ + prefix: this.fmt("SELECT EXISTS (SELECT true FROM %T "), + suffix: this.fmt(")"), + }); + } + + override key(input: ExistsInput): string { + // Coalesce equal queries. + return JSON.stringify(input); + } + + async runSingle( + input: ExistsInput, + annotations: QueryAnnotation[], + ): Promise { + const sql = + this.builder.prefix + this.builder.func(input) + this.builder.suffix; + const res = await this.clientQuery<{ exists: boolean }>( + sql, + annotations, + 1, + ); + return !!res[0].exists; + } + + async runBatch( + inputs: Map>, + annotations: QueryAnnotation[], + ): Promise> { + // SELECT EXISTS(SELECT 1 FROM ... WHERE ...) + // UNION ALL + // SELECT EXISTS(SELECT 1 FROM ... WHERE ...) + const sql = [...inputs.values()] + .map( + (input) => + this.builder.prefix + this.builder.func(input) + this.builder.suffix, + ) + .join("\n UNION ALL\n"); + const rows = await this.clientQuery<{ i: string; exists: boolean }>( + sql, + annotations, + inputs.size, + // The reasonable assumption is that, if someone uses EXISTS, they always + // want the query to match some index. + { enable_seqscan: "off" }, + ); + const outputs = new Map(); + let i = 0; + for (const key of inputs.keys()) { + outputs.set(key, !!rows[i].exists); + i++; + } + + return outputs; + } +} diff --git a/src/pg/PgQueryIDGen.ts b/src/pg/PgQueryIDGen.ts new file mode 100644 index 0000000..a67be06 --- /dev/null +++ b/src/pg/PgQueryIDGen.ts @@ -0,0 +1,70 @@ +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import { nullthrows } from "../internal/misc"; +import type { Table } from "../types"; +import { ID } from "../types"; +import { escapeIdent } from "./helpers/escapeIdent"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryIDGen extends QueryBase< + TTable, + void, // input + string, // output + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerIDGen; +} + +class PgRunnerIDGen extends PgRunner< + TTable, + void, + string +> { + static override readonly IS_WRITE = true; + private readonly idAutoInsert = nullthrows( + this.schema.table[ID].autoInsert, + `Schema for ${this.name}.${ID} must have autoInsert attribute defined`, + ); + + readonly op = "ID_GEN"; + readonly maxBatchSize = 100; + readonly default = "never_happens"; // abstract property implementation + + async runSingle( + _input: void, + annotations: QueryAnnotation[], + ): Promise { + const sql = "SELECT " + this.idAutoInsert; + const rows = await this.clientQuery<{ [k: string]: string }>( + sql, + annotations, + 1, + ); + return Object.values(rows[0])[0]; + } + + async runBatch( + inputs: Map, + annotations: QueryAnnotation[], + ): Promise> { + const parts = []; + for (const key of inputs.keys()) { + parts.push(this.idAutoInsert + " AS " + escapeIdent(key)); + } + + const sql = "SELECT " + parts.join(", "); + const rows = await this.clientQuery<{ [k: string]: string }>( + sql, + annotations, + inputs.size, + ); + const outputs = new Map(); + for (const [key, id] of Object.entries(rows[0])) { + outputs.set(key, id); + } + + return outputs; + } +} diff --git a/src/pg/PgQueryInsert.ts b/src/pg/PgQueryInsert.ts new file mode 100644 index 0000000..83666dc --- /dev/null +++ b/src/pg/PgQueryInsert.ts @@ -0,0 +1,106 @@ +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import type { Schema } from "../abstract/Schema"; +import type { InsertInput, Table } from "../types"; +import { ID } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryInsert extends QueryBase< + TTable, + InsertInput, + string | null, + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerInsert; +} + +class PgRunnerInsert extends PgRunner< + TTable, + InsertInput, + string | null +> { + static override readonly IS_WRITE = true; + private singleBuilder; + private batchBuilder; + + readonly op = "INSERT"; + readonly maxBatchSize = 100; + readonly default = null; // In case of duplicate key error, returns null. + + constructor(schema: Schema, client: PgClient) { + super(schema, client); + + const fields = this.addPK(Object.keys(this.schema.table), "append"); + + this.singleBuilder = this.createValuesBuilder({ + prefix: this.fmt("INSERT INTO %T (%FIELDS) VALUES", { fields }), + indent: " ", + fields, + suffix: this.fmt(` ON CONFLICT DO NOTHING RETURNING %PK AS ${ID}`), + }); + + // We use WITH clause in INSERT, because "ON CONFLICT DO NOTHING" clause + // doesn't emit anything in "RETURNING" clause, so we could've not + // distinguished rows which were inserted from the rows which were not. + // Having WITH solves this (see RETURNING below). + this.batchBuilder = this.createWithBuilder({ + fields, + suffix: this.fmt( + "INSERT INTO %T (%FIELDS)\n" + + "SELECT %FIELDS FROM rows OFFSET 1\n" + + `ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE %PK(rows)=%PK(%T)), %PK AS ${ID}`, + { fields }, + ), + }); + } + + override key(input: InsertInput): string { + // We must NEVER dedup inserts, because: + // 1. If the table DOESN'T have an unique key, then we must insert all + // input rows (no dedup allowed). + // 2. If the table DOES have an unique key, then we must logically ensure + // that only one concurrent promise is resolved into an inserted row ID, + // and all other are resolved with null (aka "not inserted due to + // duplicate"). + return super.key(input); + } + + async runSingle( + input: InsertInput, + annotations: QueryAnnotation[], + ): Promise { + const sql = + this.singleBuilder.prefix + + this.singleBuilder.func([["", input]]) + + this.singleBuilder.suffix; + const rows = await this.clientQuery<{ [ID]: string }>(sql, annotations, 1); + if (!rows.length) { + return undefined; + } + + return rows[0][ID]; + } + + async runBatch( + inputs: Map>, + annotations: QueryAnnotation[], + ): Promise> { + const sql = + this.batchBuilder.prefix + + this.batchBuilder.func(inputs) + + this.batchBuilder.suffix; + const rows = await this.clientQuery<{ _key: string; [ID]: string }>( + sql, + annotations, + inputs.size, + ); + const outputs = new Map(); + for (const row of rows) { + outputs.set(row._key, row[ID]); + } + + return outputs; + } +} diff --git a/src/pg/PgQueryLoad.ts b/src/pg/PgQueryLoad.ts new file mode 100644 index 0000000..fb951c7 --- /dev/null +++ b/src/pg/PgQueryLoad.ts @@ -0,0 +1,74 @@ +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import type { Schema } from "../abstract/Schema"; +import type { Row, Table } from "../types"; +import { ID } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryLoad extends QueryBase< + TTable, + string, + Row | null, + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerLoad; +} + +class PgRunnerLoad extends PgRunner< + TTable, + string, + Row | null +> { + static override readonly IS_WRITE = false; + private builder; + + readonly op = "SELECT_BY_ID"; + readonly maxBatchSize = 1000; // Select by ID is cheap, so we can have much bigger load batches. + readonly default = null; // If no row is found, returns null. + + constructor(schema: Schema, client: PgClient) { + super(schema, client); + this.builder = { + prefix: this.fmt("SELECT %SELECT_FIELDS FROM %T WHERE "), + func: this.createOneOfBuilder(ID), + suffix: this.fmt(""), + }; + } + + override key(input: string): string { + return input; + } + + async runSingle( + input: string, + annotations: QueryAnnotation[], + ): Promise | undefined> { + const sql = + this.builder.prefix + this.builder.func([input]) + this.builder.suffix; + const rows = await this.clientQuery>(sql, annotations, 1); + return rows[0]; + } + + async runBatch( + inputs: Map, + annotations: QueryAnnotation[], + ): Promise>> { + const sql = + this.builder.prefix + + this.builder.func(inputs.values()) + + this.builder.suffix; + const rows = await this.clientQuery>( + sql, + annotations, + inputs.size, + ); + const outputs = new Map>(); + for (const row of rows) { + outputs.set(row[ID], row); + } + + return outputs; + } +} diff --git a/src/pg/PgQueryLoadBy.ts b/src/pg/PgQueryLoadBy.ts new file mode 100644 index 0000000..75dd1f0 --- /dev/null +++ b/src/pg/PgQueryLoadBy.ts @@ -0,0 +1,86 @@ +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import type { Schema } from "../abstract/Schema"; +import type { LoadByInput, Row, Table, UniqueKey } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryLoadBy< + TTable extends Table, + TUniqueKey extends UniqueKey, +> extends QueryBase< + TTable, + LoadByInput, + Row | null, + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerLoadBy; +} + +class PgRunnerLoadBy< + TTable extends Table, + TUniqueKey extends UniqueKey, +> extends PgRunner< + TTable, + LoadByInput, + Row | null +> { + static override readonly IS_WRITE = false; + private builders; + + readonly op = "SELECT_UNIQ"; + override readonly maxBatchSize = 200; // Select by unique key is cheap, so we can have much bigger load batches to accumulate more data from e.g. Shard 0 for the next multi-Shard requests. + readonly default = null; // If no row is found, returns null. + + constructor(schema: Schema, client: PgClient) { + super(schema, client); + this.builders = this.createWhereBuildersFieldsEq< + LoadByInput + >({ + prefix: this.fmt("SELECT %SELECT_FIELDS FROM %T "), + fields: this.schema.uniqueKey, + suffix: this.fmt(""), + }); + } + + override key(input: LoadByInput): string { + return JSON.stringify(this.schema.uniqueKey.map((field) => input[field])); + } + + async runSingle( + input: LoadByInput, + annotations: QueryAnnotation[], + ): Promise | undefined> { + const sql = + this.builders.plain.prefix + + this.builders.plain.func([["", input]]) + + this.builders.plain.suffix; + const rows = await this.clientQuery>(sql, annotations, 1); + return rows[0]; + } + + async runBatch( + inputs: Map>, + annotations: QueryAnnotation[], + ): Promise>> { + const sql = + this.builders.optimized.prefix + + this.builders.optimized.func(inputs) + + this.builders.optimized.suffix; + const rows = await this.clientQuery>( + sql, + annotations, + inputs.size, + ); + const output = new Map>(); + for (const row of rows) { + output.set( + this.key(row as unknown as LoadByInput), + row, + ); + } + + return output; + } +} diff --git a/src/pg/PgQuerySelect.ts b/src/pg/PgQuerySelect.ts new file mode 100644 index 0000000..1944eff --- /dev/null +++ b/src/pg/PgQuerySelect.ts @@ -0,0 +1,232 @@ +import pickBy from "lodash/pickBy"; +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { QueryBase } from "../abstract/QueryBase"; +import type { Schema } from "../abstract/Schema"; +import { stringHash, hasKey, inspectCompact } from "../internal/misc"; +import type { Hints, Literal, Order, Row, SelectInput, Table } from "../types"; +import { escapeLiteral } from "./helpers/escapeLiteral"; +import { RAW_PREPEND_HINT } from "./internal/buildHintQueries"; +import { escapeString } from "./internal/escapeString"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +/** + * This is mostly to do hacks in PostgreSQL queries. Not even exposed by Ent + * framework, but can be used by PG-dependent code. + */ +export type SelectInputCustom = + | { + ctes?: Literal[]; + joins?: Literal[]; + from?: Literal; + hints?: Hints; + } + | undefined; + +const ALLOWED_ORDER = [ + "ASC", + "ASC NULLS LAST", + "ASC NULLS FIRST", + "DESC", + "DESC NULLS LAST", + "DESC NULLS FIRST", +]; + +export class PgQuerySelect extends QueryBase< + TTable, + SelectInput, + Array>, + PgClient +> { + /** @ignore */ + readonly RUNNER_CLASS = PgRunnerSelect; + + override async run( + client: PgClient, + annotation: QueryAnnotation, + ): Promise>> { + const custom = this.input.custom as SelectInputCustom; + const disableBatching = !!custom?.hints?.[RAW_PREPEND_HINT]; + return client + .batcher( + this.constructor, + this.schema, + JSON.stringify(custom?.hints) || "", + disableBatching, + () => new this.RUNNER_CLASS(this.schema, client), + ) + .run(this.input, annotation); + } +} + +class PgRunnerSelect extends PgRunner< + TTable, + SelectInput, + Array> +> { + static override readonly IS_WRITE = false; + private prefix = this.fmt("SELECT %SELECT_FIELDS FROM %T "); + private prefixUnion = this.fmt("SELECT "); + private midfixUnion = this.fmt(" AS _key, %SELECT_FIELDS FROM %T "); + private builder; + + readonly op = "SELECT"; + readonly maxBatchSize = 10; // PG crashes on large queries with lots of UNION ALL, so we keep this value low. + readonly default = []; // We just need something here. + + constructor(schema: Schema, client: PgClient) { + super(schema, client); + this.builder = this.createWhereBuilder({ + prefix: this.fmt(""), + suffix: this.fmt(""), + }); + } + + override key(input: SelectInput): string { + // Coalesce equal select queries. + const json = JSON.stringify(input); + return stringHash(json); + } + + async runSingle( + input: SelectInput, + annotations: QueryAnnotation[], + ): Promise>> { + const { sql, hints } = this.buildCustom( + input, + this.prefix + + this.builder.prefix + + this.builder.func(input.where) + + this.builder.suffix + + this.buildOptionalOrder(input.order) + + this.buildLimit(input.limit), + ); + return this.clientQuery>(sql, annotations, 1, hints); + } + + async runBatch( + inputs: Map>, + annotations: QueryAnnotation[], + ): Promise>>> { + // SELECT '...' AS _key, ... FROM ... WHERE ... + // UNION ALL + // SELECT '...' AS _key, ... FROM ... WHERE ... + const pieces: string[] = []; + let allHints: Hints = {}; + for (const [key, input] of inputs.entries()) { + const { sql, hints } = this.buildCustom( + input, + this.prefixUnion + + escapeString(key) + + this.midfixUnion + + this.builder.prefix + + this.builder.func(input.where) + + this.builder.suffix + + this.buildOptionalOrder(input.order) + + this.buildLimit(input.limit), + ); + pieces.push("(" + sql + ")"); + allHints = { ...allHints, ...pickBy(hints, (v) => v !== undefined) }; + } + + const unionRows = await this.clientQuery<{ _key: string } & Row>( + pieces.join("\n UNION ALL\n"), + annotations, + inputs.size, + Object.keys(allHints).length > 0 ? allHints : undefined, + ); + + const outputs = new Map>>(); + for (const { _key: key, ...row } of unionRows) { + let rows = outputs.get(key); + if (!rows) { + rows = []; + outputs.set(key, rows); + } + + rows.push(row as Row); + } + + return outputs; + } + + private buildCustom( + input: SelectInput, + sql: string, + ): { + sql: string; + hints: Hints | undefined; + } { + const custom = input.custom as SelectInputCustom; + if (custom?.joins?.length) { + sql = sql.replace( + / FROM \S+\s+/, + (m) => + m + + "\n" + + custom.joins!.map((join) => escapeLiteral(join)).join("\n") + + "\n", + ); + } else if (custom?.from?.length) { + sql = sql.replace( + / FROM \S+/, + () => " FROM " + escapeLiteral(custom.from!), + ); + } + + if (custom?.ctes?.length) { + sql = + "WITH\n " + + custom.ctes.map((cte) => escapeLiteral(cte)).join(",\n ") + + "\n" + + sql; + } + + return { sql, hints: custom?.hints }; + } + + private buildOptionalOrder(order: Order | undefined): string { + if (!order) { + return ""; + } + + // TS tuples support is unfortunately weak: it has hard time treating arrays + // as tuples, and also treating strings as literal strings. E.g. we can't do: + // [["field", "ASC"], ...] + // in the caller code and have Order to be tuple-based; the only + // work-around would be + // [tuple("field" as const, "ASC" as const), ...] + // which is ugly. So we use object-based order specifiers and lots of run-time + // checks around the data passed. + const pieces: string[] = []; + for (const item of order) { + if (hasKey("$literal", item)) { + if (Object.keys(item).length > 1) { + throw Error( + "Invalid order specification - $literal must be the only key: " + + inspectCompact(item), + ); + } + + pieces.push(escapeLiteral(item.$literal)); + } else { + for (const [field, dir] of Object.entries(item)) { + if (!ALLOWED_ORDER.includes("" + dir)) { + throw Error( + `Invalid order specifier: ${dir}; allowed specifiers: ` + + ALLOWED_ORDER.join(", "), + ); + } + + pieces.push(`${this.escapeField(field)} ${dir}`); + } + } + } + + return pieces.length > 0 ? " ORDER BY " + pieces.join(", ") : ""; + } + + private buildLimit(limit: number): string { + return " LIMIT " + (parseInt("" + limit) || 0); + } +} diff --git a/src/pg/PgQuerySelectBy.ts b/src/pg/PgQuerySelectBy.ts new file mode 100644 index 0000000..aa9e588 --- /dev/null +++ b/src/pg/PgQuerySelectBy.ts @@ -0,0 +1,132 @@ +import type { Query } from "../abstract/Query"; +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import type { Schema } from "../abstract/Schema"; +import type { + FieldOfPotentialUniqueKey, + Row, + SelectByInput, + Table, + UniqueKey, +} from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQuerySelectBy< + TTable extends Table, + TUniqueKey extends UniqueKey, +> implements Query>> { + readonly IS_WRITE = false; + + constructor( + public readonly schema: Schema, + public readonly input: SelectByInput, + ) {} + + async run( + client: PgClient, + annotation: QueryAnnotation, + ): Promise>> { + // Treat undefined as an absent key. This will hopefully be JITed very + // efficiently, but could still be that it won't since we enumerate object + // keys and use [] to access the values. + const fields = this.schema.uniqueKey.filter( + (field) => this.input[field] !== undefined, + ); + + // If there are no known fields, skip the entire operation. + if (fields.length === 0) { + return []; + } + + // Since we have a partial list of fields which depends on the query itself, + // we have to cache runners per updating fields list. Else we'd not be able + // to do a partial batched update. + return client + .batcher( + this.constructor, + this.schema, + fields.join(":"), + false, + () => + // This is run only once per every unique combination of field names, + // not per every row updated, so it's cheap to do whatever we want. + new PgRunnerSelectBy(this.schema, client, fields), + ) + .run(this.input, annotation); + } +} + +class PgRunnerSelectBy< + TTable extends Table, + TUniqueKey extends UniqueKey, +> extends PgRunner< + TTable, + SelectByInput, + Array> +> { + static override readonly IS_WRITE = false; + private builders; + + readonly op = "SELECT_UNIQ_PFX"; + override readonly maxBatchSize = 1000; // Select by unique key is cheap, so we can have much bigger load batches to accumulate more data from e.g. Shard 0 for the next multi-Shard requests. + readonly default = []; // If no rows are found, returns []. + + constructor( + schema: Schema, + client: PgClient, + private fields: Array>, + ) { + super(schema, client); + this.builders = this.createWhereBuildersFieldsEq< + SelectByInput + >({ + prefix: this.fmt("SELECT %SELECT_FIELDS FROM %T "), + fields, + suffix: this.fmt(""), + }); + } + + override key(input: SelectByInput | Row): string { + return JSON.stringify(this.fields.map((field) => input[field])); + } + + async runSingle( + input: SelectByInput, + annotations: QueryAnnotation[], + ): Promise>> { + const sql = + this.builders.plain.prefix + + this.builders.plain.func([["", input]]) + + this.builders.plain.suffix; + return this.clientQuery>(sql, annotations, 1); + } + + async runBatch( + inputs: Map>, + annotations: QueryAnnotation[], + ): Promise>>> { + const sql = + this.builders.optimized.prefix + + this.builders.optimized.func(inputs) + + this.builders.optimized.suffix; + const rows = await this.clientQuery>( + sql, + annotations, + inputs.size, + ); + + const outputs = new Map>>(); + for (const row of rows) { + const key = this.key(row); + let rows = outputs.get(key); + if (!rows) { + rows = []; + outputs.set(key, rows); + } + + rows.push(row); + } + + return outputs; + } +} diff --git a/src/pg/PgQueryUpdate.ts b/src/pg/PgQueryUpdate.ts new file mode 100644 index 0000000..ac892b9 --- /dev/null +++ b/src/pg/PgQueryUpdate.ts @@ -0,0 +1,191 @@ +import uniq from "lodash/uniq"; +import type { Query } from "../abstract/Query"; +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import type { Schema } from "../abstract/Schema"; +import type { Field, Table, UpdateInput } from "../types"; +import { ID } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryUpdate implements Query { + private readonly allFields; + readonly input: UpdateInput & { [ID]: string }; + readonly IS_WRITE = true; + + constructor( + public readonly schema: Schema, + id: string, + input: UpdateInput, + ) { + // A little hack to merge the updating row with its ID. + this.input = { ...input, [ID]: id }; + this.allFields = Object.keys(this.schema.table); + } + + async run(client: PgClient, annotation: QueryAnnotation): Promise { + // Treat undefined as an absent key. This will hopefully be JITed very + // efficiently, but could still be that it won't since we enumerate object + // keys and use [] to access the values. + const fields = this.allFields.filter( + (field) => field !== ID && this.input[field] !== undefined, + ); + const casFields = this.input.$cas + ? this.allFields.filter( + (field) => field !== ID && this.input.$cas![field] !== undefined, + ) + : []; + + // If there are no known fields to update, skip the entire operation. We + // return true since we don't know whether the row is in the DB or not, so + // we assume it is. + if (fields.length === 0 && !this.input.$literal) { + return true; + } + + // An UPDATE with $literal is a little hacky: we disable batching for it, + // because we can't guarantee that the SET clause in "WITH ... VALUES ... + // UPDATE ... SET ... FROM rows" batched query will be identical for all + // input rows. + const disableBatching = !!this.input.$literal; + + // Since UPDATE has partial list of fields, we have to cache runners per + // updating fields list. Else we'd not be able to do a partial batched update. + return client + .batcher( + this.constructor, + this.schema, + fields.join(":") + ":" + casFields.join(":"), + disableBatching, + () => + // This is run only once per every unique combination of field names, + // not per every row updated, so it's cheap to do whatever we want. + new PgRunnerUpdate(this.schema, client, fields, casFields), + ) + .run(this.input, annotation); + } +} + +class PgRunnerUpdate extends PgRunner< + TTable, + UpdateInput & { [ID]: string }, + boolean +> { + static override readonly IS_WRITE = true; + private singleBuilder; + private batchBuilder; + + readonly op = "UPDATE"; + readonly maxBatchSize = 100; + readonly default = false; // If nothing is updated, we return false. + + constructor( + schema: Schema, + client: PgClient, + fieldsIn: Array>, + casFieldsIn: Array>, + ) { + super(schema, client); + + // Always include all autoUpdate fields. + const fields = uniq([ + ...fieldsIn, + ...Object.keys(this.schema.table).filter( + (field) => this.schema.table[field].autoUpdate !== undefined, + ), + ]); + const casFields = casFieldsIn.map((field) => ({ + field, + alias: `$cas.${field}`, + })); + + this.singleBuilder = { + prefix: this.fmt("UPDATE %T SET "), + func1: this.createUpdateKVsBuilder(fields), + midfix: this.fmt(" WHERE %PK="), + func2: (input: { [ID]: string }) => this.escapeValue(ID, input[ID]), + cas: + casFields.length > 0 + ? this.createValuesBuilder({ + prefix: this.fmt(" AND ROW(%FIELDS) IS NOT DISTINCT FROM ROW", { + fields: casFields.map(({ field }) => field), + normalize: true, + }), + indent: "", + fields: casFields, + suffix: "", + }) + : null, + suffix: this.fmt(` RETURNING %PK AS ${ID}`), + }; + + // There can be several updates for same id (due to batching), so returning + // all keys here. + this.batchBuilder = this.createWithBuilder({ + fields: [...this.addPK(fields, "prepend"), ...casFields], + suffix: + this.fmt( + "UPDATE %T SET %UPDATE_FIELD_VALUE_PAIRS(rows)\n" + + "FROM rows WHERE %PK(%T)=%PK(rows)", + { fields }, + ) + + (casFields.length > 0 + ? " AND " + + this.fmt("ROW(%FIELDS(%T))", { + fields: casFields.map(({ field }) => field), + normalize: true, + }) + + " IS NOT DISTINCT FROM " + + this.fmt("ROW(%FIELDS(rows))", { fields: casFields }) + : "") + + this.fmt(" RETURNING rows._key"), + }); + } + + override key(input: UpdateInput & { [ID]: string }): string { + // If we have $cas, then we disable coalescing entirely. E.g. when a client + // sends 2 updates for the same ID even for the same $cas values from the + // same Node process, we want only the 1st update to succeed and the 2nd one + // to skip (same behavior as if those 2 updates were run in different Node + // processes: coalescing must not change the concurrency guarantees). + return input.$cas ? super.key(input) : input[ID]; + } + + async runSingle( + input: UpdateInput & { [ID]: string }, + annotations: QueryAnnotation[], + ): Promise { + const literal = input.$literal; + const sql = + this.singleBuilder.prefix + + this.singleBuilder.func1(input, literal) + + this.singleBuilder.midfix + + this.singleBuilder.func2(input) + + (this.singleBuilder.cas?.prefix ?? "") + + (this.singleBuilder.cas?.func?.([["", input]]) ?? "") + + (this.singleBuilder.cas?.suffix ?? "") + + this.singleBuilder.suffix; + const rows = await this.clientQuery<{ [ID]: string }>(sql, annotations, 1); + return rows.length > 0 ? true : false; + } + + async runBatch( + inputs: Map & { [ID]: string }>, + annotations: QueryAnnotation[], + ): Promise> { + const sql = + this.batchBuilder.prefix + + this.batchBuilder.func(inputs) + + this.batchBuilder.suffix; + const rows = await this.clientQuery<{ _key: string; [ID]: string }>( + sql, + annotations, + inputs.size, + ); + const outputs = new Map(); + for (const row of rows) { + outputs.set(row._key, true); + } + + return outputs; + } +} diff --git a/src/pg/PgQueryUpsert.ts b/src/pg/PgQueryUpsert.ts new file mode 100644 index 0000000..694f0ac --- /dev/null +++ b/src/pg/PgQueryUpsert.ts @@ -0,0 +1,216 @@ +import mapValues from "lodash/mapValues"; +import omit from "lodash/omit"; +import pick from "lodash/pick"; +import pickBy from "lodash/pickBy"; +import union from "lodash/union"; +import type { Query } from "../abstract/Query"; +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import type { Schema } from "../abstract/Schema"; +import { indent, nullthrows } from "../internal/misc"; +import type { Field, InsertInput, Table } from "../types"; +import { ID } from "../types"; +import type { PgClient } from "./PgClient"; +import { PgRunner } from "./PgRunner"; + +export class PgQueryUpsert implements Query { + readonly IS_WRITE = true; + + constructor( + public readonly schema: Schema, + public readonly input: InsertInput, + ) {} + + async run(client: PgClient, annotation: QueryAnnotation): Promise { + if (!this.schema.uniqueKey.length) { + throw Error( + `Define unique key fields to use upsert for ${this.schema.name}`, + ); + } + + const fieldsWithExplicitValues = Object.keys(this.schema.table).filter( + (field) => this.input[field as keyof typeof this.input] !== undefined, + ); + return client + .batcher( + this.constructor, + this.schema, + fieldsWithExplicitValues.join(":"), + false, + () => + new PgRunnerUpsert( + this.schema, + client, + fieldsWithExplicitValues, + ), + ) + .run(this.input, annotation); + } +} + +class PgRunnerUpsert extends PgRunner< + TTable, + InsertInput, + string +> { + static override readonly IS_WRITE = true; + private builder; + + readonly op = "UPSERT"; + readonly maxBatchSize = 100; + readonly default = "never_happens"; // abstract property implementation + + constructor( + schema: Schema, + client: PgClient, + fieldsWithExplicitValues: Array>, + ) { + super(schema, client); + + const table = this.schema.table; + const uniqueKey = this.schema.uniqueKey as string[]; + const allFields = this.addPK(Object.keys(table), "prepend"); + + // We must have at least some fields in the WITH CTE, because otherwise we + // won't be able to generate FROM rows WHERE ... clause for the top UPDATE. + fieldsWithExplicitValues = union(fieldsWithExplicitValues, uniqueKey); + + const insertSelectClause = { + fields: allFields, + autos: mapValues( + omit(pick(table, allFields), fieldsWithExplicitValues), + ({ autoInsert, autoUpdate }) => autoInsert ?? autoUpdate, + ), + }; + + const updateWhereClause = { + fields: uniqueKey, + autos: mapValues( + omit(pick(table, uniqueKey), fieldsWithExplicitValues), + ({ autoInsert, autoUpdate }) => autoInsert ?? autoUpdate, + ), + }; + + const updateFields = union( + fieldsWithExplicitValues, + Object.keys(pickBy(table, ({ autoUpdate }) => autoUpdate !== undefined)), + ); + const updateSetClause = { + fields: updateFields, + autos: mapValues( + omit(pick(table, updateFields), fieldsWithExplicitValues), + ({ autoUpdate }) => autoUpdate, + ), + }; + + this.builder = this.createWithBuilder({ + fields: fieldsWithExplicitValues, + skipSorting: true, // THE ORDER MATTERS!!! See FRAGILE comment below. + suffix: + ",\nupdates AS (\n" + + indent( + this.fmt("UPDATE %T ") + + this.fmt("SET %UPDATE_FIELD_VALUE_PAIRS(rows)\n", updateSetClause) + + this.fmt( + "FROM rows WHERE %WHERE_FIELD_VALUE_PAIRS(%T,rows)\n", + updateWhereClause, + ) + + this.fmt(`RETURNING rows._key, %PK(%T) AS ${ID})`), + ) + + ",\ninserts AS (\n" + + indent( + this.fmt("INSERT INTO %T (%FIELDS)\n", { fields: allFields }) + + this.fmt("SELECT %FIELDS\n", insertSelectClause) + + "FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1\n" + + this.fmt("ON CONFLICT (%FIELDS) DO UPDATE ", { + fields: uniqueKey, + }) + + this.fmt( + "SET %UPDATE_FIELD_VALUE_PAIRS(EXCLUDED)\n", + updateSetClause, + ) + + this.fmt(`RETURNING NULL AS _key, %PK AS ${ID})`), + ) + + `\nSELECT _key, ${ID} FROM updates UNION ALL SELECT _key, ${ID} FROM inserts`, + }); + } + + override key(inputIn: InsertInput): string { + const input: Partial> = inputIn; + const key: unknown[] = []; + for (const field of this.schema.uniqueKey) { + key.push( + input[field] === null || input[field] === undefined + ? { guaranteed_unique_value: super.key(inputIn) } + : input[field], + ); + } + + return JSON.stringify(key); + } + + async runSingle( + input: InsertInput, + annotations: QueryAnnotation[], + ): Promise { + const sql = + this.builder.prefix + + this.builder.func([["", input]]) + + this.builder.suffix; + const rows = await this.clientQuery<{ _key: string; [ID]: string }>( + sql, + annotations, + 1, + ); + return nullthrows(rows[0], sql)[ID]; + } + + async runBatch( + inputs: Map>, + annotations: QueryAnnotation[], + ): Promise> { + const sql = + this.builder.prefix + this.builder.func(inputs) + this.builder.suffix; + + const rows = await this.clientQuery<{ _key: string; [ID]: string }>( + sql, + annotations, + inputs.size, + ); + + if (rows.length !== inputs.size) { + throw Error( + `BUG: number of rows returned from upsert (${rows.length}) ` + + `is different from the number of input rows (${inputs.size}): ${sql}`, + ); + } + + const outputs = new Map(); + + // First, extract all top-level UPDATEd rows, we know their keys. + const inputsWithNullRowKeys = new Map(inputs); + const rowsWithNullKey = []; + for (const row of rows) { + if (row._key !== null) { + outputs.set(row._key, row[ID]); + inputsWithNullRowKeys.delete(row._key); + } else { + rowsWithNullKey.push(row); + } + } + + // FRAGILE! Then, extract INSERTed or on-conflict UPDATEd rows, we don't + // know their keys. In case insert didn't happen in "INSERT ... ON CONFLICT + // DO UPDATE ... RETURNING ..." clause, we can't match the updated row id + // with the key: one can only pull the fields of the updated table in + // RETURNING, where _key field just doesn't exist. Luckily, the order of + // rows returned is the same as the input rows order, and "ON CONFLICT DO + // UPDATE" update always succeeds entirely (or fails entirely). + let i = 0; + for (const key of inputsWithNullRowKeys.keys()) { + outputs.set(key, rowsWithNullKey[i][ID]); + i++; + } + + return outputs; + } +} diff --git a/src/pg/PgRunner.ts b/src/pg/PgRunner.ts new file mode 100644 index 0000000..4220181 --- /dev/null +++ b/src/pg/PgRunner.ts @@ -0,0 +1,1188 @@ +import assert from "assert"; +import difference from "lodash/difference"; +import last from "lodash/last"; +import random from "lodash/random"; +import uniq from "lodash/uniq"; +import type { QueryAnnotation } from "../abstract/QueryAnnotation"; +import { Runner } from "../abstract/Runner"; +import type { Schema } from "../abstract/Schema"; +import { hasKey } from "../internal/misc"; +import type { + Field, + FieldAliased, + Hints, + Literal, + Table, + Value, + Where, +} from "../types"; +import { ID } from "../types"; +import { escapeIdent } from "./helpers/escapeIdent"; +import { escapeLiteral } from "./helpers/escapeLiteral"; +import { escapeBoolean } from "./internal/escapeBoolean"; +import { escapeComposite } from "./internal/escapeComposite"; +import { escapeDate } from "./internal/escapeDate"; +import { escapeID } from "./internal/escapeID"; +import { escapeIdentComposite } from "./internal/escapeIdentComposite"; +import { escapeString } from "./internal/escapeString"; +import { escapeStringify } from "./internal/escapeStringify"; +import { parseCompositeRow } from "./internal/parseCompositeRow"; +import type { PgClient } from "./PgClient"; +import { PgError } from "./PgError"; + +const DEADLOCK_RETRY_MS_MIN = 2000; +const DEADLOCK_RETRY_MS_MAX = 5000; +const ERROR_DEADLOCK = "deadlock detected"; +const ERROR_FK = "violates foreign key constraint "; +const ERROR_CONFLICT_RECOVERY = + "canceling statement due to conflict with recovery"; +// "Class 22 — Data Exception" errors are typically caused by invalid +// input values (e.g. invalid date format or type cast). See for details: +// https://www.postgresql.org/docs/14/errcodes-appendix.html +const ERROR_CODE_PREFIX_DATA_EXCEPTION = "22"; + +/** + * A convenient pile of helper methods usable by most of PgQuery* classes. In + * some sense it's an anti-pattern, but still reduces the boilerplate. + * + * PgRunner is also responsible for stringifying the values passed to the + * queries and parsing values returned from the DB according to the field types + * specs. + */ +export abstract class PgRunner< + TTable extends Table, + TInput, + TOutput, +> extends Runner { + private escapers: Partial string>> = {}; + private oneOfBuilders: Partial< + Record string> + > = {}; + private dbValueToJs: Array<[string, (v: unknown) => unknown]> = []; + private stringify: Partial string>> = {}; + + declare ["constructor"]: typeof PgRunner; + + protected async clientQuery( + sql: string, + annotations: QueryAnnotation[], + batchFactor: number, + hints?: Hints, + ): Promise { + const rows = await this.client.query({ + query: [sql], + hints, + isWrite: this.constructor.IS_WRITE, + annotations, + op: this.op, + table: this.name, + batchFactor, + }); + + // Apply parsers only for known field names. Notice that TOutput is not + // necessarily a type of the table's row, it can be something else (in e.g. + // INSERT or DELETE operations). + if (rows.length > 0) { + for (const [field, dbValueToJs] of this.dbValueToJs) { + if (field in rows[0]) { + for (const row of rows) { + const dbValue = row[field as keyof TOutput]; + if (dbValue !== null && dbValue !== undefined) { + (row as Record)[field] = dbValueToJs(dbValue); + } + } + } + } + } + + return rows; + } + + /** + * Formats prefixes/suffixes of various compound SQL clauses. Don't use on + * performance-critical path! + */ + protected fmt( + template: string, + args: { + fields?: Array>; + autos?: Partial>; + normalize?: boolean; + } = {}, + ): string { + return template.replace( + /%(?:T|SELECT_FIELDS|FIELDS|UPDATE_FIELD_VALUE_PAIRS|WHERE_FIELD_VALUE_PAIRS|PK)(?:\(([%\w,]+)\))?/g, + (c: string, a?: string) => { + a = a?.replace(/%T/g, this.name); + + // Table name. + if (c === "%T") { + return escapeIdent(this.name); + } + + // Comma-separated list of ALL fields in the table to be used in SELECT + // clauses (always includes ID field). + if (c === "%SELECT_FIELDS") { + return uniq([...Object.keys(this.schema.table), ID]) + .map((f) => this.escapeField(f) + (f === ID ? ` AS ${ID}` : "")) + .join(", "); + } + + // Comma-separated list of the passed fields (never with AS clause). If + // `autos` is passed, then the corresponding SQL from `autos` is used + // instead of the field's name if that field is in `autos`. + if (c.startsWith("%FIELDS")) { + assert(args.fields, `BUG: no args.fields passed in ${template}`); + return args.fields + .map( + (field) => + args.autos?.[typeof field === "string" ? field : field.field] ?? + this.escapeField(field, { + withTable: a, + normalize: args.normalize, + }), + ) + .join(", "); + } + + // - when a="X": field1=X.field1, field2=X.field2, ... + // - when a="X,Y": X.field1=Y.field1, X.field2=Y.field2, ... + // - If `autos` is passed, then the corresponding SQL from `autos` is + // used instead of the field's name on the right side of "=" if that + // field is in `autos`. + // - %WHERE_FIELD_VALUE_PAIRS is the same, but uses "AND" as a + // separator, not ",". + if ( + c.startsWith("%UPDATE_FIELD_VALUE_PAIRS") || + c.startsWith("%WHERE_FIELD_VALUE_PAIRS") + ) { + assert(args.fields, `BUG: no args.fields passed in ${template}`); + assert( + a, + "BUG: you must pass an argument, source table alias name or a comma-separated pair of aliases", + ); + const [left, right] = a.includes(",") + ? a.split(",").map((s) => s.trim()) + : [undefined, a]; + return args.fields + .map( + (field) => + this.escapeField(field, { withTable: left }) + + "=" + + (args.autos?.[ + typeof field === "string" ? field : field.field + ] ?? this.escapeField(field, { withTable: right })), + ) + .join(c.startsWith("%UPDATE_FIELD_VALUE_PAIRS") ? ", " : " AND "); + } + + // Primary key (simple or composite). + if (c.startsWith("%PK")) { + return this.escapeField(ID, { withTable: a }); + } + + throw Error(`Unknown format spec "${c}" in "${template}"`); + }, + ); + } + + /** + * Escapes a value at runtime using the codegen functions created above. We + * use escapers table and the codegen for the following reasons: + * 1. We want to be sure that we know in advance, how to escape all table + * fields (and not fail at runtime). + * 2. We want to make createEscapeCode() the single source of truth about + * fields escaping, even at runtime. + */ + protected escapeValue(field: Field, value: unknown): string { + const escaper = this.nullThrowsUnknownField(this.escapers[field], field); + return escaper(value); + } + + /** + * Escapes field name identifier. + * - In case it's a composite primary key, returns its `ROW(f1,f2,...)` + * representation. + * - A field may be aliased, e.g. if `{ field: "abc", alias: "$cas.abc" }` is + * passed, then the returned value will be `"$cas.abc"`. Basically, `field` + * name is used only to verify that such field is presented in the schema. + * - If `autos` is passed, and the field is in it, returns the value of the + * `autos` record for the field. This is used to inject an + * autoInsert/autoUpdate SQL expression at the position of some field. + */ + protected escapeField( + info: FieldAliased, + { withTable, normalize }: { withTable?: string; normalize?: boolean } = {}, + ): string { + const [field, alias] = + typeof info === "string" ? [info, info] : [info.field, info.alias]; + + if (this.schema.table[field]) { + const sql = withTable + ? `${escapeIdent(withTable)}.` + escapeIdent(alias) + : escapeIdent(alias); + return normalize ? this.normalizeSQLExpr(field, sql) : sql; + } + + if (field === ID) { + return escapeIdentComposite(this.schema.uniqueKey, withTable); + } + + return this.nullThrowsUnknownField(null, field); + } + + /** + * Returns a newly created JS function which, when called with a row set, + * returns the following SQL clause: + * + * ``` + * WITH rows(id, a, b, _key) AS (VALUES + * ((NULL::tbl).id, (NULL::tbl).a, (NULL::tbl).b, 'k0'), + * ('123', 'xyz', 'nn', 'kSome'), + * ('456', 'abc', 'nn', 'kOther'), + * ... + * ) + * {suffix} + * ``` + * + * For composite primary key, its parts (fields) are always prepended. The set + * of columns is passed in specs. + */ + protected createWithBuilder({ + fields, + skipSorting, + suffix, + }: { + fields: ReadonlyArray>; + skipSorting?: boolean; + suffix: string; + }): { + prefix: string; + func: (entries: Iterable<[key: string, input: object]>) => string; + suffix: string; + } { + const cols = [ + ...fields.map((info) => { + const [field, alias] = + typeof info === "string" ? [info, info] : [info.field, info.alias]; + return { + field: escapeIdent(alias), + escapedValue: this.fmt("(NULL::%T).") + this.escapeField(field), + }; + }), + { field: "_key", escapedValue: "'k0'" }, + ]; + + const [preSuffix, postSuffix] = suffix.startsWith(",") + ? [",", suffix.substring(1).replace(/^\s*\n/s, "")] + : ["", suffix]; + + // We prepend VALUES with a row which consists of all NULL values, but typed + // to the actual table's columns types. This hints PG how to cast input. + return this.createValuesBuilder({ + prefix: + `WITH rows(${cols.map(({ field }) => field).join(", ")}) AS (VALUES\n` + + ` (${cols.map(({ escapedValue }) => escapedValue).join(", ")}),`, + indent: " ", + fields, + withKey: true, + skipSorting, + suffix: ")" + preSuffix + "\n" + postSuffix.replace(/^/gm, " "), + }); + } + + /** + * Returns a newly created JS function which, when called with a row set, + * returns the following SQL clause: + * + * When called with withKey=true: + * + * ``` + * ('123', 'xyz', 'nn', 'kSome'), + * ('456', 'abc', 'nn', 'kOther'), + * ``` + * + * When called with withKey=true, but with empty `fields`: + * + * ``` + * ('kSome'), + * ('kOther'), + * ``` + * + * When called with withKey=false: + * + * ``` + * ('123', 'xyz', 'nn'), + * ('456', 'abc', 'nn'), + * ``` + * + * The list of column names is passed in `fields`. + * + * When the builder func is called, the actual values for some field in a row + * is extracted from the same-named prop of the row, but if `{ field, alias }` + * object is passed in `fields` array, then the value is extracted from the + * `alias` sub-prop of the row. This is used to e.g. access `row.$cas.blah` + * value for a field named blah (in this case, `alias="$cas"`). + * + * Notice that either a simple primary key or a composite primary key columns + * are always prepended to the list of values since it makes no sense to + * generate VALUES clause without exact identification of the destination. + */ + protected createValuesBuilder({ + prefix, + indent, + fields, + withKey, + skipSorting, + suffix, + }: { + prefix: string; + indent: string; + fields: ReadonlyArray>; + withKey?: boolean; + skipSorting?: boolean; + suffix: string; + }): { + prefix: string; + func: (entries: Iterable<[key: string, input: TInput]>) => string; + suffix: string; + } { + const cols = fields.map((info) => { + const [field, fieldValCode] = + typeof info === "string" + ? [info, `$input.${info}`] + : [info.field, `$input.${info.alias}`]; + const spec = this.nullThrowsUnknownField(this.schema.table[field], field); + return this.createEscapeCode( + field, + fieldValCode, + spec.autoInsert !== undefined ? spec.autoInsert : spec.autoUpdate, + ); + }); + const rowFunc = this.newFunction( + "$key", + "$input", + "return " + + (indent ? `${JSON.stringify("\n" + indent)} + ` : "") + + '"("' + + (cols.length > 0 ? " + " + cols.join(" + ', ' + ") : "") + + (withKey + ? (cols.length > 0 ? ' + ", "' : "") + " + this.escapeString($key)" + : "") + + ' + ")"', + ); + + return { + prefix, + func: (entries: Iterable<[key: string, input: TInput]>) => { + const parts: string[] = []; + for (const [key, input] of entries) { + parts.push(rowFunc(key, this.unfoldCompositePK(input))); + } + + // To eliminate deadlocks in parallel batched inserts, we sort rows. + // This prevents deadlocks when two batched queries are running in + // different connections, and the table has some unique key. + if (!skipSorting) { + parts.sort(); + } + + return parts.join(","); + }, + suffix, + }; + } + + /** + * Returns a newly created JS function which, when called with an object, + * returns the following SQL clause: + * + * id='123', a='xyz', b='nnn' [, {literal}] + * + * The set of columns is passed in specs, all other columns are ignored. + */ + protected createUpdateKVsBuilder( + fields: Array>, + ): (input: object, literal?: Literal) => string { + const parts = fields.map( + (field) => + JSON.stringify(this.escapeField(field) + "=") + + " + " + + this.createEscapeCode( + field, + `$input.${field}`, + this.schema.table[field].autoUpdate, + ), + ); + const func = this.newFunction( + "$input", + "return " + (parts.length ? parts.join(" + ', ' + ") : '""'), + ); + return (input: object, literal?: Literal): string => { + const kvs = func(input); + const custom = literal ? escapeLiteral(literal) : ""; + return kvs && custom ? `${kvs}, ${custom}` : kvs ? kvs : custom; + }; + } + + /** + * Prefers to do utilize createAnyBuilder() if it can (i.e. build + * a=ANY('{...}') clause). Otherwise, builds an IN(...) clause. + */ + protected createOneOfBuilder( + field: Field, + fieldValCode = "$value", + ): (values: Iterable) => string { + const specType = this.schema.table[field]?.type; + return specType === Boolean || + specType === ID || + specType === Number || + specType === String + ? this.createAnyBuilder(field, fieldValCode) + : this.createInBuilder(field, fieldValCode); + } + + /** + * Given a list of fields, returns two builders: + * + * 1. "Optimized": a newly created JS function which, when called with a row + * set, returns one the following SQL clauses: + * + * ``` + * WHERE (field1, field2) IN(VALUES + * ((NULL::tbl).field1, (NULL::tbl).field2), + * ('aa', 'bb'), + * ('cc', 'dd')) + * + * or + * + * WHERE (field1='a' AND field2='b' AND field3 IN('a', 'b', 'c', ...)) OR (...) + * ^^^^^^^^^^prefix^^^^^^^^^ ^^^^^^^^ins^^^^^^^ + * ``` + * + * 2. "Plain": the last one builder mentioned above (good to always use for + * non-batched queries for instance). + */ + protected createWhereBuildersFieldsEq(args: { + prefix: string; + fields: ReadonlyArray>; + suffix: string; + }): { + plain: { + prefix: string; + func: (inputs: Iterable<[key: string, input: TInput]>) => string; + suffix: string; + }; + optimized: { + prefix: string; + func: (inputs: Iterable<[key: string, input: TInput]>) => string; + suffix: string; + }; + } { + const plain = this.createWhereBuilderFieldsEqOrBased(args); + return { + plain, + optimized: + args.fields.length > 1 && + args.fields.every((field) => !this.schema.table[field].allowNull) + ? this.createWhereBuilderFieldsEqTuplesBased(args) + : plain, + }; + } + + /** + * Returns a newly created JS function which, when called with a Where object, + * returns the generated SQL WHERE clause. + * + * - The building is relatively expensive, since it traverses the Where object + * at run-time and doesn't know the shape beforehand. + * - If the Where object is undefined, skips the entire WHERE clause. + */ + protected createWhereBuilder({ + prefix, + suffix, + }: { + prefix: string; + suffix: string; + }): { + prefix: string; + func: (where: Where) => string; + suffix: string; + } { + return { + prefix: prefix + "WHERE ", + func: (where: Where) => + this.buildWhere(this.schema.table, where, true), + suffix, + }; + } + + /** + * Prepends or appends a primary key to the list of fields. In case the + * primary key is plain (i.e. "id" field), it's just added as a field; + * otherwise, the unique key fields are added. + * + * For INSERT/UPSERT operations, we want to append the primary key, since it's + * often types pre-generated as a random-looking value. In many places, we + * sort batched lists of rows before e.g. inserting them, so we order them by + * their natural data order which prevents deadlocks on unique key conflict + * when multiple concurrent transactions try to insert the same set of rows in + * different order ("while inserting index tuple"). + * + * For UPDATE operations though, we want to prepend the primary key, to make + * sure we run batched updates in the same order in multiple concurrent + * transactions. This lowers the chances of deadlocks too. + */ + protected addPK( + fields: ReadonlyArray>, + mode: "prepend" | "append", + ): string[] { + const pkFields = this.schema.table[ID] ? [ID] : this.schema.uniqueKey; + fields = difference(fields, pkFields); + return mode === "prepend" + ? [...pkFields, ...fields] + : [...fields, ...pkFields]; + } + + constructor( + public readonly schema: Schema, + private client: PgClient, + ) { + super(schema.name); + + // For tables with composite primary key and no explicit "id" column, we + // still need an ID escaper (where id looks like "(1,2)" anonymous row). + for (const field of [ID, ...Object.keys(this.schema.table)]) { + const body = "return " + this.createEscapeCode(field, "$value"); + this.escapers[field] = this.newFunction("$value", body); + this.oneOfBuilders[field] = this.createOneOfBuilder(field); + } + + for (const [field, { type }] of Object.entries(this.schema.table)) { + if (hasKey("dbValueToJs", type) && hasKey("stringify", type)) { + this.dbValueToJs.push([field, type.dbValueToJs.bind(type)]); + this.stringify[field] = type.stringify.bind(type); + } + } + } + + delayForSingleQueryRetryOnError( + e: unknown, + ): number | "immediate_retry" | "no_retry" { + // Deadlocks may happen when a simple query involves multiple rows (e.g. + // deleting a row by ID, but this row has foreign keys, especially with ON + // DELETE CASCADE). + return e instanceof PgError && e.message.includes(ERROR_DEADLOCK) + ? random(DEADLOCK_RETRY_MS_MIN, DEADLOCK_RETRY_MS_MAX) + : e instanceof PgError && e.message.includes(ERROR_CONFLICT_RECOVERY) + ? "immediate_retry" + : "no_retry"; + } + + shouldDebatchOnError(e: unknown): boolean { + return ( + // Debatch some of SQL WRITE query errors. + (e instanceof PgError && e.message.includes(ERROR_DEADLOCK)) || + (e instanceof PgError && e.message.includes(ERROR_FK)) || + // Debatch "conflict with recovery" errors (we support retries only after + // debatching, so have to return true here). + (e instanceof PgError && e.message.includes(ERROR_CONFLICT_RECOVERY)) || + (e instanceof PgError && + !!e.cause?.code?.startsWith(ERROR_CODE_PREFIX_DATA_EXCEPTION)) + ); + } + + /** + * Given a list of fields, returns a newly created JS function which, when + * called with a row set, returns the following SQL clause: + * + * ``` + * WHERE (field1='a' AND field2='b' AND field3 IN('a', 'b', 'c', ...)) OR (...) + * ^^^^^^^^^^prefix^^^^^^^^^ ^^^^^^^^ins^^^^^^^ + * ``` + * + * The assumption is that the last field in the list is the most variable, + * whilst all previous fields compose a more or less static prefix + * + * - ATTENTION: if at least one OR is produced, it will likely result in a + * slower Bitmap Index Scan. + * - Used in runSingle() (no ORs there) or when optimized builder is not + * available (e.g. when unique key contains nullable fields). + */ + private createWhereBuilderFieldsEqOrBased({ + prefix, + fields, + suffix, + }: { + prefix: string; + fields: ReadonlyArray>; + suffix: string; + }): { + prefix: string; + func: (inputs: Iterable<[key: string, input: TInput]>) => string; + suffix: string; + } { + const lastField = last(fields)!; + + // fieldN IN('aa', 'bb', 'cc', ...) + const lastFieldOneOf = this.createOneOfBuilder( + lastField, + `$value[1].${lastField}`, + ); + + if (fields.length === 1) { + // If we have only one field, we can use the plain oneOfBuilder (which is + // either an IN(...) or =ANY(...) clause). + return { + prefix: prefix + "WHERE ", + func: lastFieldOneOf, + suffix, + }; + } + + return { + prefix: prefix + "WHERE ", + func: (inputs: Iterable<[key: string, input: TInput]>) => { + const insByPrefix = new Map< + string, + Array<[key: string, input: TInput]> + >(); + for (const input of inputs) { + let prefix = ""; + for (let i = 0; i < fields.length - 1; i++) { + const field = fields[i]; + if (prefix !== "") { + prefix += " AND "; + } + + const value = (input[1] as Record)[field]; + prefix += + value !== null + ? field + "=" + this.escapeValue(field, value) + : field + " IS NULL"; + } + + let ins = insByPrefix.get(prefix); + if (!ins) { + ins = []; + insByPrefix.set(prefix, ins); + } + + ins.push(input); + } + + let sql = ""; + for (const [prefix, ins] of insByPrefix) { + if (sql !== "") { + sql += " OR "; + } + + const inClause = lastFieldOneOf(ins); + if (prefix !== "") { + sql += "(" + prefix + " AND " + inClause + ")"; + } else { + sql += inClause; + } + } + + return sql; + }, + suffix, + }; + } + + /** + * Given a list of fields, returns a newly created JS function which, when + * called with a row set, returns the following SQL clause: + * + * ``` + * WHERE (field1, field2) IN(VALUES + * ((NULL::tbl).field1, (NULL::tbl).field2), + * ('aa', 'bb'), + * ('cc', 'dd')) + * ``` + * + * The assumption is that all fields are non-nullable. + * + * - This clause always produces an Index Scan (not Bitmap Index Scan). + * - Used in most of the cases in runBatch(), e.g. when unique key has >1 + * fields, and they are all non-nullable. + */ + private createWhereBuilderFieldsEqTuplesBased({ + prefix, + fields, + suffix, + }: { + prefix: string; + fields: ReadonlyArray>; + suffix: string; + }): { + prefix: string; + func: (entries: Iterable<[key: string, input: TInput]>) => string; + suffix: string; + } { + const escapedFields = fields.map((f) => this.escapeField(f)); + return this.createValuesBuilder({ + prefix: + prefix + + `WHERE (${escapedFields.join(", ")}) IN(VALUES\n` + + " (" + + escapedFields.map((f) => this.fmt(`(NULL::%T).${f}`)).join(", ") + + "),", + indent: " ", + fields, + skipSorting: true, // for JS perf + suffix: ")" + suffix, + }); + } + + private buildWhere( + specs: TTable, + where: Where, + isTopLevel: boolean = false, + ): string { + const pieces: string[] = []; + for (const key of Object.keys(where)) { + const value = where[key]; + if (value === undefined) { + continue; + } + + if (key[0] === "$") { + continue; + } + + let foundOp = false; + if (hasKey("$gte", value)) { + pieces.push(this.buildFieldBinOp(key, ">=", value.$gte)); + foundOp = true; + } + + if (hasKey("$gt", value)) { + pieces.push(this.buildFieldBinOp(key, ">", value.$gt)); + foundOp = true; + } + + if (hasKey("$lte", value)) { + pieces.push(this.buildFieldBinOp(key, "<=", value.$lte)); + foundOp = true; + } + + if (hasKey("$lt", value)) { + pieces.push(this.buildFieldBinOp(key, "<", value.$lt)); + foundOp = true; + } + + if (hasKey("$ne", value)) { + pieces.push(this.buildFieldNe(key, value.$ne)); + foundOp = true; + } + + if (hasKey("$isDistinctFrom", value)) { + pieces.push(this.buildFieldIsDistinctFrom(key, value.$isDistinctFrom)); + foundOp = true; + } + + if (hasKey("$overlap", value)) { + pieces.push(this.buildFieldBinOp(key, "&&", value.$overlap)); + foundOp = true; + } + + if (!foundOp) { + pieces.push(this.buildFieldEq(key, value)); + } + } + + if (hasKey("$and", where)) { + const clause = this.buildLogical(specs, "AND", where.$and); + if (clause.length) { + pieces.push(clause); + } + } + + if (hasKey("$or", where)) { + const clause = this.buildLogical(specs, "OR", where.$or); + if (clause.length) { + pieces.push(clause); + } + } + + if (hasKey("$not", where)) { + pieces.push(this.buildNot(specs, where.$not)); + } + + if (hasKey("$literal", where)) { + // $literal clause in WHERE may look like "abc OR def", and to make sure + // this OR doesn't interfere with priorities of other operators around, we + // always wrap the literal with (). We must wrap in WHERE only, not in + // e.g. ORDER BY or CTEs. + pieces.push("(" + escapeLiteral(where.$literal) + ")"); + } + + if (!pieces.length) { + // This is for cases like { [$and]: [{}, {}] } + pieces.push("true"); + } + + const sql = pieces.join(" AND "); + return pieces.length > 1 && !isTopLevel ? "(" + sql + ")" : sql; + } + + private buildFieldBinOp>( + field: TField, + binOp: string, + value: NonNullable>, + ): string { + return this.escapeField(field) + binOp + this.escapeValue(field, value); + } + + private buildFieldIsDistinctFrom>( + field: TField, + value: Value, + ): string { + return ( + this.escapeField(field) + + " IS DISTINCT FROM " + + this.escapeValue(field, value) + ); + } + + private buildFieldEq>( + field: TField, + value: Where[TField], + ): string { + if (value === null) { + return this.escapeField(field) + " IS NULL"; + } else if (value instanceof Array) { + const inBuilder = this.nullThrowsUnknownField( + this.oneOfBuilders[field], + field, + ); + return inBuilder(value); + } else { + return this.escapeField(field) + "=" + this.escapeValue(field, value); + } + } + + private buildLogical( + specs: TTable, + op: "OR" | "AND", + items: ReadonlyArray>, + ): string { + const clause = op === "OR" ? " OR " : " AND "; + if (items.length === 0) { + return ` false /* Empty${clause}*/ `; + } + + const sql = items.map((item) => this.buildWhere(specs, item)).join(clause); + return items.length > 1 ? "(" + sql + ")" : sql; + } + + private buildNot(specs: TTable, where: Where): string { + return "NOT " + this.buildWhere(specs, where); + } + + private buildFieldNe>( + field: TField, + value: Value | ReadonlyArray>, + ): string { + if (value === null) { + return this.escapeField(field) + " IS NOT NULL"; + } else if (value instanceof Array) { + let andIsNotNull = false; + const pieces: string[] = []; + for (const v of value) { + if (v === null) { + andIsNotNull = true; + } else { + pieces.push(this.escapeValue(field, v)); + } + } + + const sql = pieces.length + ? this.escapeField(field) + " NOT IN(" + pieces.join(",") + ")" + : "true/*empty_NOT_IN*/"; + return andIsNotNull + ? "(" + sql + " AND " + this.escapeField(field) + " IS NOT NULL)" + : sql; + } else { + return this.escapeField(field) + "<>" + this.escapeValue(field, value); + } + } + + /** + * Returns a newly created JS function which, when called with an array of + * values, returns one of following SQL clauses: + * + * - $field=ANY('{aaa,bbb,ccc}') + * - ($field=ANY('{aaa,bbb}') OR $field IS NULL) + * - $field='aaa' (see below, why) + * - ($field='aaa' OR $field IS NULL) + * - $field IS NULL + * - false + */ + private createAnyBuilder( + field: Field, + fieldValCode = "$value", + ): ($values: Iterable) => string { + // Notes: + // + // - See arrayfuncs.c, array_out() function (needquote logic): + // https://github.com/postgres/postgres/blob/4ddfbd2/src/backend/utils/adt/arrayfuncs.c#L1136-L1156 + // - Why will it work not worse (index wise) than multi-value IN(): + // https://www.postgresql.org/message-id/1761901.1668657080%40sss.pgh.pa.us + // - We can't easily use a general-purpose quoting function here, because we + // must exclude nulls from the values, to add an explicit "OR IS NULL" + // clause instead. + // - We sacrifice performance a little and not quote everything blindly. + // This is to gain the generated SQL queries some more readability. + // + // Also one more thing. Imagine we have a `btree(a, b)` index. Compare two + // queries for one-element use case: + // + // 1. `a='aaa' AND b=ANY('{bbb}')` + // 2. `a='aaa' AND b IN('bbb')` + // + // They may produce different plans: IN() always coalesces to `b='bbb'` in + // the plan (and thus, to an btree index scan), whilst =ANY() always remains + // =ANY(). This causes PG to choose a "post-filtering" plan for one-element + // use case sometimes: + // + // 1. For =ANY: Index Cond: (a='aaa'); Filter: b=ANY('{bbb}') - BAD! + // 2. For IN(): Index Cond: (a='aaa') AND (b='bbb') + // + // So to be on a safe side, we never emit a one-element =ANY(); instead, we + // turn `b=ANY('{bbb}')` into `b='bbb'`. + // + const escapedFieldCode = JSON.stringify(this.escapeField(field)); + const body = ` + let sql = ''; + let lastValue = null; + let nonNullCount = 0; + let hasIsNull = false; + for (const $value of $values) { + if (${fieldValCode} != null) { + if (sql) sql += ','; + nonNullCount++; + lastValue = "" + ${fieldValCode}; + sql += lastValue.match(/^$|^NULL$|[ \\t\\n\\r\\v\\f]|["\\\\{},]/is) + ? '"' + lastValue.replace(/\\\\/g, '\\\\\\\\').replace(/"/g, '\\\\"') + '"' + : lastValue; + } else { + hasIsNull = true; + } + } + if (sql) { + if (nonNullCount > 1) { + sql = '{' + sql + '}'; + sql = ${escapedFieldCode} + '=ANY(' + this.escapeString(sql) + ')'; + } else { + sql = ${escapedFieldCode} + '=' + this.escapeString(lastValue); + } + } + return sql && hasIsNull + ? '(' + sql + ' OR ' + ${escapedFieldCode} + ' IS NULL)' + : hasIsNull + ? ${escapedFieldCode} + ' IS NULL' + : sql + ? sql + : 'false/*empty_ANY*/'; + `; + return this.newFunction("$values", body); + } + + /** + * Returns a newly created JS function which, when called with an array of + * values, returns one of following SQL clauses: + * + * - $field IN('aaa', 'bbb', 'ccc') + * - ($field IN('aaa', 'bbb') OR $field IS NULL) + * - $field IS NULL + * - false + * + * This only works for primitive types. + */ + private createInBuilder( + field: Field, + fieldValCode = "$value", + ): ($values: Iterable) => string { + const escapedFieldCode = JSON.stringify(this.escapeField(field)); + const valueCode = this.createEscapeCode(field, fieldValCode); + const body = ` + let sql = ''; + let hasIsNull = false; + for (const $value of $values) { + if (${fieldValCode} != null) { + if (sql) sql += ','; + sql += ${valueCode}; + } else { + hasIsNull = true; + } + } + if (sql) { + sql = ${escapedFieldCode} + ' IN(' + sql + ')'; + } + return sql && hasIsNull + ? '(' + sql + ' OR ' + ${escapedFieldCode} + ' IS NULL)' + : hasIsNull + ? ${escapedFieldCode} + ' IS NULL' + : sql + ? sql + : 'false/*empty_IN*/'; + `; + return this.newFunction("$values", body); + } + + /** + * For codegen, returns the following piece of JS code: + * + * '($fieldValCode !== undefined ? this.escapeXyz($fieldValCode) : "$defSQL")' + * + * It's expected that, while running the generated code, `this` points to an + * object with a) `escapeXyz()` functions, b) `stringify` object containing + * the table fields custom to-string converters. + */ + private createEscapeCode( + field: Field, + fieldValCode: string, + defSQL?: string, + ): string { + const specType = this.schema.table[field]?.type; + if (!specType && field !== ID) { + throw Error(`BUG: cannot find the field "${field}" in the schema`); + } + + const escapeCode = + specType === undefined && field === ID + ? `this.escapeComposite(${fieldValCode})` + : specType === Boolean + ? `this.escapeBoolean(${fieldValCode})` + : specType === Date + ? `this.escapeDate(${fieldValCode}, ${JSON.stringify(field)})` + : specType === ID + ? `this.escapeID(${fieldValCode})` + : specType === Number + ? `this.escapeString(${fieldValCode})` + : specType === String + ? `this.escapeString(${fieldValCode})` + : hasKey("stringify", specType) + ? `this.escapeStringify(${fieldValCode}, this.stringify.${field})` + : (() => { + throw Error( + `BUG: unknown spec type ${specType} for field ${field}`, + ); + })(); + if (defSQL !== undefined) { + return ( + `(${fieldValCode} !== undefined ` + + `? ${escapeCode} ` + + `: ${JSON.stringify(defSQL)})` + ); + } else { + return escapeCode; + } + } + + /** + * Compiles a function body with `this` bound to some well-known properties + * which are available in the body. + * + * For each table, we compile frequently accessible pieces of code which + * serialize data in SQL format. This allows to remove lots of logic and "ifs" + * from runtime and speed up hot code paths. + */ + private newFunction(...argsAndBody: string[]): (...args: unknown[]) => never { + return new Function(...argsAndBody).bind({ + escapeComposite, + escapeBoolean, + escapeDate, + escapeID, + escapeString, + escapeStringify, + stringify: this.stringify, + }); + } + + /** + * The problem: PG is not working fine with queries like: + * + * ``` + * WITH rows(composite_id, c) AS ( + * VALUES + * ( ROW((NULL::tbl).x, (NULL::tbl).y), (NULL::tbl).c ), + * ( ROW(1,2), 3 ), + * ( ROW(3,4), 5 ) + * ) + * UPDATE tbl SET c=rows.c + * FROM rows WHERE ROW(tbl.x, tbl.y)=composite_id + * ``` + * + * It cannot match the type of composite_id with the row, and even the trick + * with NULLs doesn't help it to infer types. It's a limitation of WITH clause + * (because in INSERT ... VALUES, there is no such problem). + * + * So the only solution is to parse/decompose the row string into individual + * unique key columns at runtime for batched UPDATEs. And yes, it's slow. + * + * ``` + * WITH rows(x, y, c) AS ( + * VALUES + * ( (NULL::tbl).x, (NULL::tbl).y, (NULL::tbl).c ), + * ( 1, 2, 3 ), + * ( 3, 4, 5 ) + * ) + * UPDATE tbl SET c=rows.c + * FROM rows WHERE ROW(tbl.x, tbl.y)=ROW(rows.x, ROW.y) + * ``` + */ + private unfoldCompositePK(inputIn: TInput): TInput { + let input = inputIn as Record; + if (!this.schema.table[ID] && typeof input[ID] === "string") { + const compositePK = parseCompositeRow(input[ID]); + input = { ...input }; + for (const [i, field] of this.schema.uniqueKey.entries()) { + input[field] = compositePK[i]; + } + } + + return input as TInput; + } + + /** + * Some data types are different between PG and JS. Here we have a chance to + * "normalize" them. E.g. in JS, Date is truncated to milliseconds (3 digits), + * whilst in PG, it's by default of 6 digits precision (so if we didn't + * normalize, then JS Date would've been never equal to a PG timestamp). + */ + private normalizeSQLExpr(field: Field, sql: string): string { + const spec = this.nullThrowsUnknownField(this.schema.table[field], field); + if (spec.type === Date) { + // Notice that `CAST(x AS timestamptz(3))` does ROUNDING, and we need + // TRUNCATION, since it's the default behavior of postgres-date (they + // changed it to rounding once, but then reverted intentionally) and + // node-postgres. See https://github.com/brianc/node-postgres/issues/1200 + sql = `date_trunc('ms', ${sql})`; + } + + return sql; + } + + /** + * Throws an exception about some field being not mentioned in the table + * schema if the passed data is undefined. Notice that ID is treated as always + * available in this message. + */ + private nullThrowsUnknownField( + data: T, + field: Field, + ): Exclude { + if (data === null || data === undefined) { + throw Error( + `Unknown field: ${field}; allowed fields: ` + + [ID, ...Object.keys(this.schema.table)], + ); + } else { + return data as Exclude; + } + } +} diff --git a/src/pg/PgSchema.ts b/src/pg/PgSchema.ts new file mode 100644 index 0000000..737b8a8 --- /dev/null +++ b/src/pg/PgSchema.ts @@ -0,0 +1,76 @@ +import type { Query } from "../abstract/Query"; +import { Schema } from "../abstract/Schema"; +import type { + CountInput, + ExistsInput, + InsertInput, + LoadByInput, + Row, + SelectByInput, + SelectInput, + Table, + UniqueKey, + UpdateInput, +} from "../types"; +import { PgQueryCount } from "./PgQueryCount"; +import { PgQueryDelete } from "./PgQueryDelete"; +import { PgQueryExists } from "./PgQueryExists"; +import { PgQueryIDGen } from "./PgQueryIDGen"; +import { PgQueryInsert } from "./PgQueryInsert"; +import { PgQueryLoad } from "./PgQueryLoad"; +import { PgQueryLoadBy } from "./PgQueryLoadBy"; +import { PgQuerySelect } from "./PgQuerySelect"; +import { PgQuerySelectBy } from "./PgQuerySelectBy"; +import { PgQueryUpdate } from "./PgQueryUpdate"; +import { PgQueryUpsert } from "./PgQueryUpsert"; + +export class PgSchema< + TTable extends Table, + TUniqueKey extends UniqueKey, +> extends Schema { + idGen(): Query { + return new PgQueryIDGen(this); + } + + insert(input: InsertInput): Query { + return new PgQueryInsert(this, input); + } + + upsert(input: InsertInput): Query { + return new PgQueryUpsert(this, input); + } + + update(id: string, input: UpdateInput): Query { + return new PgQueryUpdate(this, id, input); + } + + delete(id: string): Query { + return new PgQueryDelete(this, id); + } + + load(id: string): Query | null> { + return new PgQueryLoad(this, id); + } + + loadBy(input: LoadByInput): Query | null> { + return new PgQueryLoadBy(this, input); + } + + selectBy( + input: SelectByInput, + ): Query>> { + return new PgQuerySelectBy(this, input); + } + + select(input: SelectInput): Query>> { + return new PgQuerySelect(this, input); + } + + count(input: CountInput): Query { + return new PgQueryCount(this, input); + } + + exists(input: ExistsInput): Query { + return new PgQueryExists(this, input); + } +} diff --git a/src/pg/PgShardNamer.ts b/src/pg/PgShardNamer.ts new file mode 100644 index 0000000..f81d37d --- /dev/null +++ b/src/pg/PgShardNamer.ts @@ -0,0 +1,56 @@ +import { ShardError } from "../abstract/ShardError"; +import { ShardNamer } from "../abstract/ShardNamer"; +import { sanitizeIDForDebugPrinting } from "../internal/misc"; +import { parseCompositeRow } from "./internal/parseCompositeRow"; + +/** + * ShardNamer implementation for PG. + */ +export class PgShardNamer extends ShardNamer { + /** + * Synchronously extracts Shard number from an ID. Can also extract from PG + * composite rows (to support composite IDs). + */ + shardNoByID(id: string): number { + // Composite ID: `(100008888888,1023499999999)` - try extracting non-zero + // Shard from parts (left to right) first, and if there is none, allow shard + // zero too. + if (typeof id === "string" && id.startsWith("(") && id.endsWith(")")) { + let no = NaN; + for (const subID of parseCompositeRow(id)) { + const tryNo = + subID && subID.length >= this.shardNoPadLen + 1 + ? parseInt(subID.substring(1, this.shardNoPadLen + 1)) + : NaN; + if (!isNaN(tryNo)) { + if (tryNo > 0) { + return tryNo; + } else if (isNaN(no)) { + no = tryNo; + } + } + } + + if (isNaN(no)) { + const idSafe = sanitizeIDForDebugPrinting(id); + throw Error( + `Cannot extract shard number from the composite ID ${idSafe}`, + ); + } + + return no; + } + + // Plain ID. + const no = + typeof id === "string" && id.length >= this.shardNoPadLen + 1 + ? parseInt(id.substring(1, this.shardNoPadLen + 1)) + : NaN; + if (isNaN(no)) { + const idSafe = sanitizeIDForDebugPrinting(id); + throw new ShardError(`Cannot parse ID ${idSafe} to detect shard number`); + } + + return no; + } +} diff --git a/src/pg/PgTimelineStorage.ts b/src/pg/PgTimelineStorage.ts new file mode 100644 index 0000000..7b7cc12 --- /dev/null +++ b/src/pg/PgTimelineStorage.ts @@ -0,0 +1,150 @@ +import defaults from "lodash/defaults"; +import first from "lodash/first"; +import sortBy from "lodash/sortBy"; +import type { Cluster } from "../abstract/Cluster"; +import { MASTER } from "../abstract/Shard"; +import { ShardError } from "../abstract/ShardError"; +import type { TimelineStorageOptions } from "../ent/TimelineStorage"; +import { TimelineStorage } from "../ent/TimelineStorage"; +import { maybeCall, type PickPartial } from "../internal/misc"; +import type { Literal } from "../types"; +import { escapeIdent } from "./helpers/escapeIdent"; +import type { PgClient } from "./PgClient"; + +export interface PgTimelineStorageOptions extends TimelineStorageOptions { + cluster: Cluster; + table?: string; +} + +/** + * An append-only (with compaction) timeline storage for PG. The timelines are + * always appended to the table, but from time to time, when the number of + * chunks per principal exceeds the limit, the timelines are read back, + * compacted and written back as a single row. This is race condition safe, + * since timelines merging is an associative and idempotent operation, i.e. + * (T1+T2)+T3 == T1+(T2+T3); in the worst case, we'll just have slightly + * suboptimal timeline rows. + * + * The expected table schema is: + * ``` + * CREATE UNLOGGED TABLE timelines( + * id bigserial PRIMARY KEY, + * principal text NOT NULL, + * data text NOT NULL, + * created_at timestamptz NOT NULL + * ); + * CREATE INDEX timelines_principal ON timelines (principal); + * ``` + * + * Notes: + * 1. Index on `principal` must be non-unique, since there may be multiple + * records with the same value. + * 2. The `id` field should have sequential auto-increment, since it's used for + * garbage collection. + * 3. The table must exist in all microshards (including global shard). + */ +export class PgTimelineStorage extends TimelineStorage { + /** Default values for the constructor options. */ + static override readonly DEFAULT_OPTIONS: Required< + PickPartial + > = { + ...super.DEFAULT_OPTIONS, + table: "timelines", + maxChunksPerPrincipal: 10, + }; + + /** PgTimelineStorage configuration options. */ + override readonly options: Required; + + /** + * Initializes an instance of PgTimelineStorage. + */ + constructor(options: PgTimelineStorageOptions) { + super(options); + this.options = defaults( + {}, + options, + (this as TimelineStorage).options, + PgTimelineStorage.DEFAULT_OPTIONS, + ); + } + + async load(principal: string): Promise { + const rows = await this.query<{ id: string | number; data: string }>( + principal, + "TIMELINES_SELECT", + ["SELECT data FROM %T WHERE principal=?", principal], + ); + return sortBy(rows, (row) => row.id).map((row) => row.data); + } + + async save(principal: string, dataStr: string): Promise { + const row = first( + await this.query<{ id: string | number; chunks: string[] | null }>( + principal, + "TIMELINES_INSERT", + [ + "INSERT INTO %T (principal, data, created_at) VALUES (?, ?, now())\n" + + "RETURNING id, (SELECT array_agg(id||':'||data) FROM %T WHERE principal=?) AS chunks", + principal, + dataStr, + principal, + ], + ), + )!; + if ( + !row.chunks || + row.chunks.length <= maybeCall(this.options.maxChunksPerPrincipal) - 1 + ) { + return; + } + + let dataStrs = [[String(row.id), dataStr]]; + for (const chunk of row.chunks) { + const pos = chunk.indexOf(":"); + dataStrs.push([chunk.substring(0, pos), chunk.substring(pos + 1)]); + } + + dataStrs = sortBy(dataStrs, ([id, _]) => id); + dataStr = this.options.merge(dataStrs.map(([_, data]) => data)); + + await this.query(principal, "TIMELINES_INSERT", [ + "INSERT INTO %T (principal, data, created_at) VALUES (?, ?, now())", + principal, + dataStr, + ]); + await this.query(principal, "TIMELINES_DELETE", [ + "DELETE FROM %T WHERE id=ANY(?)", + dataStrs.map(([id]) => id), + ]); + } + + private async query( + principal: string, + op: string, + query: Literal, + ): Promise { + let shard; + try { + shard = this.options.cluster.shard(principal); + } catch (e) { + if (e instanceof ShardError) { + shard = this.options.cluster.globalShard(); + } else { + throw e; + } + } + + const client = await shard.client(MASTER); + return client.query({ + query: [ + String(query[0]).replace(/%T/g, escapeIdent(this.options.table)), + ...query.slice(1), + ], + isWrite: true, + annotations: [], + op, + table: this.options.table, + }); + } +} diff --git a/src/pg/__benchmarks__/batched-inserts.benchmark.ts b/src/pg/__benchmarks__/batched-inserts.benchmark.ts new file mode 100644 index 0000000..0c7a7b2 --- /dev/null +++ b/src/pg/__benchmarks__/batched-inserts.benchmark.ts @@ -0,0 +1,105 @@ +import delay from "delay"; +import range from "lodash/range"; +import { Pool } from "pg"; +import { TEST_CONFIG } from "../__tests__/test-utils"; + +const MAX_TIME_MS = 20000; + +const TBL = "batched_bench"; + +const COMMON_HEADER = "SET LOCAL search_path TO public"; + +const TEMPLATE_BATCHED = [ + "WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES\n" + + `((NULL::${TBL}).name, (NULL::${TBL}).url_name, (NULL::${TBL}).some_flag, (NULL::${TBL}).json_text_field, (NULL::${TBL}).json_strongly_typed_field, (NULL::${TBL}).jsonb_field, (NULL::${TBL}).encrypted_field, (NULL::${TBL}).created_at, (NULL::${TBL}).updated_at, (NULL::${TBL}).parent_id, (NULL::${TBL}).id, ''),`, + ` ('abc%s', 'aaa', true, '{"a":10,"b":{"c":20},"m":"%m"}', '{"a":42}', '{"a":"%m","b%m":"10"}', 'encrypted:ufyu', now(), now(), NULL, nextval('${TBL}_id_seq'), %k)`, + ")\n" + + `INSERT INTO ${TBL} (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id)\n` + + "SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET 1\n" + + `ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id=${TBL}.id), id`, +]; + +const TEMPLATE_MULTI = + `INSERT INTO ${TBL} (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES\n` + + `('abc%s', 'aaa', true, '{"a":10,"b":{"c":20},"m":"%m"}', '{"a":42}', '{"a":"%m","b%m":"10"}', 'encrypted:ufyu', now(), now(), NULL, nextval('${TBL}_id_seq')) ON CONFLICT DO NOTHING RETURNING id`; + +let seq = 0; +let seqKey = 0; + +export default async function* (): AsyncGenerator { + const pool = new Pool({ + ...TEST_CONFIG, + max: 1, + idleTimeoutMillis: 30000, + }); + + yield { + name: "beforeEach", + func: async () => { + seq = 0; + await pool.query(` + BEGIN; + ${COMMON_HEADER}; + DROP TABLE IF EXISTS ${TBL}; + CREATE TABLE ${TBL}( + id bigserial NOT NULL PRIMARY KEY, + name text NOT NULL, + url_name text, + some_flag boolean, + json_text_field text, + json_strongly_typed_field text, + jsonb_field jsonb, + encrypted_field text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + parent_id bigint, + UNIQUE (name) + ); + CREATE INDEX ${TBL}_json_text_field ON ${TBL}(json_text_field); + CREATE INDEX ${TBL}_jsonb_field ON ${TBL} USING gin (jsonb_field); + COMMIT; + `); + await pool.query("CHECKPOINT"); + await delay(2000); + }, + }; + + for (const batchSize of [2, 10, 20, 50, 100, 200, 400]) { + const RANGE = range(0, batchSize); + + for (let i = 0; i < 1; i++) { + yield { + name: `each request = 1 batched query (${batchSize} rows each)`, + maxTimeMs: MAX_TIME_MS, + func: async () => { + const query = + `${COMMON_HEADER};\n` + + `${TEMPLATE_BATCHED[0]}\n` + + RANGE.map(() => replace(TEMPLATE_BATCHED[1])).join(",\n") + + `\n${TEMPLATE_BATCHED[2]}`; + await pool.query(query); + }, + }; + } + + for (let i = 0; i < 1; i++) { + yield { + name: `each request = ${batchSize} queries ("multi-query")`, + maxTimeMs: MAX_TIME_MS, + func: async () => { + const query = + `${COMMON_HEADER};\n` + + RANGE.map(() => replace(TEMPLATE_MULTI)).join(";\n"); + await pool.query(query); + }, + }; + } + } +} + +function replace(template: string): string { + return template + .replace(/%s/g, `${seq++}`) + .replace(/%m/g, `${seq++ % 10}`) + .replace(/%k/g, `'${seqKey++}'`); +} diff --git a/src/pg/__tests__/PgClient.test.ts b/src/pg/__tests__/PgClient.test.ts new file mode 100644 index 0000000..683b249 --- /dev/null +++ b/src/pg/__tests__/PgClient.test.ts @@ -0,0 +1,106 @@ +import range from "lodash/range"; +import { MASTER } from "../../abstract/Shard"; +import { mapJoin } from "../../internal/misc"; +import { PgClient, type PgClientOptions } from "../PgClient"; +import { testCluster } from "./test-utils"; + +let options: PgClientOptions; + +beforeEach(async () => { + options = (await testCluster.globalShard().client(MASTER)).options; +}); + +test("custom Pool implementation is type safe", async () => { + const master = await testCluster.shardByNo(2).client(MASTER); + expect(master.pool().some()).toEqual("some"); +}); + +test("sub-pools are distinct", async () => { + const master = await testCluster.shardByNo(2).client(MASTER); + const client = new PgClient(master.options); + + const defaultPool = client.pool(); + const subPoolA = client.pool({ name: "a" }); + const subPoolB = client.pool({ name: "b" }); + + expect(subPoolA).toBe(client.pool({ name: "a", max: 42 })); + expect(subPoolA).not.toBe(subPoolB); + expect(subPoolA).not.toBe(defaultPool); + + let conn; + try { + conn = await client.acquireConn({ name: "a" }); + expect(subPoolA.totalCount).toEqual(1); + expect(subPoolB.totalCount).toEqual(0); + expect(defaultPool.totalCount).toEqual(0); + } finally { + conn?.release(); + } + + await client.end(); + expect(subPoolA.totalCount).toEqual(0); +}); + +test("acquireConn does not leak when release() is called", async () => { + await mapJoin(range(1000), async (i) => { + let conn; + try { + const shard = testCluster.shardByNo(2); + const master = await shard.client(MASTER); + conn = await master.acquireConn(); + const res = await conn.query("SELECT $1 AS i", [i]); + expect(res.rows[0].i).toEqual(i.toString()); + } finally { + // If we comment this out, it will blow up. + conn?.release(); + } + }); +}); + +test("static role=master", async () => { + const client = new PgClient({ + ...options, + role: "master", + maxReplicationLagMs: undefined, + }); + await client.query({ + query: ["SELECT 1"], + isWrite: true, + annotations: [], + op: "", + table: "", + }); + expect(client.options.maxReplicationLagMs).toBeLessThan(10000); + expect(await client.timelineManager.currentPos()).toEqual(BigInt(1)); + expect(client.role()).toEqual("master"); +}); + +test("static role=replica", async () => { + const client = new PgClient({ + ...options, + role: "replica", + maxReplicationLagMs: 20000, + }); + await client.query({ + query: ["SELECT 1"], + isWrite: false, + annotations: [], + op: "", + table: "", + }); + expect(await client.timelineManager.currentPos()).toEqual(BigInt(0)); + expect(client.role()).toEqual("replica"); +}); + +test("dynamic role=unknown", async () => { + const client = new PgClient({ ...options, role: "unknown" }); + await client.query({ + query: ["SELECT 1"], + isWrite: false, + annotations: [], + op: "", + table: "", + }); + expect(await client.timelineManager.currentPos()).toBeGreaterThan(BigInt(1)); + expect(client.role()).toEqual("master"); +}); diff --git a/src/pg/__tests__/PgError.test.ts b/src/pg/__tests__/PgError.test.ts new file mode 100644 index 0000000..7698f95 --- /dev/null +++ b/src/pg/__tests__/PgError.test.ts @@ -0,0 +1,10 @@ +import { inspectCompact } from "../../internal/misc"; +import { PgError } from "../PgError"; + +test("sql", () => { + const error = new PgError(Error("test"), "some", "SELECT 1", "mytable"); + expect(inspectCompact(error)).not.toContain("sql:"); + expect(inspectCompact(error.message)).not.toContain("mytable"); + expect(error.table).toEqual("mytable"); + expect(error.sql).toEqual("SELECT 1"); +}); diff --git a/src/pg/__tests__/PgSchema.alias-pk.test.ts b/src/pg/__tests__/PgSchema.alias-pk.test.ts new file mode 100644 index 0000000..30471ca --- /dev/null +++ b/src/pg/__tests__/PgSchema.alias-pk.test.ts @@ -0,0 +1,119 @@ +import type { Shard } from "../../abstract/Shard"; +import { MASTER } from "../../abstract/Shard"; +import { nullthrows } from "../../internal/misc"; +import { ID } from "../../types"; +import { PgQueryDeleteWhere } from "../PgQueryDeleteWhere"; +import { PgSchema } from "../PgSchema"; +import type { TestPgClient } from "./test-utils"; +import { recreateTestTables, shardRun, testCluster } from "./test-utils"; + +const schema = new PgSchema( + 'pg-schema.alias-pk"table', + { + user_id: { type: ID, autoInsert: "id_gen()" }, + name: { type: String }, + }, + ["user_id"], +); + +let shard: Shard; +let master: TestPgClient; + +beforeEach(async () => { + await recreateTestTables([ + { + CREATE: [ + `CREATE TABLE %T( + user_id bigint NOT NULL PRIMARY KEY, + name text NOT NULL + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); + + shard = await testCluster.randomShard(); + master = await shard.client(MASTER); +}); + +test("single ops", async () => { + const id1 = await shardRun( + shard, + schema.insert({ user_id: "1", name: "n1" }), + ); + master.toMatchSnapshot(); + expect(id1).toEqual("1"); + + const id2 = await shardRun( + shard, + schema.upsert({ user_id: "2", name: "n2" }), + ); + master.toMatchSnapshot(); + + const id3 = await shardRun(shard, schema.insert({ name: "n3" })); + master.toMatchSnapshot(); + + { + const res = await shardRun(shard, schema.update(nullthrows(id1), {})); // no DB query is sent here + master.toMatchSnapshot(); + expect(res).toEqual(true); + } + + { + const res = await shardRun( + shard, + schema.upsert({ user_id: "101", name: "n11" }), + ); + master.toMatchSnapshot(); + expect(res).toEqual("101"); + } + + { + const res = await shardRun(shard, schema.delete(nullthrows(id1))); + master.toMatchSnapshot(); + expect(res).toEqual(true); + } + + { + const res = await shardRun( + shard, + new PgQueryDeleteWhere(schema, { id: [id2!], $literal: ["1=1"] }), + ); + master.toMatchSnapshot(); + expect(res).toEqual([id2]); + } + + { + const res = await shardRun(shard, schema.load(id3!)); + master.toMatchSnapshot(); + expect(res).toMatchObject({ id: id3, name: "n3" }); + } + + { + const res = await shardRun(shard, schema.loadBy({ user_id: id3! })); + master.toMatchSnapshot(); + expect(res).toMatchObject({ id: id3, name: "n3" }); + } + + { + const res = await shardRun(shard, schema.count({ user_id: id3! })); + master.toMatchSnapshot(); + expect(res).toEqual(1); + } + + { + const res = await shardRun(shard, schema.exists({ user_id: id3! })); + master.toMatchSnapshot(); + expect(res).toStrictEqual(true); + } + + { + const res = await shardRun( + shard, + schema.select({ where: { user_id: id3! }, limit: 10 }), + ); + master.toMatchSnapshot(); + expect(res).toMatchObject([{ user_id: id3 }]); + } +}); diff --git a/src/pg/__tests__/PgSchema.async-islands.test.ts b/src/pg/__tests__/PgSchema.async-islands.test.ts new file mode 100644 index 0000000..68cbee6 --- /dev/null +++ b/src/pg/__tests__/PgSchema.async-islands.test.ts @@ -0,0 +1,93 @@ +import delay from "delay"; +import first from "lodash/first"; +import uniq from "lodash/uniq"; +import waitForExpect from "wait-for-expect"; +import { maybeCall } from "../../internal/misc"; +import { TEST_CONFIG, testCluster } from "./test-utils"; + +jest.useFakeTimers({ advanceTimers: true }); + +let nodesFn: () => Promise>; + +beforeEach(async () => { + nodesFn = async () => [TEST_CONFIG]; + testCluster.options.shardsDiscoverIntervalMs = 20000; + testCluster.options.reloadIslandsIntervalMs = 100; + testCluster.options.islands = async () => [{ no: 0, nodes: await nodesFn() }]; + + // Resolve all long delay() calls from the previous tests, to not let them + // stuck the cluster rediscover() below. + jest.advanceTimersByTime(2_000_000); + + await testCluster.rediscover(); + expect(first(await testCluster.islands())!.clients).toHaveLength(1); +}); + +test("when 2nd call to Cluster#options.islands is slow, it doesn't block rediscovery", async () => { + let reloadIslandsEntered = false; + let reloadIslandsExited = false; + nodesFn = async () => + delay(1) + .then(() => (reloadIslandsEntered = true)) + .then(async () => delay(1_000_000)) + .then(() => (reloadIslandsExited = true)) + .then(() => [TEST_CONFIG, TEST_CONFIG]); + + // Make sure the long delay code got triggered. + await jest.advanceTimersByTimeAsync(1000); + expect(reloadIslandsEntered).toBeTruthy(); + + // Still uses the cached Islands quickly, even though the new run of + // `options.islands()` is very slow. + await jest.advanceTimersByTimeAsync( + maybeCall(testCluster.options.shardsDiscoverIntervalMs) * 4, + ); + expect(reloadIslandsExited).toBeFalsy(); + expect(first(await testCluster.islands())!.clients).toHaveLength(1); + + // Eventually, uses the updated Islands. + await jest.advanceTimersByTimeAsync(1_200_000); + await waitForExpect( + async () => + expect( + first(await testCluster.islands())!.clients.map( + (c) => c.options.config, + ), + ).toHaveLength(2), + 10000, + ); +}); + +test("when 2nd call to Cluster#options.islands throws, it doesn't block rediscovery", async () => { + let reloadIslandsEntered = false; + let reloadIslandsThrew = false; + nodesFn = async () => + delay(1) + .then(() => (reloadIslandsEntered = true)) + .then(async () => delay(10_000)) + .then(() => (reloadIslandsThrew = true)) + .then(async () => + Promise.reject(new Error("Fake error in options.islands()")), + ); + + // Make sure the long delay code got triggered. + await jest.advanceTimersByTimeAsync(1000); + expect(reloadIslandsEntered).toBeTruthy(); + expect(reloadIslandsThrew).toBeFalsy(); + + // Despite `options.islands()` threw, we are still able to use the old cache. + await jest.advanceTimersByTimeAsync(12000); + expect(reloadIslandsThrew).toBeTruthy(); + const errors = uniq( + jest + .mocked(testCluster.options.loggers.swallowedErrorLogger!) + .mock.calls.map((call) => "" + call[0].error), + ).join("\n"); + expect(errors).toContain("Fake error"); + expect(errors).toContain("islands() did not complete in"); + + await testCluster.rediscover("shards"); + expect( + first(await testCluster.islands())!.clients.map((c) => c.options.config), + ).toHaveLength(1); +}); diff --git a/src/pg/__tests__/PgSchema.cas.test.ts b/src/pg/__tests__/PgSchema.cas.test.ts new file mode 100644 index 0000000..829f5da --- /dev/null +++ b/src/pg/__tests__/PgSchema.cas.test.ts @@ -0,0 +1,330 @@ +import assert from "assert"; +import type { Shard } from "../../abstract/Shard"; +import { MASTER } from "../../abstract/Shard"; +import { join, nullthrows } from "../../internal/misc"; +import { PgSchema } from "../PgSchema"; +import { ByteaBufferType } from "../types/ByteaBufferType"; +import type { TestPgClient } from "./test-utils"; +import { + EncryptedValue, + TEST_TIMELINE, + recreateTestTables, + shardRun, + testCluster, +} from "./test-utils"; + +const schema = new PgSchema( + 'pg-schema.cas"table', + { + name: { type: String }, + ts: { type: Date, allowNull: true }, + encrypted_field: { + type: EncryptedValue, + allowNull: true, + autoInsert: "NULL", + }, + buffer_field: { + type: ByteaBufferType(), + allowNull: true, + autoInsert: "NULL", + }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + id: { type: String, autoInsert: "id_gen()" }, + }, + ["name"], +); + +let shard: Shard; +let master: TestPgClient; +let replica: TestPgClient; + +beforeEach(async () => { + await recreateTestTables([ + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL, + ts timestamptz, + encrypted_field text, + buffer_field bytea, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + UNIQUE (name) + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); + + shard = await testCluster.randomShard(); + master = await shard.client(MASTER); + master.resetSnapshot(); + replica = await shard.client(TEST_TIMELINE); + replica.resetSnapshot(); +}); + +test("CAS single success", async () => { + const ts1 = new Date("2000-01-01T00:00:00.001Z"); + const ts1upd = new Date("2000-02-01T00:00:00.001Z"); + + const [id1, id2] = await join([ + shardRun( + shard, + schema.insert({ + name: "a", + ts: ts1, + encrypted_field: await EncryptedValue.encrypt("a", 42), + buffer_field: Buffer.from([1, 2, 3]), + }), + ), + shardRun(shard, schema.insert({ name: "b", ts: null })), + ]); + assert(id1 && id2); + const [row1, row2] = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + ]); + assert(row1 && row2); + master.resetSnapshot(); + + const r1 = await shardRun( + shard, + schema.update(id1, { + name: "a-upd", + ts: ts1upd, + $cas: { + name: "a", + ts: ts1, + encrypted_field: await EncryptedValue.encrypt("a", 42), + buffer_field: Buffer.from([1, 2, 3]), + updated_at: row1.updated_at, + }, + }), + ); + const r2 = await shardRun( + shard, + schema.update(id2, { + name: "b-upd", + ts: row2.updated_at, + $cas: { name: "b", ts: null, updated_at: row2.updated_at }, + }), + ); + master.toMatchSnapshot(); + + expect(r1).toBeTruthy(); + expect(r2).toBeTruthy(); + + const rows = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + ]); + expect(rows).toMatchObject([ + { id: id1, name: "a-upd", ts: ts1upd }, + { id: id2, name: "b-upd", ts: row2.updated_at }, + ]); +}); + +test("CAS single skip", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", ts: new Date() })), + shardRun( + shard, + schema.insert({ + name: "b", + ts: null, + buffer_field: Buffer.from([1, 2, 3]), + }), + ), + ]); + assert(id1 && id2); + const [row1, row2] = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + ]); + assert(row1 && row2); + master.resetSnapshot(); + + const r1Skip = await shardRun( + shard, + schema.update(id1, { + name: "a-skip", + $cas: { name: "a-old", updated_at: row1.updated_at }, + }), + ); + const r2Skip = await shardRun( + shard, + schema.update(id2, { + name: "b-skip", + $cas: { buffer_field: Buffer.from([4, 5, 6]) }, + }), + ); + master.toMatchSnapshot(); + + expect(r1Skip).toBeFalsy(); + expect(r2Skip).toBeFalsy(); + + const rows = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + ]); + expect(rows).toMatchObject([ + { id: id1, name: "a", ts: row1.ts, updated_at: row1.updated_at }, + { id: id2, name: "b", ts: null, updated_at: row2.updated_at }, + ]); +}); + +test("CAS batched success", async () => { + const ts1 = new Date("2000-01-01T00:00:00.001Z"); + const ts1upd = new Date("2000-02-01T00:00:00.001Z"); + + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", ts: ts1 })), + shardRun(shard, schema.insert({ name: "b", ts: null })), + ]); + assert(id1 && id2); + const [row1, row2] = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + ]); + assert(row1 && row2); + master.resetSnapshot(); + + const [r1, r2] = await join([ + shardRun( + shard, + schema.update(id1, { + name: "a-upd", + ts: ts1upd, + $cas: { name: "a", ts: ts1, updated_at: row1.updated_at }, + }), + ), + shardRun( + shard, + schema.update(id2, { + name: "b-upd", + ts: row2.updated_at, + $cas: { name: "b", ts: null, updated_at: row2.updated_at }, + }), + ), + ]); + master.toMatchSnapshot(); + + expect(r1).toBeTruthy(); + expect(r2).toBeTruthy(); + + const rows = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + ]); + expect(rows).toMatchObject([ + { id: id1, name: "a-upd", ts: ts1upd }, + { id: id2, name: "b-upd", ts: row2.updated_at }, + ]); +}); + +test("CAS batched success and skip", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", ts: new Date() })), + shardRun(shard, schema.insert({ name: "b", ts: null })), + ]); + assert(id1 && id2); + const [row1, row2] = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + ]); + assert(row1 && row2); + + const [r1, r2] = await join([ + shardRun( + shard, + schema.update(id1, { + name: "a-upd", + $cas: { name: "a", updated_at: row1.updated_at }, + }), + ), + shardRun( + shard, + schema.update(id2, { + name: "b-upd", + $cas: { name: "b-old", updated_at: row2.updated_at }, + }), + ), + ]); + expect(r1).toBeTruthy(); + expect(r2).toBeFalsy(); + + const rows = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + ]); + expect(rows).toMatchObject([ + { id: id1, name: "a-upd" }, + { id: id2, name: "b", updated_at: row2.updated_at }, + ]); +}); + +test("CAS batched success and skip for the same row but different $cas values", async () => { + const id1 = nullthrows( + await shardRun(shard, schema.insert({ name: "a", ts: new Date() })), + ); + const row1 = nullthrows(await shardRun(shard, schema.load(id1))); + master.resetSnapshot(); + + const [r1, r2] = await join([ + shardRun( + shard, + schema.update(id1, { + name: "a-upd", + $cas: { name: "a", updated_at: row1.updated_at }, + }), + ), + shardRun( + shard, + schema.update(id1, { + name: "a-skip", + $cas: { name: "a", updated_at: new Date(42) }, + }), + ), + ]); + master.toMatchSnapshot(); + + expect(r1).toBeTruthy(); + expect(r2).toBeFalsy(); + + const rows = await join([shardRun(shard, schema.load(id1))]); + expect(rows).toMatchObject([{ id: id1, name: "a-upd" }]); +}); + +test("CAS disables coalescing for the same row with the same $cas values", async () => { + const id = nullthrows( + await shardRun(shard, schema.insert({ name: "a", ts: null })), + ); + master.resetSnapshot(); + + const [r1, r2] = await join([ + shardRun( + shard, + schema.update(id, { + name: "a-upd", + $cas: { name: "a" }, + }), + ), + shardRun( + shard, + schema.update(id, { + name: "a-skip", + $cas: { name: "a" }, + }), + ), + ]); + master.toMatchSnapshot(); + + expect(r1).toBeTruthy(); + expect(r2).toBeFalsy(); + + const rows = await join([shardRun(shard, schema.load(id))]); + expect(rows).toMatchObject([{ id, name: "a-upd" }]); +}); diff --git a/src/pg/__tests__/PgSchema.composite-pk.test.ts b/src/pg/__tests__/PgSchema.composite-pk.test.ts new file mode 100644 index 0000000..fb38ce8 --- /dev/null +++ b/src/pg/__tests__/PgSchema.composite-pk.test.ts @@ -0,0 +1,302 @@ +import type { Shard } from "../../abstract/Shard"; +import { MASTER } from "../../abstract/Shard"; +import { join, nullthrows } from "../../internal/misc"; +import { ID } from "../../types"; +import { PgQueryDeleteWhere } from "../PgQueryDeleteWhere"; +import { PgSchema } from "../PgSchema"; +import type { TestPgClient } from "./test-utils"; +import { recreateTestTables, shardRun, testCluster } from "./test-utils"; + +const schema = new PgSchema( + 'pg-schema.composite-pk"table', + { + tenant_id: { type: ID }, + user_id: { type: ID }, + name: { type: String }, + }, + ["tenant_id", "user_id"], +); + +let shard: Shard; +let master: TestPgClient; + +beforeEach(async () => { + await recreateTestTables([ + { + CREATE: [ + `CREATE TABLE %T( + tenant_id bigint NOT NULL, + user_id bigint NOT NULL, + name text NOT NULL, + PRIMARY KEY (tenant_id, user_id) + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); + + shard = await testCluster.randomShard(); + master = await shard.client(MASTER); +}); + +test("single ops", async () => { + const id1 = await shardRun( + shard, + schema.insert({ tenant_id: "1", user_id: "1", name: "n1" }), + ); + master.toMatchSnapshot(); + expect(id1).toEqual("(1,1)"); + + const id2 = await shardRun( + shard, + schema.upsert({ tenant_id: "1", user_id: "2", name: "n2" }), + ); + await shardRun( + shard, + schema.insert({ tenant_id: "1", user_id: "3", name: "n3" }), + ); + master.resetSnapshot(); + + { + const res = await shardRun(shard, schema.update(nullthrows(id1), {})); // no DB query is sent here + master.toMatchSnapshot(); + expect(res).toEqual(true); + } + + { + const res = await shardRun( + shard, + schema.upsert({ tenant_id: "1", user_id: "101", name: "n11" }), + ); + master.toMatchSnapshot(); + expect(res).toEqual("(1,101)"); + } + + { + const res = await shardRun( + shard, + schema.update("(42,42)", { name: "absent" }), + ); + master.toMatchSnapshot(); + expect(res).toBeFalsy(); + } + + { + const res = await shardRun( + shard, + schema.update(nullthrows(id1), { name: "new-name" }), + ); + master.toMatchSnapshot(); + expect(res).toEqual(true); + } + + { + const res = await shardRun(shard, schema.delete(nullthrows(id1))); + master.toMatchSnapshot(); + expect(res).toEqual(true); + } + + { + const res = await shardRun( + shard, + new PgQueryDeleteWhere(schema, { id: [id2!], $literal: ["1=1"] }), + ); + master.toMatchSnapshot(); + expect(res).toEqual([id2]); + } + + { + const res = await shardRun(shard, schema.count({ user_id: "3" })); + master.toMatchSnapshot(); + expect(res).toEqual(1); + } + + { + const res = await shardRun(shard, schema.exists({ user_id: "3" })); + master.toMatchSnapshot(); + expect(res).toStrictEqual(true); + } + + { + const res = await shardRun( + shard, + schema.select({ where: { user_id: "3" }, limit: 10 }), + ); + master.toMatchSnapshot(); + expect(res).toMatchObject([{ tenant_id: "1", user_id: "3" }]); + } +}); + +test("batched ops", async () => { + const [id1, id2, id3, id4] = await join([ + shardRun( + shard, + schema.insert({ tenant_id: "1", user_id: "1", name: "n1" }), + ), + shardRun( + shard, + schema.insert({ tenant_id: "1", user_id: "2", name: "n2" }), + ), + shardRun( + shard, + schema.insert({ tenant_id: "1", user_id: "3", name: "n3" }), + ), + shardRun( + shard, + schema.insert({ tenant_id: "1", user_id: "4", name: "n4" }), + ), + ]); + master.toMatchSnapshot(); + expect([id1, id2]).toEqual(["(1,1)", "(1,2)"]); + + { + const res = await join([ + shardRun(shard, schema.update(nullthrows(id1), {})), // no SQL query sent + shardRun(shard, schema.update(nullthrows(id2), {})), + ]); + master.toMatchSnapshot(); + expect(res).toEqual([true, true]); + } + + { + const res = await join([ + shardRun(shard, schema.update(id1!, { name: "new-name-1" })), + shardRun(shard, schema.update(id2!, { name: "new-name-2" })), + shardRun(shard, schema.update("(42,42)", { name: "absent" })), + ]); + master.toMatchSnapshot(); + expect(res).toEqual([true, true, false]); + } + + { + const res = await join([ + shardRun( + shard, + schema.upsert({ tenant_id: "1", user_id: "1", name: "n11" }), + ), + shardRun( + shard, + schema.upsert({ tenant_id: "9", user_id: "9", name: "n9" }), + ), + ]); + master.toMatchSnapshot(); + expect(res).toEqual(["(1,1)", "(9,9)"]); + } + + { + const res = await join([ + shardRun(shard, schema.delete(id1!)), + shardRun(shard, schema.delete(id2!)), + ]); + master.toMatchSnapshot(); + expect(res).toEqual([true, true]); + } + + { + const res = await join([ + shardRun(shard, schema.load(id3!)), + shardRun(shard, schema.load("(1,424)")), + ]); + master.toMatchSnapshot(); + expect(res).toMatchObject([{ id: id3, name: "n3" }, null]); + } + + { + const res = await join([ + shardRun(shard, schema.loadBy({ tenant_id: "1", user_id: "3" })), + shardRun(shard, schema.loadBy({ tenant_id: "1", user_id: "4" })), + ]); + master.toMatchSnapshot(); + expect(res).toMatchObject([ + { id: id3, name: "n3" }, + { id: id4, name: "n4" }, + ]); + } + + { + const res = await join([ + shardRun(shard, schema.count({ user_id: "3" })), + shardRun(shard, schema.count({ tenant_id: "1" })), + ]); + master.toMatchSnapshot(); + expect(res).toEqual([1, 2]); + } + + { + const res = await join([ + shardRun(shard, schema.exists({ user_id: "3" })), + shardRun(shard, schema.exists({ tenant_id: "1" })), + shardRun(shard, schema.exists({ tenant_id: "199999" })), + ]); + master.toMatchSnapshot(); + expect(res).toEqual([true, true, false]); + } + + { + const res = await join([ + shardRun( + shard, + schema.select({ + where: { user_id: "3" }, + order: [{ user_id: "ASC" }], + limit: 10, + }), + ), + shardRun( + shard, + schema.select({ + where: { tenant_id: "1" }, + order: [{ user_id: "ASC" }], + limit: 10, + }), + ), + shardRun( + shard, + schema.select({ + where: { tenant_id: "101" }, + order: [{ user_id: "ASC" }], + limit: 10, + }), + ), + ]); + master.toMatchSnapshot(); + expect(res).toMatchObject([ + [{ id: id3, tenant_id: "1", user_id: "3" }], + [ + { id: id3, tenant_id: "1", user_id: "3" }, + { id: id4, tenant_id: "1", user_id: "4" }, + ], + [], + ]); + } + + { + const res = await join([ + shardRun( + shard, + schema.select({ + where: { id: id3! }, + order: [{ user_id: "ASC" }], + limit: 10, + }), + ), + shardRun( + shard, + schema.select({ + where: { id: [id3!, id4!] }, + order: [{ user_id: "ASC" }], + limit: 10, + }), + ), + ]); + master.toMatchSnapshot(); + expect(res).toMatchObject([ + [{ id: id3, tenant_id: "1", user_id: "3" }], + [ + { id: id3, tenant_id: "1", user_id: "3" }, + { id: id4, tenant_id: "1", user_id: "4" }, + ], + ]); + } +}); diff --git a/src/pg/__tests__/PgSchema.connect-timeout.test.ts b/src/pg/__tests__/PgSchema.connect-timeout.test.ts new file mode 100644 index 0000000..a1cfd51 --- /dev/null +++ b/src/pg/__tests__/PgSchema.connect-timeout.test.ts @@ -0,0 +1,127 @@ +import delay from "delay"; +import waitForExpect from "wait-for-expect"; +import { MASTER, type Shard } from "../../abstract/Shard"; +import { maybeCall, runInVoid } from "../../internal/misc"; +import { escapeIdent } from "../helpers/escapeIdent"; +import { PgSchema } from "../PgSchema"; +import type { TestPgClient } from "./test-utils"; +import { + TEST_CONFIG, + recreateTestTables, + shardRun, + testCluster, + TCPProxyServer, + TEST_ISLANDS, +} from "./test-utils"; + +jest.useFakeTimers({ advanceTimers: true }); + +const schema = new PgSchema( + 'pg-schema.connect-timeout"table', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + }, + [], +); + +const TABLE_BAK = `${schema.name}_bak`; + +let shard: Shard; +let connStuckServer: TCPProxyServer; +let connStuckTestConfig: typeof TEST_CONFIG; + +beforeEach(async () => { + testCluster.options.runOnShardErrorRetryCount = 2; + testCluster.options.runOnShardErrorRediscoverClusterDelayMs = 100; + testCluster.options.reloadIslandsIntervalMs = 20000; // intentionally large + testCluster.options.shardsDiscoverIntervalMs = 100000; + + testCluster.options.islands = TEST_ISLANDS; + await testCluster.rediscover(); + + await recreateTestTables([ + { + CREATE: [ + `DROP TABLE IF EXISTS ${escapeIdent(TABLE_BAK)} CASCADE`, + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); + + shard = await testCluster.randomShard(); + + connStuckServer = new TCPProxyServer({ + host: TEST_CONFIG.host!, + port: TEST_CONFIG.port!, + delayOnConnect: 1000000, + }); + connStuckTestConfig = { + ...TEST_CONFIG, + isAlwaysLaggingReplica: true, + ...(await connStuckServer.hostPort()), + connectionTimeoutMillis: 30000, + }; +}); + +afterEach(async () => { + await connStuckServer.destroy(); +}); + +test("when connection gets stuck during background rediscovery, it does not slowdown queries", async () => { + const errorSpy = jest.mocked( + connStuckTestConfig.loggers.swallowedErrorLogger, + ); + testCluster.options.islands = [ + { no: 0, nodes: [TEST_CONFIG, connStuckTestConfig] }, + ]; + await jest.advanceTimersByTimeAsync( + maybeCall(testCluster.options.reloadIslandsIntervalMs), + ); + await connStuckServer.waitForAtLeastConnections(1); + + expect(await shardRun(shard, schema.insert({ name: "abc" }))).toBeTruthy(); + expect(errorSpy).not.toBeCalled(); + + jest.advanceTimersByTime(connStuckTestConfig.connectionTimeoutMillis!); + await waitForExpect(() => + expect("" + errorSpy.mock.lastCall?.[0].error).toContain("timeout"), + ); + await testCluster.islands(); +}); + +test("when rediscovery is triggered by a failed query, and connection gets stuck, then the query waits until rediscovery finishes", async () => { + const master = await shard.client(MASTER); + await master.rows("ALTER TABLE %T RENAME TO %T", schema.name, TABLE_BAK); + + const errorSpy = jest.mocked( + connStuckTestConfig.loggers.swallowedErrorLogger, + ); + testCluster.options.islands = [ + { no: 0, nodes: [TEST_CONFIG, connStuckTestConfig] }, + ]; + + // The query below fails and thus triggers rediscovery. + let shardRunResult: unknown = undefined; + runInVoid( + shardRun(shard, schema.insert({ name: "abc" })) + .then(() => (shardRunResult = true)) + .catch((e: unknown) => (shardRunResult = e)), + ); + await connStuckServer.waitForAtLeastConnections(1); + + await master.rows("ALTER TABLE %T RENAME TO %T", TABLE_BAK, schema.name); + await delay(1000); + expect(shardRunResult).toBeUndefined(); + + jest.advanceTimersByTime(connStuckTestConfig.connectionTimeoutMillis!); + await waitForExpect(() => expect(shardRunResult).toEqual(true)); + + expect(errorSpy.mock.calls).toHaveLength(1); + expect("" + errorSpy.mock.calls[0][0].error).toContain("timeout"); +}); diff --git a/src/pg/__tests__/PgSchema.failover.test.ts b/src/pg/__tests__/PgSchema.failover.test.ts new file mode 100644 index 0000000..939e6b6 --- /dev/null +++ b/src/pg/__tests__/PgSchema.failover.test.ts @@ -0,0 +1,95 @@ +import delay from "delay"; +import range from "lodash/range"; +import { type Shard } from "../../abstract/Shard"; +import { mapJoin } from "../../internal/misc"; +import { PgSchema } from "../PgSchema"; +import type { TestPgClient } from "./test-utils"; +import { recreateTestTables, shardRun, testCluster } from "./test-utils"; + +const schema = new PgSchema( + 'pg-schema.failover"table', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + }, + [], +); + +let shard: Shard; +let islandClient1: TestPgClient; +let islandClient2: TestPgClient; + +beforeEach(async () => { + testCluster.options.shardsDiscoverIntervalMs = 1_000_000; + testCluster.options.runOnShardErrorRetryCount = 1; + testCluster.options.runOnShardErrorRediscoverIslandDelayMs = 100; + testCluster.options.runOnShardErrorRediscoverClusterDelayMs = 10000000; + await testCluster.rediscover(); + + const island0 = await testCluster.island(0); + islandClient1 = island0.master(); + islandClient2 = island0.replica(); + islandClient1.options.hints = { transaction: "read write" }; + islandClient2.options.hints = { transaction: "read only" }; + + await recreateTestTables([ + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); + + shard = await testCluster.randomShard(); +}); + +test("query retries on new master when switchover happens", async () => { + islandClient1.options.hints = { transaction: "read only" }; + islandClient2.options.hints = { transaction: "read write" }; + + const islandRediscoverSpy = jest + .spyOn(await testCluster.island(0), "rediscover") + .mockImplementation(async () => { + // After the query() fails, pretend that islandClient1 became replica and + // islandClient2 became master, so after rediscovery and a retry to + // query(), it will choose the right master (islandClient2). + jest.spyOn(islandClient1, "role").mockReturnValue("replica"); + jest.spyOn(islandClient2, "role").mockReturnValue("master"); + return []; + }); + + // Produce a burst of erroring queries. + const queries = range(50).map((i) => schema.insert({ name: `test${i}` })); + await mapJoin(queries, async (query, i) => { + await delay((i / queries.length) * 10); + await shardRun(shard, query); + }); + + // All 10 queries fail and cause Island rediscovery to happen. + expect(testCluster.options.loggers.runOnShardErrorLogger).toBeCalledTimes( + queries.length, + ); + + // Island#rediscover() requests should be coalesced into significantly less + // calls (i.e. Cluster#rediscoverIsland() is coalesce-memoized). Despite we + // have 50 parallel queries, the calls to Island rediscovery were coalesced to + // just a few. + expect(islandRediscoverSpy.mock.calls.length).toBeLessThan( + queries.length / 3, + ); +}); + +test("query fails when no master appears after a retry", async () => { + islandClient1.options.hints = { transaction: "read only" }; + islandClient2.options.hints = { transaction: "read only" }; + + await expect( + shardRun(shard, schema.insert({ name: "test" })), + ).rejects.toThrow(/read_only_sql_transaction/); + expect(testCluster.options.loggers.runOnShardErrorLogger).toBeCalledTimes(2); +}); diff --git a/src/pg/__tests__/PgSchema.generic.test.ts b/src/pg/__tests__/PgSchema.generic.test.ts new file mode 100644 index 0000000..3774733 --- /dev/null +++ b/src/pg/__tests__/PgSchema.generic.test.ts @@ -0,0 +1,1440 @@ +import delay from "delay"; +import range from "lodash/range"; +import sortBy from "lodash/sortBy"; +import { escapeIdent } from ".."; +import type { Shard } from "../../abstract/Shard"; +import { MASTER, STALE_REPLICA } from "../../abstract/Shard"; +import { join, nullthrows } from "../../internal/misc"; +import type { UpdateInput } from "../../types"; +import { ID } from "../../types"; +import { PgError } from "../PgError"; +import { PgQueryDeleteWhere } from "../PgQueryDeleteWhere"; +import { PgQueryIDGen } from "../PgQueryIDGen"; +import { PgQueryInsert } from "../PgQueryInsert"; +import { PgSchema } from "../PgSchema"; +import type { TestPgClient } from "./test-utils"; +import { + EncryptedValue, + TEST_TIMELINE, + recreateTestTables, + shardRun, + testCluster, +} from "./test-utils"; + +const schema = new PgSchema( + 'pg-schema.generic"table', + { + name: { type: String }, + url_name: { type: String, allowNull: true }, + some_flag: { type: Boolean, allowNull: true, autoInsert: "false" }, + json_text_field: { + type: { + dbValueToJs: (v: string) => + JSON.parse(v) as { a: number; b: { c: number } }, + stringify: JSON.stringify, + parse: JSON.parse, + }, + allowNull: true, + autoInsert: "NULL", + }, + json_strongly_typed_field: { + type: { + dbValueToJs: (v: unknown) => v as { a: number }, // node-postgres does conversion from JSON internally + stringify: JSON.stringify, + parse: JSON.parse, + }, + allowNull: true, + autoInsert: "NULL", + }, + jsonb_field: { + type: { + dbValueToJs: (v: unknown) => v, // node-postgres does conversion from JSON internally + stringify: JSON.stringify, + parse: JSON.parse, + }, + allowNull: true, + autoInsert: "NULL", + }, + encrypted_field: { + // This is another use-case, a fully custom class carrying a DB value. It + // still works via the same parse/stringify framework, BUT we may + // sometimes want to do asynchronous stuff in read/write path which just + // can't be done in parse/stringify (since they operate below Ent/VC + // abstraction layer). + type: EncryptedValue, + allowNull: true, + autoInsert: "NULL", + }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + parent_id: { type: ID, allowNull: true, autoInsert: "NULL" }, + // id in the end since we pre-sort rows in UPDATE queries, and IDs + // are random; we want to have consistent ordering of results across + // tests runs to match the test snapshots + id: { type: String, autoInsert: "id_gen()" }, + }, + ["name"], +); + +const schemaNullableUniqueKey = new PgSchema( + 'pg-schema.generic"table_nullable_unique_key', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String, allowNull: true }, + url_name: { type: String, allowNull: true, autoInsert: "NULL" }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + }, + ["url_name"], +); + +const schemaUniqueKeyWithAutoInsert = new PgSchema( + 'pg-schema.generic"table_unique_key_with_auto_insert', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String, autoInsert: "'myname'" }, + url_name: { type: String, autoInsert: "'n'" }, + }, + ["url_name"], +); + +const schema2Col = new PgSchema( + 'pg-schema.generic"table_2col', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + url_name: { type: String }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + }, + ["name", "url_name"], +); + +const schema2ColNullableUniqueKey = new PgSchema( + 'pg-schema.generic"table_2col_nullable_unique_key', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String, allowNull: true }, + url_name: { type: String, allowNull: true }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + }, + ["name", "url_name"], +); + +const schema3Col = new PgSchema( + 'pg-schema.generic"table_3col', + { + id: { type: String, autoInsert: "id_gen()" }, + type: { type: String }, + id1: { type: String }, + id2: { type: String }, + created_at: { type: Date, autoInsert: "now()" }, + updated_at: { type: Date, autoUpdate: "now()" }, + }, + ["type", "id1", "id2"], +); + +const schemaDate = new PgSchema( + 'pg-schema.generic"table_date', + { + date_id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + some_date: { type: Date, allowNull: true, autoInsert: "NULL" }, + }, + ["date_id"], +); + +const schemaIDUniqueKey = new PgSchema( + 'pg-schema.generic"table_id_unique_key', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + }, + ["id"], +); + +const schemaNameWithAutoInsert = new PgSchema( + 'pg-schema.generic"table_name_with_auto_insert', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String, allowNull: true, autoInsert: "'n'" }, + url_name: { type: String }, + }, + ["url_name"], +); + +let shard: Shard; +let master: TestPgClient; +let replica: TestPgClient; + +beforeEach(async () => { + await recreateTestTables([ + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name varchar(1024) NOT NULL, + url_name text, + some_flag boolean, + json_text_field text, + json_strongly_typed_field json, + jsonb_field jsonb, + encrypted_field text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + parent_id bigint, + UNIQUE (name) + )`, + "ALTER TABLE %T ADD CONSTRAINT fk_parent_id FOREIGN KEY (parent_id) REFERENCES %T(id)", + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text, + url_name text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + UNIQUE (url_name) + )`, + ], + SCHEMA: schemaNullableUniqueKey, + SHARD_AFFINITY: [], + }, + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL, + url_name text NOT NULL, + UNIQUE (url_name) + )`, + ], + SCHEMA: schemaUniqueKeyWithAutoInsert, + SHARD_AFFINITY: [], + }, + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL, + url_name text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + UNIQUE (name, url_name) + )`, + ], + SCHEMA: schema2Col, + SHARD_AFFINITY: [], + }, + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text, + url_name text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + UNIQUE (name, url_name) + )`, + ], + SCHEMA: schema2ColNullableUniqueKey, + SHARD_AFFINITY: [], + }, + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + type text NOT NULL, + id1 text NOT NULL, + id2 text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + UNIQUE (type, id1, id2) + )`, + ], + SCHEMA: schema3Col, + SHARD_AFFINITY: [], + }, + { + CREATE: [ + `CREATE TABLE %T( + date_id bigint NOT NULL PRIMARY KEY, + name text, + some_date timestamptz + )`, + ], + SCHEMA: schemaDate, + SHARD_AFFINITY: [], + }, + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text + )`, + ], + SCHEMA: schemaIDUniqueKey, + SHARD_AFFINITY: [], + }, + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text, + url_name text NOT NULL, + UNIQUE (url_name) + )`, + ], + SCHEMA: schemaNameWithAutoInsert, + SHARD_AFFINITY: [], + }, + ]); + + shard = await testCluster.randomShard(); + master = await shard.client(MASTER); + master.resetSnapshot(); + replica = await shard.client(TEST_TIMELINE); + replica.resetSnapshot(); +}); + +test("idGen single", async () => { + master.resetSnapshot(); + const id1 = await shardRun(shard, schema.idGen()); + const id2 = await shardRun(shard, schema.idGen()); + master.toMatchSnapshot(); + expect(id1).not.toEqual(id2); +}); + +test("idGen batched", async () => { + master.resetSnapshot(); + const [id1, id2] = await join([ + shardRun(shard, schema.idGen()), + shardRun(shard, schema.idGen()), + ]); + master.toMatchSnapshot(); + expect(id1).not.toEqual(id2); +}); + +test("idGen with large batch", async () => { + const query = new PgQueryIDGen(schema); + const maxBatchSize = new query.RUNNER_CLASS(schema, master).maxBatchSize; + master.resetSnapshot(); + await join( + range(maxBatchSize * 2 - 10).map(async () => + shardRun(shard, schema.idGen()), + ), + ); + expect(master.queries).toHaveLength(2); +}); + +test("insert single", async () => { + const id1Gen = await shardRun(shard, schema.idGen()); + const id1 = await shardRun( + shard, + schema.insert({ + name: "a'b\x00'c", + url_name: "aaa", + some_flag: true, + json_text_field: { a: 10, b: { c: 20 } }, + json_strongly_typed_field: { a: 42 }, + jsonb_field: { a: 42 }, + encrypted_field: await EncryptedValue.encrypt("text", 1), + id: id1Gen, + }), + ); + const id2 = await shardRun( + shard, + schema.insert({ name: "a'b'c", url_name: "aaa_dup" }), + ); + master.toMatchSnapshot(); + + expect(id1).toEqual(id1Gen); + const row = await shardRun(shard, schema.load(id1!)); + expect(row).toMatchObject({ + name: "a'b'c", + some_flag: true, + json_text_field: { a: 10, b: { c: 20 } }, + json_strongly_typed_field: { a: 42 }, + jsonb_field: { a: 42 }, + }); + expect(await row!.encrypted_field!.decrypt(1)).toEqual("text"); + + expect(id2).toBeNull(); +}); + +test("insert pre-generated id with uniq key violation", async () => { + const id1 = await shardRun(shard, schema.idGen()); + const id2 = await shardRun(shard, schema.idGen()); + const [res1, res2] = await join([ + shardRun(shard, schema.insert({ id: id1, name: "some", url_name: "aaa" })), + shardRun(shard, schema.insert({ id: id2, name: "some", url_name: "aaa" })), + ]); + master.toMatchSnapshot(); + expect(res1 === null || res2 === null).toBeTruthy(); +}); + +test("insert batched", async () => { + const id2Gen = await shardRun(shard, schema.idGen()); + const [id1, id2, id3] = await join([ + shardRun( + shard, + schema.insert({ + name: "z'b'c", + url_name: null, + some_flag: true, + jsonb_field: null, + }), + ), + shardRun( + shard, + schema.insert({ name: "a", url_name: "u", some_flag: null, id: id2Gen }), + ), + shardRun(shard, schema.insert({ name: "a", url_name: "u_dup" })), + ]); + master.toMatchSnapshot(); + + const rows = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + ]); + expect(rows).toMatchObject([ + { name: "z'b'c", some_flag: true, jsonb_field: null }, + { name: "a", url_name: "u", some_flag: null, id: id2Gen }, + ]); + + expect(id3).toBeNull(); +}); + +test("insert large batch", async () => { + const query = new PgQueryInsert(schema, { name: "", url_name: null }); + const maxBatchSize = new query.RUNNER_CLASS(schema, master).maxBatchSize; + master.resetSnapshot(); + await join( + range(maxBatchSize * 2 - 10).map(async (i) => + shardRun(shard, schema.insert({ name: `aaa${i}`, url_name: `uuu${i}` })), + ), + ); + expect(master.queries).toHaveLength(2); +}); + +test("insert is never dedupped", async () => { + const [id1, id2] = await join([ + shardRun( + shard, + schema.insert({ name: "aaa", url_name: "uuu", some_flag: null }), + ), + shardRun( + shard, + schema.insert({ name: "aaa", url_name: "uuu", some_flag: null }), + ), + ]); + master.toMatchSnapshot(); + expect((id1 === null) !== (id2 === null)).toBeTruthy(); +}); + +test("insert de-batches pg error", async () => { + const [res1, res2] = await join([ + shardRun( + shard, + // should succeed + schema.insert({ name: "some", url_name: null }), + ).catch((e: unknown) => e), + shardRun( + shard, + // should fail with FK constraint error + schema.insert({ name: "other", url_name: null, parent_id: "0" }), + ).catch((e: unknown) => e), + ]); + master.toMatchSnapshot(); + expect(typeof res1).toEqual("string"); + expect(res2).toBeInstanceOf(PgError); + expect((res2 as PgError).isFKError()).toBeTruthy(); + expect((res2 as PgError).isFKError("fk_parent_id")).toBeTruthy(); +}); + +test("insert de-batches pg error: data type: varchar", async () => { + const [res1, res2] = await join([ + shardRun( + shard, + // should succeed + schema.insert({ name: "some", url_name: null }), + ).catch((e: unknown) => e), + shardRun( + shard, + // should fail with "value too long for type character varying" error + schema.insert({ name: `l${"o".repeat(1024)}g`, url_name: null }), + ).catch((e: unknown) => e), + ]); + master.toMatchSnapshot(); + expect(typeof res1).toEqual("string"); + expect(res2).toBeInstanceOf(PgError); + expect(`${res2}`).toContain("value too long for type character varying"); +}); + +test("insert de-batches pg error: data type: bigint", async () => { + const [res1, res2] = await join([ + shardRun( + shard, + // should succeed + schema.insert({ name: "some", url_name: null }), + ).catch((e: unknown) => e), + shardRun( + shard, + // should fail with "invalid input syntax for type bigint" error + schema.insert({ + name: "other", + url_name: null, + id: "not a bigint", + }), + ).catch((e: unknown) => e), + ]); + master.toMatchSnapshot(); + expect(typeof res1).toEqual("string"); + expect(res2).toBeInstanceOf(PgError); + expect(`${res2}`).toContain("invalid input syntax for type bigint"); +}); + +test("upsert single", async () => { + const id1 = await shardRun( + shard, + schema.upsert({ name: "a'b'c", url_name: "aaa" }), + ); + const origRow1 = await master.rows( + "SELECT created_at FROM %T WHERE id=?", + ...[schema.name, id1], + ); + await delay(20); // to check that created_at is constant + + const id2 = await shardRun( + shard, + schema.upsert({ name: "a'b'c", url_name: "aaa_dup" }), + ); + const id3 = await shardRun( + shard, + schema.upsert({ name: "zzz", url_name: "n" }), + ); + master.toMatchSnapshot(); + + expect(id2).toEqual(id1); + const rows = await master.rows( + "SELECT id, name, created_at FROM %T ORDER BY NAME", + ...[schema.name, id1], + ); + expect(rows).toMatchObject([ + { id: id1, name: "a'b'c", created_at: origRow1[0]["created_at"] }, + { id: id3, name: "zzz" }, + ]); + + const id4 = await shardRun( + shard, + schemaIDUniqueKey.upsert({ id: id1, name: "a" }), + ); + master.toMatchSnapshot(); + expect(id4).toEqual(id1); +}); + +test("upsert batched does not mess up the rows order", async () => { + const [id1, id2, id3] = await join([ + shardRun(shard, schema.upsert({ name: "zzz", url_name: "zzz" })), + shardRun(shard, schema.upsert({ name: "bbb", url_name: "bbb" })), + shardRun(shard, schema.upsert({ name: "ccc", url_name: "ccc" })), + ]); + master.toMatchSnapshot(); + const [row1, row2, row3] = await join([ + shardRun(shard, schema.load(id1)), + shardRun(shard, schema.load(id2)), + shardRun(shard, schema.load(id3)), + ]); + expect(row1).toMatchObject({ name: "zzz" }); + expect(row2).toMatchObject({ name: "bbb" }); + expect(row3).toMatchObject({ name: "ccc" }); +}); + +test("upsert batched normal", async () => { + const [id1, id2, id3] = await join([ + shardRun(shard, schema.upsert({ name: "a'b'c", url_name: "aaa" })), + shardRun(shard, schema.upsert({ name: "bbb", url_name: "bbb" })), + shardRun(shard, schema.upsert({ name: "ccc", url_name: "ccc" })), + ]); + master.resetSnapshot(); + + const [r1, r2, r4, r4dup] = await join([ + shardRun(shard, schema.upsert({ name: "a'b'c", url_name: "aaa_new" })), + shardRun(shard, schema.upsert({ name: "bbb", url_name: "bbb_new" })), + shardRun(shard, schema.upsert({ name: "zzz", url_name: "zzz" })), + shardRun(shard, schema.upsert({ name: "zzz", url_name: "zzz_dup" })), + ]); + master.toMatchSnapshot(); + + expect(r1).toEqual(id1); + expect(r2).toEqual(id2); + expect(r4dup).toEqual(r4); + + const rows = await master.rows( + "SELECT id, name, url_name FROM %T ORDER BY name", + schema.name, + ); + expect(rows).toMatchObject([ + { id: id1, url_name: "aaa_new" }, + { id: id2, url_name: "bbb_new" }, + { id: id3, url_name: "ccc" }, + { id: r4, name: "zzz", url_name: "zzz_dup" }, + ]); +}); + +test("upsert batched with nullable unique key must treat nulls as non-equal", async () => { + const [id0] = await join([ + shardRun( + shard, + schemaNullableUniqueKey.upsert({ name: "000", url_name: "0" }), + ), + ]); + const [id1, id2, id3] = await join([ + shardRun( + shard, + schemaNullableUniqueKey.upsert({ name: "a'b'c", url_name: null }), + ), + shardRun( + shard, + schemaNullableUniqueKey.upsert({ name: "bbb", url_name: null }), + ), + shardRun( + shard, + schemaNullableUniqueKey.upsert({ name: "ccc", url_name: null }), + ), + ]); + master.toMatchSnapshot(); + + expect(id0 !== id1).toBeTruthy(); + expect(id1 !== id2).toBeTruthy(); + expect(id2 !== id3).toBeTruthy(); + + const [id4] = await join([ + shardRun( + shard, + schemaNullableUniqueKey.upsert({ name: "ddd", url_name: null }), + ), + ]); + expect(id4 !== id3).toBeTruthy(); + + const rows = await master.rows( + "SELECT id, name, url_name FROM %T ORDER BY name", + schemaNullableUniqueKey.name, + ); + expect(rows).toMatchObject([ + { id: id0, name: "000", url_name: "0" }, + { id: id1, name: "a'b'c", url_name: null }, + { id: id2, name: "bbb", url_name: null }, + { id: id3, name: "ccc", url_name: null }, + { id: id4, name: "ddd", url_name: null }, + ]); +}); + +test("upsert with optional nullable unique key", async () => { + const id0 = await shardRun( + shard, + schemaNullableUniqueKey.upsert({ + name: "000", + // no url_name since it has autoInsert and can be omitted + }), + ); + const id1 = await shardRun( + shard, + schemaNullableUniqueKey.upsert({ + name: "111", + // no url_name since it has autoInsert and can be omitted + }), + ); + const rows = await master.rows( + "SELECT id, name, url_name FROM %T ORDER BY name", + schemaNullableUniqueKey.name, + ); + expect(rows).toMatchObject([ + { id: id0, name: "000", url_name: null }, + { id: id1, name: "111", url_name: null }, + ]); +}); + +test("upsert with autoInsert name creates it", async () => { + const id0 = await shardRun( + shard, + schemaUniqueKeyWithAutoInsert.upsert({ + // no name since it has autoInsert and can be omitted + // no url_name since it has autoInsert and can be omitted + }), + ); + master.toMatchSnapshot(); + const rows = await master.rows( + "SELECT id, name, url_name FROM %T ORDER BY name", + schemaUniqueKeyWithAutoInsert.name, + ); + expect(rows).toMatchObject([{ id: id0, name: "myname", url_name: "n" }]); +}); + +test("upsert with autoInsert name does not override on update when name is omitted", async () => { + const id0 = await shardRun( + shard, + schemaUniqueKeyWithAutoInsert.upsert({ + name: "000", + // no url_name since it has autoInsert and can be omitted + }), + ); + const id1 = await shardRun( + shard, + schemaUniqueKeyWithAutoInsert.upsert({ + // no name since it has autoInsert and can be omitted + // no url_name since it has autoInsert and can be omitted + }), + ); + master.toMatchSnapshot(); + const rows = await master.rows( + "SELECT id, name, url_name FROM %T ORDER BY name", + schemaUniqueKeyWithAutoInsert.name, + ); + expect(rows).toMatchObject([{ id: id0, name: "000", url_name: "n" }]); + expect(id0).toEqual(id1); +}); + +test("upsert does not override autoInsert field when it updates", async () => { + const id0 = await shardRun( + shard, + schemaNameWithAutoInsert.upsert({ name: "my", url_name: "aaa" }), + ); + const id1 = await shardRun( + shard, + schemaNameWithAutoInsert.upsert({ url_name: "aaa" }), + ); + master.toMatchSnapshot(); + expect(id0).toEqual(id1); + + const [id2, id3] = await join([ + shardRun( + shard, + schemaNameWithAutoInsert.upsert({ name: "my", url_name: "bbb" }), + ), + shardRun(shard, schemaNameWithAutoInsert.upsert({ url_name: "bbb" })), + ]); + master.toMatchSnapshot(); + expect(id2).toEqual(id3); + + const id4 = await shardRun( + shard, + schemaNameWithAutoInsert.upsert({ url_name: "ccc" }), + ); + const id5 = await shardRun( + shard, + schemaNameWithAutoInsert.upsert({ name: "my", url_name: "ccc" }), + ); + master.toMatchSnapshot(); + expect(id4).toEqual(id5); + + const rows = await master.rows( + "SELECT id, name, url_name FROM %T ORDER BY name", + schemaNameWithAutoInsert.name, + ); + expect(rows).toMatchObject([ + { id: id0, name: "my", url_name: "aaa" }, + { id: id2, name: "my", url_name: "bbb" }, + { id: id4, name: "my", url_name: "ccc" }, + ]); +}); + +test("update single", async () => { + const [id1, id2, id3] = await join([ + shardRun(shard, schema.insert({ name: "a'url", url_name: "a" })), + shardRun(shard, schema.insert({ name: "b'url", url_name: "b" })), + shardRun(shard, schema.insert({ name: "null'url", url_name: null })), + ]); + + const origRows = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + shardRun(shard, schema.load(id3!)), + ]); + await delay(20); // to check updated_at change + + master.resetSnapshot(); + await shardRun(shard, schema.update(nullthrows(id1), {})); // no DB query is sent here + const [r1, r2, r3] = await join([ + shardRun(shard, schema.update(nullthrows(id1), { name: "a'upd" })), + shardRun( + shard, + schema.update(nullthrows(id2), { + name: "b'upd", + url_name: null, + some_flag: true, + }), + ), + shardRun(shard, schema.update(nullthrows(id3), { url_name: "n" })), + ]); + const r4 = await shardRun(shard, schema.update("42", { name: "absent" })); + master.toMatchSnapshot(); + + expect(r1).toBeTruthy(); + expect(r2).toBeTruthy(); + expect(r3).toBeTruthy(); + expect(r4).toBeFalsy(); + + const rows = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + shardRun(shard, schema.load(id3!)), + ]); + expect(rows).toMatchObject([ + { id: id1, name: "a'upd", url_name: "a" }, + { id: id2, name: "b'upd", url_name: null, some_flag: true }, + { id: id3, name: "null'url", url_name: "n" }, + ]); + expect(rows[2]!.updated_at.getTime()).toBeGreaterThan( + origRows[2]!.updated_at.getTime(), + ); +}); + +test("update skips if no known fields present", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a'url", url_name: "a" })), + shardRun(shard, schema.insert({ name: "b'url", url_name: "b" })), + ]); + master.resetSnapshot(); + + const unrelatedFields = { + unrelated: 123, + [Symbol("symbol")]: 42, + } as UpdateInput; + const [r1, r2, r3] = await join([ + shardRun(shard, schema.update(nullthrows(id1), unrelatedFields)), + shardRun(shard, schema.update(nullthrows(id2), unrelatedFields)), + shardRun(shard, schema.update("101", unrelatedFields)), + ]); + const r4 = await shardRun( + shard, + schema.update(nullthrows(id1), unrelatedFields), + ); + const r5 = await shardRun(shard, schema.update("42", { name: "absent" })); + + master.toMatchSnapshot(); // only r5 update will actually be sent to the DB + expect(r1).toBeTruthy(); + expect(r2).toBeTruthy(); + expect(r3).toBeTruthy(); // although there is no such row, we still return true since the DB query is skipped + expect(r4).toBeTruthy(); + expect(r5).not.toBeTruthy(); +}); + +test("update batched", async () => { + const [id1, id2, id3] = await join([ + shardRun(shard, schema.insert({ name: "a'url", url_name: "a" })), + shardRun(shard, schema.insert({ name: "b'url", url_name: "b" })), + shardRun(shard, schema.insert({ name: "null'url", url_name: null })), + ]); + master.resetSnapshot(); + + await join([ + shardRun(shard, schema.update(nullthrows(id1), {})), + shardRun(shard, schema.update(nullthrows(id2), {})), + ]); + const [r1, r2, r3, r3dup, r4] = await join([ + shardRun( + shard, + schema.update(nullthrows(id1), { name: "a'upd", some_flag: true }), + ), + shardRun( + shard, + schema.update(nullthrows(id2), { + name: "b'upd", + url_name: null, + }), + ), + shardRun( + shard, + schema.update(nullthrows(id3), { + name: "null'upd", + url_name: "n", + }), + ), + shardRun( + shard, + schema.update(nullthrows(id3), { + name: "null'upd", + url_name: "n1", + }), + ), + shardRun(shard, schema.update("42", { name: "absent", some_flag: true })), + ]); + master.toMatchSnapshot(); + + expect(r1).toBeTruthy(); + expect(r2).toBeTruthy(); + expect(r3).toBeTruthy(); + expect(r3dup).toBeTruthy(); + expect(r4).toBeFalsy(); + + const rows = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + shardRun(shard, schema.load(id3!)), + ]); + expect(rows).toMatchObject([ + { id: id1, name: "a'upd", url_name: "a", some_flag: true }, + { id: id2, name: "b'upd", url_name: null }, + { id: id3, name: "null'upd", url_name: "n1" }, + ]); +}); + +test("update date", async () => { + const [id1, id2] = await join([ + shardRun( + shard, + schemaDate.insert({ name: "a", some_date: new Date(1234567890) }), + ), + shardRun(shard, schemaDate.insert({ name: "b" })), + ]); + master.resetSnapshot(); + + await shardRun(shard, schemaDate.update(id1!, { some_date: undefined })); + await shardRun( + shard, + schemaDate.update(id2!, { name: "bb", some_date: undefined }), + ); + await join([ + shardRun(shard, schemaDate.update(id1!, { some_date: undefined })), + shardRun(shard, schemaDate.update(id2!, { some_date: undefined })), + ]); + master.toMatchSnapshot(); + + const [row1, row2] = await join([ + shardRun(shard, schemaDate.load(id1!)), + shardRun(shard, schemaDate.load(id2!)), + ]); + expect(row1!.some_date).toBeTruthy(); + expect(row2!.name).toEqual("bb"); +}); + +test("update literal", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", url_name: "a" })), + shardRun(shard, schema.insert({ name: "b", url_name: "b" })), + ]); + master.resetSnapshot(); + + await join([ + shardRun( + shard, + schema.update(id1!, { $literal: ["name = name || ?", 42] }), + ), + shardRun(shard, schema.update(id2!, { name: "bbb" })), + shardRun( + shard, + schema.update(id1!, { $literal: ["name = name || ?", 42] }), + ), + shardRun( + shard, + schemaDate.update("42", { $literal: ["name = name || ?", 42] }), + ), + ]); + master.toMatchSnapshot(); + + const [row1, row2] = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + ]); + expect(row1!.name).toEqual("a4242"); + expect(row2!.name).toEqual("bbb"); +}); + +test("delete single", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", url_name: "a" })), + shardRun(shard, schema.insert({ name: "b", url_name: "b" })), + ]); + master.resetSnapshot(); + + const r1 = await shardRun(shard, schema.delete(nullthrows(id1))); + const r2 = await shardRun(shard, schema.delete("42")); + const r3 = await shardRun(shard, schema.delete(null as unknown as string)); + master.toMatchSnapshot(); + + expect(r1).toBeTruthy(); + expect(r2).toBeFalsy(); + expect(r3).toBeFalsy(); + + const rows = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + ]); + expect(rows).toMatchObject([null, { id: id2 }]); +}); + +test("delete batched", async () => { + const [id1, id2, id3] = await join([ + shardRun(shard, schema.insert({ name: "a", url_name: "a" })), + shardRun(shard, schema.insert({ name: "b", url_name: "b" })), + shardRun(shard, schema.insert({ name: "n", url_name: null })), + ]); + master.resetSnapshot(); + + const [r1, r2, r3] = await join([ + shardRun(shard, schema.delete(nullthrows(id1))), + shardRun(shard, schema.delete(nullthrows(id2))), + shardRun(shard, schema.delete("42")), + shardRun(shard, schema.delete(null as unknown as string)), + ]); + master.toMatchSnapshot(); + + expect(r1).toBeTruthy(); + expect(r2).toBeTruthy(); + expect(r3).toBeFalsy(); + + const rows = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + shardRun(shard, schema.load(id3!)), + ]); + expect(rows).toMatchObject([null, null, { id: id3 }]); +}); + +test("delete where", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", url_name: "a" })), + shardRun(shard, schema.insert({ name: "b", url_name: "b" })), + ]); + master.resetSnapshot(); + + const res = await shardRun( + shard, + new PgQueryDeleteWhere(schema, { id: [id1!, id2!], $literal: ["1=1"] }), + ); + master.toMatchSnapshot(); + expect(res.length).toEqual(2); + + const rows = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + ]); + expect(rows).toEqual([null, null]); +}); + +test("load batched", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", url_name: "a" })), + shardRun(shard, schema.insert({ name: "b", url_name: "b" })), + ]); + + master.resetSnapshot(); + const rows = await join([ + shardRun(shard, schema.load(id1!)), + shardRun(shard, schema.load(id2!)), + shardRun(shard, schema.load(id1!)), + ]); + master.toMatchSnapshot(); + expect(rows).toMatchObject([{ id: id1 }, { id: id2 }, { id: id1 }]); +}); + +test("loadBy single one column", async () => { + const [, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", url_name: "aaa" })), + shardRun(shard, schema.insert({ name: "b", url_name: "bbb" })), + ]); + + const row = await shardRun( + shard, + schema.loadBy({ name: "b" }), + STALE_REPLICA, + ); + replica.toMatchSnapshot(); + expect(row).toMatchObject({ id: id2, name: "b" }); +}); + +test("loadBy batched one column", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: String.raw`a\a`, url_name: "aaa" })), + shardRun(shard, schema.insert({ name: String.raw`b\nb`, url_name: "bbb" })), + ]); + + master.resetSnapshot(); + const rows = await join([ + shardRun(shard, schema.loadBy({ name: String.raw`a\a` })), + shardRun(shard, schema.loadBy({ name: String.raw`b\nb` })), + shardRun(shard, schema.loadBy({ name: "no value" })), + shardRun(shard, schema.loadBy({ name: String.raw`a\a` })), + ]); + master.toMatchSnapshot(); + expect(rows).toMatchObject([{ id: id1 }, { id: id2 }, null, { id: id1 }]); +}); + +test("loadBy single two columns", async () => { + const [, id2] = await join([ + shardRun(shard, schema2Col.insert({ name: "a", url_name: "aaa" })), + shardRun(shard, schema2Col.insert({ name: "b", url_name: "bbb" })), + ]); + + master.resetSnapshot(); + const row = await shardRun( + shard, + schema2Col.loadBy({ name: "b", url_name: "bbb" }), + ); + master.toMatchSnapshot(); + expect(row).toMatchObject({ id: id2, name: "b" }); +}); + +test("loadBy batched two columns", async () => { + const [id1, id2, id3, id4, id5] = await join([ + shardRun(shard, schema2Col.insert({ name: "z", url_name: "z1" })), + shardRun(shard, schema2Col.insert({ name: "z", url_name: "z,2" })), + shardRun(shard, schema2Col.insert({ name: "b", url_name: "b{1}" })), + shardRun(shard, schema2Col.insert({ name: "c", url_name: "NuLL" })), + shardRun(shard, schema2Col.insert({ name: "c", url_name: "" })), + ]); + + master.resetSnapshot(); + const rows = await join([ + shardRun(shard, schema2Col.loadBy({ name: "z", url_name: "z1" })), + shardRun(shard, schema2Col.loadBy({ name: "z", url_name: "z,2" })), + shardRun( + shard, + schema2Col.loadBy({ name: "b", url_name: String.raw`no\value` }), + ), + shardRun(shard, schema2Col.loadBy({ name: "b", url_name: "b{1}" })), + shardRun(shard, schema2Col.loadBy({ name: "c", url_name: "NuLL" })), + shardRun(shard, schema2Col.loadBy({ name: "c", url_name: "" })), + ]); + master.toMatchSnapshot(); + expect(rows).toMatchObject([ + { id: id1 }, + { id: id2 }, + null, + { id: id3 }, + { id: id4 }, + { id: id5 }, + ]); +}); + +test("loadBy single two columns with nullable unique key", async () => { + const [, id2] = await join([ + shardRun( + shard, + schema2ColNullableUniqueKey.insert({ name: "a", url_name: "aaa" }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.insert({ name: "b", url_name: null }), + ), + ]); + + master.resetSnapshot(); + const row = await shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: "b", url_name: null }), + ); + await shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: null, url_name: "a" }), + ); + await shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: null, url_name: null }), + ); + master.toMatchSnapshot(); + expect(row).toMatchObject({ id: id2, name: "b" }); +}); + +test("loadBy batched with two columns nullable unique key", async () => { + const [id1, id2, id3, id4, id5] = await join([ + shardRun( + shard, + schema2ColNullableUniqueKey.insert({ name: "z", url_name: "z1" }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.insert({ name: "z", url_name: "z2" }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.insert({ name: "b", url_name: null }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.insert({ name: "c", url_name: null }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.insert({ name: "c", url_name: "c2" }), + ), + ]); + + master.resetSnapshot(); + const rows = await join([ + shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: "z", url_name: "z1" }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: "z", url_name: "z2" }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: "no", url_name: "Null" }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: "b", url_name: null }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: "c", url_name: null }), + ), + shardRun( + shard, + schema2ColNullableUniqueKey.loadBy({ name: "c", url_name: "c2" }), + ), + ]); + master.toMatchSnapshot(); + expect(rows).toMatchObject([ + { id: id1 }, + { id: id2 }, + null, + { id: id3 }, + { id: id4 }, + { id: id5 }, + ]); +}); + +test("selectBy single three columns", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema3Col.insert({ type: "a", id1: "1", id2: "21" })), + shardRun(shard, schema3Col.insert({ type: "a", id1: "1", id2: "22" })), + ]); + + master.resetSnapshot(); + const rows = await shardRun( + shard, + schema3Col.selectBy({ type: "a", id1: "1" }), + ); + master.toMatchSnapshot(); + expect(sortBy(rows, (row) => row.id2)).toMatchObject([ + { id: id1, type: "a", id1: "1", id2: "21" }, + { id: id2, type: "a", id1: "1", id2: "22" }, + ]); +}); + +test("selectBy batched three columns", async () => { + const [id1, id2, id3, id4] = await join([ + shardRun(shard, schema3Col.insert({ type: "a", id1: "1", id2: "21" })), + shardRun(shard, schema3Col.insert({ type: "a", id1: "1", id2: "22" })), + shardRun(shard, schema3Col.insert({ type: "a", id1: "2", id2: "23" })), + shardRun(shard, schema3Col.insert({ type: "b", id1: "1", id2: "24" })), + ]); + + master.resetSnapshot(); + const [rows1, rows2, rows3, rows4] = await join([ + shardRun(shard, schema3Col.selectBy({ type: "a", id1: "1" })), + shardRun(shard, schema3Col.selectBy({ type: "a", id1: "2" })), + shardRun(shard, schema3Col.selectBy({ type: "b", id1: "1" })), + shardRun(shard, schema3Col.selectBy({ type: "b" })), + ]); + master.toMatchSnapshot(); + expect(sortBy(rows1, (row) => row.id2)).toMatchObject([ + { id: id1, type: "a", id1: "1", id2: "21" }, + { id: id2, type: "a", id1: "1", id2: "22" }, + ]); + expect(sortBy(rows2, (row) => row.id2)).toMatchObject([ + { id: id3, type: "a", id1: "2", id2: "23" }, + ]); + expect(sortBy(rows3, (row) => row.id2)).toMatchObject([ + { id: id4, type: "b", id1: "1", id2: "24" }, + ]); + expect(sortBy(rows4, (row) => row.id2)).toMatchObject([ + { id: id4, type: "b", id1: "1", id2: "24" }, + ]); +}); + +test("select and count batched", async () => { + const [id1, id2] = await join([ + shardRun( + shard, + schema.insert({ name: "a\na", url_name: "a1", some_flag: true }), + ), + shardRun( + shard, + schema.insert({ + name: String.raw`a\a`, + url_name: "aa1", + some_flag: true, + }), + ), + shardRun(shard, schema.insert({ name: "c", url_name: "c1" })), + shardRun(shard, schema.insert({ name: "d", url_name: "d1" })), + shardRun(shard, schema.insert({ name: "e", url_name: "ce1" })), + ]); + master.resetSnapshot(); + + const input: Parameters[0] = { + order: [{ name: "ASC" }, { url_name: "DESC" }, { $literal: ["1=?", 2] }], + where: { + name: ["a\na", String.raw`a\a`], + some_flag: true, + $or: [ + { name: "a\na" }, + { name: String.raw`a\a` }, + { url_name: [] }, + { url_name: [null, "zzz"] }, + ], + $and: [ + { name: ["a\na", String.raw`a\a`] }, + { name: { $ne: "kk" } }, + { name: { $isDistinctFrom: "dd" } }, + { url_name: { $isDistinctFrom: null } }, + { url_name: { $ne: ["kk", null] } }, + { url_name: { $ne: [] } }, + { $literal: ["? > '2'", "5"] }, + { name: { $lte: "z", $gte: "a" } }, + ], + $not: { name: "zz", $literal: ["? < '2'", 5] }, + }, + limit: 10, + }; + const [rows1] = await join([ + shardRun(shard, schema.select(input)), + shardRun(shard, schema.select({ where: { name: "b" }, limit: 10 })), + ]); + + const [count1, count2, count3, count4] = await join([ + shardRun(shard, schema.count(input.where!)), + shardRun(shard, schema.count(input.where!)), + shardRun(shard, schema.count({ ...input.where!, url_name: "a1" })), + shardRun(shard, schema.count({ name: "b" })), + ]); + const count5 = await shardRun(shard, schema.count(input.where!)); + + const [exists1, exists2, exists3, exists4] = await join([ + shardRun(shard, schema.exists(input.where!)), + shardRun(shard, schema.exists(input.where!)), + shardRun(shard, schema.exists({ ...input.where!, url_name: "a1" })), + shardRun(shard, schema.exists({ name: "b" })), + ]); + const exists5 = await shardRun(shard, schema.exists(input.where!)); + + master.toMatchSnapshot(); + expect(rows1).toMatchObject([{ id: id1 }, { id: id2 }]); + expect(count1).toEqual(2); + expect(count2).toEqual(2); + expect(count3).toEqual(1); + expect(count4).toEqual(0); + expect(count5).toEqual(2); + expect(exists1).toStrictEqual(true); + expect(exists2).toStrictEqual(true); + expect(exists3).toStrictEqual(true); + expect(exists4).toStrictEqual(false); + expect(exists5).toStrictEqual(true); +}); + +test("select custom", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", url_name: "a1" })), + shardRun(shard, schema.insert({ name: "b", url_name: "b1" })), + ]); + master.resetSnapshot(); + + const input: Parameters[0] = { + where: { + name: ["a", "aa"], + $literal: ["? > '2'", "5"], + }, + order: [{ name: "ASC" }, { $literal: ["cte1_v"] }], + limit: 10, + custom: { + ctes: [ + ["cte1 AS (SELECT 1+? AS cte1_v FROM generate_series(1,1))", "1"], + ["cte2 AS (SELECT 10+? AS cte2_v)", "10"], + ], + joins: [ + ["JOIN cte1 ON true", "1"], + ["JOIN (SELECT 10+? AS join2_v) join2 ON true", "10"], + ], + hints: { + enable_seqscan: "off", + }, + }, + }; + + const [rows1] = await join([shardRun(shard, schema.select(input))]); + expect(rows1).toMatchObject([{ id: id1 }]); + + const [rows1a, rows2] = await join([ + shardRun(shard, schema.select(input)), + shardRun( + shard, + schema.select({ + where: { name: "b" }, + limit: 10, + custom: { hints: { enable_seqscan: "off" } }, + }), + ), + shardRun(shard, schema.select({ where: { name: "c" }, limit: 10 })), + ]); + expect(rows1a).toMatchObject([{ id: id1 }]); + expect(rows2).toMatchObject([{ id: id2 }]); + + master.toMatchSnapshot(); +}); + +test("rawPrepend hint", async () => { + const [id1, id2] = await join([ + shardRun(shard, schema.insert({ name: "a", url_name: "a1" })), + shardRun(shard, schema.insert({ name: "b", url_name: "b1" })), + ]); + master.resetSnapshot(); + + const [rows1, rows2] = await join([ + shardRun( + shard, + schema.select({ + where: { name: "a" }, + limit: 10, + custom: { + hints: { "": `/*+SeqScan(${escapeIdent(schema.name)})*/` }, + }, + }), + ), + shardRun( + shard, + schema.select({ + where: { name: "b" }, + limit: 10, + custom: { + hints: { "": `/*+SeqScan(${escapeIdent(schema.name)})*/` }, + }, + }), + ), + ]); + expect(rows1).toMatchObject([{ id: id1 }]); + expect(rows2).toMatchObject([{ id: id2 }]); + master.toMatchSnapshot(); +}); + +test("test empty $or and $and", async () => { + await join([ + shardRun( + shard, + schema.insert({ name: "a", url_name: "a1", some_flag: true }), + ), + shardRun( + shard, + schema.insert({ name: "aa", url_name: "aa1", some_flag: true }), + ), + ]); + + const [all, emptyOR, emptyAND] = await join([ + shardRun(shard, schema.select({ where: {}, limit: 2 })), + shardRun(shard, schema.select({ where: { $or: [] }, limit: 2 })), + shardRun(shard, schema.select({ where: { $and: [] }, limit: 2 })), + ]); + expect(all.length).toBe(2); + expect(emptyOR.length).toBe(0); + expect(emptyAND.length).toBe(0); +}); diff --git a/src/pg/__tests__/PgSchema.islands-reconfig.test.ts b/src/pg/__tests__/PgSchema.islands-reconfig.test.ts new file mode 100644 index 0000000..14b508a --- /dev/null +++ b/src/pg/__tests__/PgSchema.islands-reconfig.test.ts @@ -0,0 +1,159 @@ +import delay from "delay"; +import pDefer from "p-defer"; +import waitForExpect from "wait-for-expect"; +import { STALE_REPLICA } from "../../abstract/Shard"; +import { PgSchema } from "../PgSchema"; +import { + TEST_CONFIG, + TEST_ISLANDS, + reconfigureToTwoIslands, + recreateTestTables, + shardRun, + testCluster, +} from "./test-utils"; + +jest.useFakeTimers({ advanceTimers: true }); + +const TEST_ID = "100001234"; + +const schema = new PgSchema( + 'pg-schema.islands-reconfig"table', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + }, + [], +); + +beforeEach(async () => { + testCluster.options.runOnShardErrorRetryCount = 1; + testCluster.options.shardsDiscoverIntervalMs = 20000; // very large intentionally + testCluster.options.reloadIslandsIntervalMs = 10; + testCluster.options.clientEndDelayMs = 0; + + testCluster.options.islands = TEST_ISLANDS; + await testCluster.rediscover(); + + await recreateTestTables([ + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); +}); + +test("node added dynamically appears in the cluster, old client is removed", async () => { + const oldIsland0 = await testCluster.island(0); + const oldIsland0Shards = oldIsland0.shards(); + const oldMaster0 = oldIsland0.master(); // will be reused + const oldReplica0 = oldIsland0.replica(); // will disappear + + await reconfigureToTwoIslands(); + + await delay(1); + expect(oldMaster0.isEnded()).toBeFalsy(); + expect(oldReplica0.isEnded()).toBeTruthy(); + + const newIsland0 = await testCluster.island(0); + const newIsland1 = await testCluster.island(1); + const newMaster0 = newIsland0.master(); + const newMaster1 = newIsland1.master(); + expect(newMaster0).toBe(oldMaster0); + expect(newMaster1).not.toBe(newMaster0); + + expect(newIsland0.shards()).toEqual([]); + expect(newIsland1.shards()).toEqual(oldIsland0Shards); +}); + +test("shard client changes when cluster is reconfigured", async () => { + const oldShard0Replica = await testCluster + .shard(TEST_ID) + .client(STALE_REPLICA); + await reconfigureToTwoIslands(); + const newShard0Replica = await testCluster + .shard(TEST_ID) + .client(STALE_REPLICA); + expect(newShard0Replica).not.toBe(oldShard0Replica); +}); + +test("when old client is returned to the shard code, but then ended, the operation is retried", async () => { + const shard = await testCluster.randomShard(); + + const oldReplica = await shard.client(STALE_REPLICA); + const oldReplicaQueryCalledDefer = pDefer(); + const oldReplicaQueryUnfreezeDefer = pDefer(); + jest.spyOn(oldReplica, "query").mockImplementationOnce(async (...args) => { + oldReplicaQueryCalledDefer.resolve(); + await delay(1); + expect(oldReplica.isEnded()).toBeFalsy(); + await oldReplicaQueryUnfreezeDefer.promise; + await delay(1); + expect(oldReplica.isEnded()).toBeTruthy(); + return oldReplica.query(...args); + }); + const promise = shardRun(shard, schema.select({ where: {}, limit: 1 })); + await oldReplicaQueryCalledDefer.promise; + + testCluster.options.islands = () => [ + { + no: 0, + nodes: [ + TEST_CONFIG, + { + ...TEST_CONFIG, + isAlwaysLaggingReplica: true, + nameSuffix: "modified", + }, + ], + }, + ]; + await testCluster.rediscover(); + + // By this time, Client#query() is called for an already ended Client, and + // it's frozen till oldReplicaQueryUnfreezeDefer is resolved. Sharded calls + // should be retried, so a new replica Client should be chosen internally. + const replicaQuerySpy = jest.spyOn( + await shard.client(STALE_REPLICA), + "query", + ); + oldReplicaQueryUnfreezeDefer.resolve(); + await promise; + expect(replicaQuerySpy).toBeCalledTimes(1); +}); + +test("low level (non-sharded) client queries are not retried if the client is ended", async () => { + const island = await testCluster.island(0); + const oldReplica = island.replica(); + + const clientEndLoggerSpy = jest.spyOn( + testCluster.options.loggers, + "clientEndLogger", + ); + const clientConnectedLoggerSpy = jest.spyOn( + testCluster.options.loggers, + "clientConnectedLogger", + ); + testCluster.options.islands = () => [ + { no: 0, nodes: [TEST_CONFIG, { ...TEST_CONFIG, nameSuffix: "modified" }] }, + ]; + await testCluster.rediscover(); + + expect(clientEndLoggerSpy).toBeCalled(); + expect(clientConnectedLoggerSpy).toBeCalled(); + await waitForExpect(() => expect(oldReplica.isEnded()).toBeTruthy()); + await expect( + oldReplica.query({ + query: ["SELECT 1"], + isWrite: false, + annotations: [], + op: "test", + table: "test", + }), + ).rejects.toThrow(/ended/); +}); diff --git a/src/pg/__tests__/PgSchema.node-down.test.ts b/src/pg/__tests__/PgSchema.node-down.test.ts new file mode 100644 index 0000000..d20b068 --- /dev/null +++ b/src/pg/__tests__/PgSchema.node-down.test.ts @@ -0,0 +1,117 @@ +import waitForExpect from "wait-for-expect"; +import { ClientError } from "../../abstract/ClientError"; +import { type Shard } from "../../abstract/Shard"; +import { maybeCall } from "../../internal/misc"; +import { PgSchema } from "../PgSchema"; +import type { TestPgClient } from "./test-utils"; +import { + TCPProxyServer, + TEST_CONFIG, + recreateTestTables, + shardRun, + testCluster, +} from "./test-utils"; + +jest.useFakeTimers({ advanceTimers: true }); + +const schema = new PgSchema( + 'pg-schema.node-down"table', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + }, + [], +); + +let shard: Shard; +let proxyServer: TCPProxyServer; + +beforeEach(async () => { + testCluster.options.runOnShardErrorRetryCount = 1; + testCluster.options.runOnShardErrorRediscoverClusterDelayMs = 10000; + testCluster.options.reloadIslandsIntervalMs = 10; + testCluster.options.shardsDiscoverIntervalMs = 100000; + + shard = await testCluster.randomShard(); + + proxyServer = new TCPProxyServer({ + host: TEST_CONFIG.host!, + port: TEST_CONFIG.port!, + }); + const proxyTestConfig = { + ...TEST_CONFIG, + isAlwaysLaggingReplica: true, + ...(await proxyServer.hostPort()), + connectionTimeoutMillis: 30000, + }; + + testCluster.options.islands = [ + { no: 0, nodes: [TEST_CONFIG, proxyTestConfig] }, + ]; + await testCluster.rediscover(); + + await recreateTestTables([ + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); +}); + +afterEach(async () => { + await proxyServer.destroy(); +}); + +test("chooses another replica if new connection can't be opened", async () => { + await proxyServer.destroy(); + const promise = shardRun(shard, schema.select({ where: {}, limit: 1 })); + + jest.advanceTimersByTime( + maybeCall(testCluster.options.runOnShardErrorRediscoverClusterDelayMs) * + 1.5, + ); + await waitForExpect(() => + expect( + "" + + jest.mocked(testCluster.options.loggers.runOnShardErrorLogger!).mock + .lastCall?.[0].error, + ).toContain("ECONNREFUSED"), + ); + + await jest.advanceTimersByTimeAsync( + maybeCall(testCluster.options.runOnShardErrorRediscoverClusterDelayMs) * 3, + ); + await promise; +}); + +test("retries on another replica if connection is aborted mid-query", async () => { + const island0 = await testCluster.island(0); + const replica = island0.replica(); + await proxyServer.abortConnections(); + + const promise = replica + .query({ + query: ["SELECT pg_sleep(40000)"], + isWrite: false, + annotations: [], + op: "test", + table: "test", + }) + .catch((e: unknown) => e); + + await proxyServer.waitForAtLeastConnections(1); + await proxyServer.abortConnections(); + + const error = await promise; + if (!(error instanceof ClientError)) { + throw error; + } + + expect(error.postAction).toEqual("choose-another-client"); +}); diff --git a/src/pg/__tests__/PgSchema.rediscover.test.ts b/src/pg/__tests__/PgSchema.rediscover.test.ts new file mode 100644 index 0000000..539b639 --- /dev/null +++ b/src/pg/__tests__/PgSchema.rediscover.test.ts @@ -0,0 +1,99 @@ +import range from "lodash/range"; +import waitForExpect from "wait-for-expect"; +import type { Shard } from "../../abstract/Shard"; +import { MASTER } from "../../abstract/Shard"; +import { CachedRefreshedValue } from "../../internal/CachedRefreshedValue"; +import { mapJoin, maybeCall } from "../../internal/misc"; +import { escapeIdent } from "../helpers/escapeIdent"; +import { PgSchema } from "../PgSchema"; +import type { TestPgClient } from "./test-utils"; +import { recreateTestTables, shardRun, testCluster } from "./test-utils"; + +const schema = new PgSchema( + 'pg-schema.rediscover"table', + { + id: { type: String, autoInsert: "id_gen()" }, + name: { type: String }, + }, + [], +); + +const TABLE_BAK = `${schema.name}_bak`; +const ID_FROM_UNKNOWN_SHARD = "510001234567"; + +let shard: Shard; +let master: TestPgClient; + +beforeEach(async () => { + await recreateTestTables([ + { + CREATE: [ + `DROP TABLE IF EXISTS ${escapeIdent(TABLE_BAK)} CASCADE`, + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + name text NOT NULL + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); + + testCluster.options.runOnShardErrorRediscoverClusterDelayMs = 1000; + testCluster.options.shardsDiscoverIntervalMs = 1_000_000; + await testCluster.rediscover(); + + shard = await testCluster.randomShard(); + master = await shard.client(MASTER); +}); + +test("shard relocation error when accessing a table should be retried", async () => { + testCluster.options.runOnShardErrorRetryCount = 30; + + await master.rows("ALTER TABLE %T RENAME TO %T", schema.name, TABLE_BAK); + + const waitRefreshSpy = jest.spyOn( + CachedRefreshedValue.prototype, + "refreshAndWait", + ); + + const queries = range(50).map((i) => schema.insert({ name: `test${i}` })); + const queryRunSpies = queries.map((query) => jest.spyOn(query, "run")); + const resPromise = mapJoin(queries, async (query) => shardRun(shard, query)); // runs one batched query + + // Pause until we have at least 2 retries happened. + await waitForExpect( + () => expect(queryRunSpies[0]).toBeCalledTimes(2), + maybeCall(testCluster.options.runOnShardErrorRediscoverClusterDelayMs) * 4, // timeout + maybeCall(testCluster.options.runOnShardErrorRediscoverClusterDelayMs), // retry interval + ); + await expect(queryRunSpies[0].mock.results[0].value).rejects.toThrow( + /undefined_object/, + ); + + // Check that calls to refreshAndWait() were coalesced (i.e. + // Cluster#rediscoverCluster() is coalesce-memoized). Despite we have 50 + // parallel queries, the calls to whole-Cluster rediscovery were coalesced to + // just a few. + expect(waitRefreshSpy.mock.calls.length).toBeLessThan(queries.length / 3); + + // Now, after we had some retries, continue & rename the table back. + await master.rows("ALTER TABLE %T RENAME TO %T", TABLE_BAK, schema.name); + + // The queries succeed (no downtime). + expect((await resPromise)[0]).toMatch(/^\d+$/); +}); + +test("shard-to-island resolution failure should NOT cause rediscovery when running a query", async () => { + const shard = testCluster.shard(ID_FROM_UNKNOWN_SHARD); + await expect( + shardRun(shard, schema.load(ID_FROM_UNKNOWN_SHARD)), + ).rejects.toThrow(/not discoverable/); + expect(testCluster.options.loggers.runOnShardErrorLogger).toBeCalledTimes(1); +}); + +test("shard-to-island resolution failure should NOT cause rediscover when just getting a client", async () => { + const shard = testCluster.shard(ID_FROM_UNKNOWN_SHARD); + await expect(shard.client(MASTER)).rejects.toThrow(/not discoverable/); + expect(testCluster.options.loggers.runOnShardErrorLogger).toBeCalledTimes(1); +}); diff --git a/src/pg/__tests__/PgSchema.stress.test.ts b/src/pg/__tests__/PgSchema.stress.test.ts new file mode 100644 index 0000000..1f1e4ba --- /dev/null +++ b/src/pg/__tests__/PgSchema.stress.test.ts @@ -0,0 +1,90 @@ +import delay from "delay"; +import range from "lodash/range"; +import hash from "object-hash"; +import type { Shard } from "../../abstract/Shard"; +import { mapJoin, nullthrows } from "../../internal/misc"; +import { PgSchema } from "../PgSchema"; +import type { TestPgClient } from "./test-utils"; +import { recreateTestTables, shardRun, testCluster } from "./test-utils"; + +const schema = new PgSchema( + 'pg-schema.stress"table', + { + id: { type: String, autoInsert: "id_gen()" }, + prefix: { type: String }, + name: { type: String }, + }, + ["prefix", "name"], +); + +let shard: Shard; + +beforeEach(async () => { + await recreateTestTables([ + { + CREATE: [ + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + prefix text NOT NULL, + name text NOT NULL, + UNIQUE (prefix, name) + )`, + ], + SCHEMA: schema, + SHARD_AFFINITY: [], + }, + ]); + + shard = await testCluster.randomShard(); +}); + +test("stress", async () => { + await runStress(50, 18, async (uniq) => { + const names = range(2).map((i) => i + "-" + uniq); + + const ids = await mapJoin(names, async (name) => + shardRun(shard, schema.insert({ prefix: "pfx", name })), + ); + + await mapJoin(ids, async (id, i) => { + const row = nullthrows( + await shardRun(shard, schema.load(nullthrows(id))), + ); + expect(row.name).toEqual(names[i]); + }); + + await mapJoin(ids, async (id, i) => { + const row = nullthrows( + await shardRun(shard, schema.loadBy({ prefix: "pfx", name: names[i] })), + ); + expect(row.id).toEqual(id); + }); + + await mapJoin(ids, async (id, i) => { + const updated = await shardRun( + shard, + schema.update(nullthrows(id), { name: `upd${names[i]}` }), + ); + expect(updated).toBeTruthy(); + const row = nullthrows( + await shardRun(shard, schema.load(nullthrows(id))), + ); + expect(row.name).toEqual(`upd${names[i]}`); + }); + }); +}); + +async function runStress( + parallelism: number, + iterations: number, + func: (uniq: string) => Promise, +): Promise { + let uniq = ""; + await mapJoin(range(parallelism), async () => { + for (let i = 0; i < iterations; i++) { + await delay(Math.round(Math.random() * 5)); + uniq = hash(uniq); + await func(uniq); + } + }); +} diff --git a/src/pg/__tests__/PgTimelineStorage.test.ts b/src/pg/__tests__/PgTimelineStorage.test.ts new file mode 100644 index 0000000..73541be --- /dev/null +++ b/src/pg/__tests__/PgTimelineStorage.test.ts @@ -0,0 +1,61 @@ +import { GLOBAL_SHARD } from "../../ent/ShardAffinity"; +import { PgTimelineStorage } from "../PgTimelineStorage"; +import { recreateTestTables, testCluster } from "./test-utils"; + +let storage: PgTimelineStorage; + +beforeEach(async () => { + const ddl = { + CREATE: [ + `CREATE UNLOGGED TABLE %T( + id bigserial PRIMARY KEY, + principal text NOT NULL, + data text NOT NULL, + created_at timestamptz NOT NULL + )`, + "CREATE INDEX ON %T (principal)", + ], + SCHEMA: { name: 'pg-timeline-storage.timelines"table' }, + }; + await recreateTestTables([ + { ...ddl, SHARD_AFFINITY: [] }, + { ...ddl, SHARD_AFFINITY: GLOBAL_SHARD }, + ]); + + storage = new PgTimelineStorage({ + cluster: testCluster, + table: ddl.SCHEMA.name, + merge: (dataStrs) => dataStrs.join(";"), + maxChunksPerPrincipal: 3, + }); +}); + +test("load and save in a microshard", async () => { + const PRINCIPAL = "100020000042"; + + expect(await storage.load(PRINCIPAL)).toEqual([]); + + await storage.save(PRINCIPAL, "chunk1"); + expect(await storage.load(PRINCIPAL)).toEqual(["chunk1"]); + + await storage.save(PRINCIPAL, "chunk2"); + await storage.save(PRINCIPAL, "chunk3"); + expect(await storage.load(PRINCIPAL)).toEqual(["chunk1", "chunk2", "chunk3"]); + + await storage.save(PRINCIPAL, "chunk4"); + expect(await storage.load(PRINCIPAL)).toEqual([ + "chunk1;chunk2;chunk3;chunk4", + ]); + + await storage.save(PRINCIPAL, "chunk5"); + expect(await storage.load(PRINCIPAL)).toEqual([ + "chunk1;chunk2;chunk3;chunk4", + "chunk5", + ]); +}); + +test("load and save in global shard", async () => { + const PRINCIPAL = "some"; + await storage.save(PRINCIPAL, "chunk1"); + expect(await storage.load(PRINCIPAL)).toEqual(["chunk1"]); +}); diff --git a/src/pg/__tests__/__snapshots__/PgSchema.alias-pk.test.ts.snap b/src/pg/__tests__/__snapshots__/PgSchema.alias-pk.test.ts.snap new file mode 100644 index 0000000..6fa089e --- /dev/null +++ b/src/pg/__tests__/__snapshots__/PgSchema.alias-pk.test.ts.snap @@ -0,0 +1,159 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`single ops 1`] = ` +" +INSERT INTO "pg-schema.alias-pk""table" (name, user_id) VALUES + ('n1', '1') ON CONFLICT DO NOTHING RETURNING user_id AS id +---- +INSERT INTO "pg-schema.alias-pk""table" (name, user_id) VALUES + ('?', '?') ON CONFLICT DO NOTHING RETURNING user_id AS id +" +`; + +exports[`single ops 2`] = ` +" +WITH rows(user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.alias-pk""table").user_id, (NULL::"pg-schema.alias-pk""table").name, ''), + ('2', 'n2', '')), + updates AS ( + UPDATE "pg-schema.alias-pk""table" SET user_id=rows.user_id, name=rows.name + FROM rows WHERE "pg-schema.alias-pk""table".user_id=rows.user_id + RETURNING rows._key, "pg-schema.alias-pk""table".user_id AS id), + inserts AS ( + INSERT INTO "pg-schema.alias-pk""table" (user_id, name) + SELECT user_id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (user_id) DO UPDATE SET user_id=EXCLUDED.user_id, name=EXCLUDED.name + RETURNING NULL AS _key, user_id AS id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.alias-pk""table").user_id, (NULL::"pg-schema.alias-pk""table").name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.alias-pk""table" SET user_id=rows.user_id, name=rows.name + FROM rows WHERE "pg-schema.alias-pk""table".user_id=rows.user_id + RETURNING rows._key, "pg-schema.alias-pk""table".user_id AS id), + inserts AS ( + INSERT INTO "pg-schema.alias-pk""table" (user_id, name) + SELECT user_id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (user_id) DO UPDATE SET user_id=EXCLUDED.user_id, name=EXCLUDED.name + RETURNING NULL AS _key, user_id AS id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`single ops 3`] = ` +" +INSERT INTO "pg-schema.alias-pk""table" (name, user_id) VALUES + ('n3', id_gen()) ON CONFLICT DO NOTHING RETURNING user_id AS id +---- +INSERT INTO "pg-schema.alias-pk""table" (name, user_id) VALUES + ('?', id_gen()) ON CONFLICT DO NOTHING RETURNING user_id AS id +" +`; + +exports[`single ops 4`] = `""`; + +exports[`single ops 5`] = ` +" +WITH rows(user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.alias-pk""table").user_id, (NULL::"pg-schema.alias-pk""table").name, ''), + ('101', 'n11', '')), + updates AS ( + UPDATE "pg-schema.alias-pk""table" SET user_id=rows.user_id, name=rows.name + FROM rows WHERE "pg-schema.alias-pk""table".user_id=rows.user_id + RETURNING rows._key, "pg-schema.alias-pk""table".user_id AS id), + inserts AS ( + INSERT INTO "pg-schema.alias-pk""table" (user_id, name) + SELECT user_id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (user_id) DO UPDATE SET user_id=EXCLUDED.user_id, name=EXCLUDED.name + RETURNING NULL AS _key, user_id AS id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.alias-pk""table").user_id, (NULL::"pg-schema.alias-pk""table").name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.alias-pk""table" SET user_id=rows.user_id, name=rows.name + FROM rows WHERE "pg-schema.alias-pk""table".user_id=rows.user_id + RETURNING rows._key, "pg-schema.alias-pk""table".user_id AS id), + inserts AS ( + INSERT INTO "pg-schema.alias-pk""table" (user_id, name) + SELECT user_id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (user_id) DO UPDATE SET user_id=EXCLUDED.user_id, name=EXCLUDED.name + RETURNING NULL AS _key, user_id AS id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`single ops 6`] = ` +" +DELETE FROM "pg-schema.alias-pk""table" WHERE user_id IN('1') RETURNING user_id AS id +---- +DELETE FROM "pg-schema.alias-pk""table" WHERE user_id IN('?') RETURNING user_id AS id +" +`; + +exports[`single ops 7`] = ` +" +DELETE FROM "pg-schema.alias-pk""table" WHERE user_id IN('2') AND (1=1) RETURNING user_id AS id +---- +DELETE FROM "pg-schema.alias-pk""table" WHERE user_id IN('?') AND (?=?) RETURNING user_id AS id +" +`; + +exports[`single ops 8`] = ` +" +SELECT user_id, name, user_id AS id FROM "pg-schema.alias-pk""table" + WHERE user_id IN('') +---- +SELECT user_id, name, user_id AS id FROM "pg-schema.alias-pk""table" + WHERE user_id IN('?') +" +`; + +exports[`single ops 9`] = ` +" +SELECT user_id, name, user_id AS id FROM "pg-schema.alias-pk""table" + WHERE user_id='' +---- +SELECT user_id, name, user_id AS id FROM "pg-schema.alias-pk""table" + WHERE user_id='?' +" +`; + +exports[`single ops 10`] = ` +" +SELECT COUNT(1) AS count FROM "pg-schema.alias-pk""table" + WHERE user_id='' +---- +SELECT COUNT(?) AS count FROM "pg-schema.alias-pk""table" + WHERE user_id='?' +" +`; + +exports[`single ops 11`] = ` +" +SELECT EXISTS (SELECT true FROM "pg-schema.alias-pk""table" + WHERE user_id='') +---- +SELECT EXISTS (SELECT true FROM "pg-schema.alias-pk""table" + WHERE user_id='?') +" +`; + +exports[`single ops 12`] = ` +" +SELECT user_id, name, user_id AS id FROM "pg-schema.alias-pk""table" + WHERE user_id='' + LIMIT 10 +---- +SELECT user_id, name, user_id AS id FROM "pg-schema.alias-pk""table" + WHERE user_id='?' + LIMIT ? +" +`; diff --git a/src/pg/__tests__/__snapshots__/PgSchema.cas.test.ts.snap b/src/pg/__tests__/__snapshots__/PgSchema.cas.test.ts.snap new file mode 100644 index 0000000..7b88c11 --- /dev/null +++ b/src/pg/__tests__/__snapshots__/PgSchema.cas.test.ts.snap @@ -0,0 +1,77 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CAS batched success 1`] = ` +" +WITH rows(id, name, ts, updated_at, "$cas.name", "$cas.ts", "$cas.updated_at", _key) AS (VALUES + ((NULL::"pg-schema.cas""table").id, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").ts, (NULL::"pg-schema.cas""table").updated_at, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").ts, (NULL::"pg-schema.cas""table").updated_at, ''), + ('', 'a-upd', '', now(), 'a', '', '', ''), + ('', 'b-upd', '', now(), 'b', NULL, '', '')) + UPDATE "pg-schema.cas""table" SET name=rows.name, ts=rows.ts, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.cas""table".id=rows.id AND ROW("pg-schema.cas""table".name, date_trunc('ms', "pg-schema.cas""table".ts), date_trunc('ms', "pg-schema.cas""table".updated_at)) IS NOT DISTINCT FROM ROW(rows."$cas.name", rows."$cas.ts", rows."$cas.updated_at") RETURNING rows._key +---- +WITH rows(id, name, ts, updated_at, "$cas.name", "$cas.ts", "$cas.updated_at", _key) AS (VALUES + ((NULL::"pg-schema.cas""table").id, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").ts, (NULL::"pg-schema.cas""table").updated_at, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").ts, (NULL::"pg-schema.cas""table").updated_at, '?'), + ('?', '?', '?', now(), '?', '?', '?', '?'), + ('?', '?', '?', now(), '?', NULL, '?', '?')) + UPDATE "pg-schema.cas""table" SET name=rows.name, ts=rows.ts, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.cas""table".id=rows.id AND ROW("pg-schema.cas""table".name, date_trunc('?', "pg-schema.cas""table".ts), date_trunc('?', "pg-schema.cas""table".updated_at)) IS NOT DISTINCT FROM ROW(rows."$cas.name", rows."$cas.ts", rows."$cas.updated_at") RETURNING rows._key +" +`; + +exports[`CAS batched success and skip for the same row but different $cas values 1`] = ` +" +WITH rows(id, name, updated_at, "$cas.name", "$cas.updated_at", _key) AS (VALUES + ((NULL::"pg-schema.cas""table").id, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").updated_at, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").updated_at, ''), + ('', 'a-skip', now(), 'a', '', ''), + ('', 'a-upd', now(), 'a', '', '')) + UPDATE "pg-schema.cas""table" SET name=rows.name, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.cas""table".id=rows.id AND ROW("pg-schema.cas""table".name, date_trunc('ms', "pg-schema.cas""table".updated_at)) IS NOT DISTINCT FROM ROW(rows."$cas.name", rows."$cas.updated_at") RETURNING rows._key +---- +WITH rows(id, name, updated_at, "$cas.name", "$cas.updated_at", _key) AS (VALUES + ((NULL::"pg-schema.cas""table").id, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").updated_at, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").updated_at, '?'), + ('?', '?', now(), '?', '?', '?'), ...) + UPDATE "pg-schema.cas""table" SET name=rows.name, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.cas""table".id=rows.id AND ROW("pg-schema.cas""table".name, date_trunc('?', "pg-schema.cas""table".updated_at)) IS NOT DISTINCT FROM ROW(rows."$cas.name", rows."$cas.updated_at") RETURNING rows._key +" +`; + +exports[`CAS disables coalescing for the same row with the same $cas values 1`] = ` +" +WITH rows(id, name, updated_at, "$cas.name", _key) AS (VALUES + ((NULL::"pg-schema.cas""table").id, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").updated_at, (NULL::"pg-schema.cas""table").name, ''), + ('', 'a-skip', now(), 'a', ''), + ('', 'a-upd', now(), 'a', '')) + UPDATE "pg-schema.cas""table" SET name=rows.name, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.cas""table".id=rows.id AND ROW("pg-schema.cas""table".name) IS NOT DISTINCT FROM ROW(rows."$cas.name") RETURNING rows._key +---- +WITH rows(id, name, updated_at, "$cas.name", _key) AS (VALUES + ((NULL::"pg-schema.cas""table").id, (NULL::"pg-schema.cas""table").name, (NULL::"pg-schema.cas""table").updated_at, (NULL::"pg-schema.cas""table").name, '?'), + ('?', '?', now(), '?', '?'), ...) + UPDATE "pg-schema.cas""table" SET name=rows.name, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.cas""table".id=rows.id AND ROW("pg-schema.cas""table".name) IS NOT DISTINCT FROM ROW(rows."$cas.name") RETURNING rows._key +" +`; + +exports[`CAS single skip 1`] = ` +" +UPDATE "pg-schema.cas""table" SET name='a-skip', updated_at=now() WHERE id='' AND ROW(name, date_trunc('ms', updated_at)) IS NOT DISTINCT FROM ROW('a-old', '') RETURNING id +---- +UPDATE "pg-schema.cas""table" SET name='?', updated_at=now() WHERE id='?' AND ROW(name, date_trunc('?', updated_at)) IS NOT DISTINCT FROM ROW('?', '?') RETURNING id + +UPDATE "pg-schema.cas""table" SET name='b-skip', updated_at=now() WHERE id='' AND ROW(buffer_field) IS NOT DISTINCT FROM ROW('\\x040506') RETURNING id +---- +UPDATE "pg-schema.cas""table" SET name='?', updated_at=now() WHERE id='?' AND ROW(buffer_field) IS NOT DISTINCT FROM ROW('?') RETURNING id +" +`; + +exports[`CAS single success 1`] = ` +" +UPDATE "pg-schema.cas""table" SET name='a-upd', ts='', updated_at=now() WHERE id='' AND ROW(name, date_trunc('ms', ts), encrypted_field, buffer_field, date_trunc('ms', updated_at)) IS NOT DISTINCT FROM ROW('a', '', 'encrypted:‹', '\\x010203', '') RETURNING id +---- +UPDATE "pg-schema.cas""table" SET name='?', ts='?', updated_at=now() WHERE id='?' AND ROW(name, date_trunc('?', ts), encrypted_field, buffer_field, date_trunc('?', updated_at)) IS NOT DISTINCT FROM ROW('?', '?', '?', '?', '?') RETURNING id + +UPDATE "pg-schema.cas""table" SET name='b-upd', ts='', updated_at=now() WHERE id='' AND ROW(name, date_trunc('ms', ts), date_trunc('ms', updated_at)) IS NOT DISTINCT FROM ROW('b', NULL, '') RETURNING id +---- +UPDATE "pg-schema.cas""table" SET name='?', ts='?', updated_at=now() WHERE id='?' AND ROW(name, date_trunc('?', ts), date_trunc('?', updated_at)) IS NOT DISTINCT FROM ROW('?', NULL, '?') RETURNING id +" +`; diff --git a/src/pg/__tests__/__snapshots__/PgSchema.composite-pk.test.ts.snap b/src/pg/__tests__/__snapshots__/PgSchema.composite-pk.test.ts.snap new file mode 100644 index 0000000..2cf336b --- /dev/null +++ b/src/pg/__tests__/__snapshots__/PgSchema.composite-pk.test.ts.snap @@ -0,0 +1,311 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`batched ops 1`] = ` +" +WITH rows(name, tenant_id, user_id, _key) AS (VALUES + ((NULL::"pg-schema.composite-pk""table").name, (NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id, ''), + ('n1', '1', '1', ''), + ('n2', '1', '2', ''), + ('n3', '1', '3', ''), + ('n4', '1', '4', '')) + INSERT INTO "pg-schema.composite-pk""table" (name, tenant_id, user_id) + SELECT name, tenant_id, user_id FROM rows OFFSET 1 + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE ROW(rows.tenant_id,rows.user_id)=ROW("pg-schema.composite-pk""table".tenant_id,"pg-schema.composite-pk""table".user_id)), ROW(tenant_id,user_id) AS id +---- +WITH rows(name, tenant_id, user_id, _key) AS (VALUES + ((NULL::"pg-schema.composite-pk""table").name, (NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id, '?'), + ('?', '?', '?', '?'), ...) + INSERT INTO "pg-schema.composite-pk""table" (name, tenant_id, user_id) + SELECT name, tenant_id, user_id FROM rows OFFSET ? + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE ROW(rows.tenant_id,rows.user_id)=ROW("pg-schema.composite-pk""table".tenant_id,"pg-schema.composite-pk""table".user_id)), ROW(tenant_id,user_id) AS id +" +`; + +exports[`batched ops 2`] = `""`; + +exports[`batched ops 3`] = ` +" +WITH rows(tenant_id, user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id, (NULL::"pg-schema.composite-pk""table").name, ''), + ('1', '1', 'new-name-1', '(1,1)'), + ('1', '2', 'new-name-2', '(1,2)'), + ('42', '42', 'absent', '(42,42)')) + UPDATE "pg-schema.composite-pk""table" SET name=rows.name + FROM rows WHERE ROW("pg-schema.composite-pk""table".tenant_id,"pg-schema.composite-pk""table".user_id)=ROW(rows.tenant_id,rows.user_id) RETURNING rows._key +---- +WITH rows(tenant_id, user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id, (NULL::"pg-schema.composite-pk""table").name, '?'), + ('?', '?', '?', '?'), ...) + UPDATE "pg-schema.composite-pk""table" SET name=rows.name + FROM rows WHERE ROW("pg-schema.composite-pk""table".tenant_id,"pg-schema.composite-pk""table".user_id)=ROW(rows.tenant_id,rows.user_id) RETURNING rows._key +" +`; + +exports[`batched ops 4`] = ` +" +WITH rows(tenant_id, user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id, (NULL::"pg-schema.composite-pk""table").name, ''), + ('1', '1', 'n11', '["1","1"]'), + ('9', '9', 'n9', '["9","9"]')), + updates AS ( + UPDATE "pg-schema.composite-pk""table" SET tenant_id=rows.tenant_id, user_id=rows.user_id, name=rows.name + FROM rows WHERE "pg-schema.composite-pk""table".tenant_id=rows.tenant_id AND "pg-schema.composite-pk""table".user_id=rows.user_id + RETURNING rows._key, ROW("pg-schema.composite-pk""table".tenant_id,"pg-schema.composite-pk""table".user_id) AS id), + inserts AS ( + INSERT INTO "pg-schema.composite-pk""table" (tenant_id, user_id, name) + SELECT tenant_id, user_id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (tenant_id, user_id) DO UPDATE SET tenant_id=EXCLUDED.tenant_id, user_id=EXCLUDED.user_id, name=EXCLUDED.name + RETURNING NULL AS _key, ROW(tenant_id,user_id) AS id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(tenant_id, user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id, (NULL::"pg-schema.composite-pk""table").name, '?'), + ('?', '?', '?', '?'), ...), + updates AS ( + UPDATE "pg-schema.composite-pk""table" SET tenant_id=rows.tenant_id, user_id=rows.user_id, name=rows.name + FROM rows WHERE "pg-schema.composite-pk""table".tenant_id=rows.tenant_id AND "pg-schema.composite-pk""table".user_id=rows.user_id + RETURNING rows._key, ROW("pg-schema.composite-pk""table".tenant_id,"pg-schema.composite-pk""table".user_id) AS id), + inserts AS ( + INSERT INTO "pg-schema.composite-pk""table" (tenant_id, user_id, name) + SELECT tenant_id, user_id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (tenant_id, user_id) DO UPDATE SET tenant_id=EXCLUDED.tenant_id, user_id=EXCLUDED.user_id, name=EXCLUDED.name + RETURNING NULL AS _key, ROW(tenant_id,user_id) AS id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`batched ops 5`] = ` +" +DELETE FROM "pg-schema.composite-pk""table" WHERE ROW(tenant_id,user_id) IN(ROW('1','1'),ROW('1','2')) RETURNING ROW(tenant_id,user_id) AS id +---- +DELETE FROM "pg-schema.composite-pk""table" WHERE ROW(tenant_id,user_id) IN(ROW('?','?'), ...) RETURNING ROW(tenant_id,user_id) AS id +" +`; + +exports[`batched ops 6`] = ` +" +SELECT tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE ROW(tenant_id,user_id) IN(ROW('1','3'),ROW('1','424')) +---- +SELECT tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE ROW(tenant_id,user_id) IN(ROW('?','?'), ...) +" +`; + +exports[`batched ops 7`] = ` +" +SELECT tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE (tenant_id, user_id) IN(VALUES + ((NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id), + ('1', '3'), + ('1', '4')) +---- +SELECT tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE (tenant_id, user_id) IN(VALUES + ((NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id), + ('?', '?'), ...) +" +`; + +exports[`batched ops 8`] = ` +" +SELECT COUNT(1) AS count FROM "pg-schema.composite-pk""table" + WHERE user_id='3' + UNION ALL +SELECT COUNT(1) AS count FROM "pg-schema.composite-pk""table" + WHERE tenant_id='1' +---- +SELECT COUNT(?) AS count FROM "pg-schema.composite-pk""table" + WHERE user_id='?' + UNION ALL +SELECT COUNT(?) AS count FROM "pg-schema.composite-pk""table" + WHERE tenant_id='?' +" +`; + +exports[`batched ops 9`] = ` +" +SELECT EXISTS (SELECT true FROM "pg-schema.composite-pk""table" + WHERE user_id='3') + UNION ALL +SELECT EXISTS (SELECT true FROM "pg-schema.composite-pk""table" + WHERE tenant_id='1') + UNION ALL +SELECT EXISTS (SELECT true FROM "pg-schema.composite-pk""table" + WHERE tenant_id='199999') +-- Hints:enable_seqscan=off +---- +SELECT EXISTS (SELECT true FROM "pg-schema.composite-pk""table" + WHERE user_id='?') + UNION ALL +SELECT EXISTS (SELECT true FROM "pg-schema.composite-pk""table" + WHERE tenant_id='?') + UNION ALL ...-- Hints:enable_seqscan=off +" +`; + +exports[`batched ops 10`] = ` +" +(SELECT '' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE user_id='3' + ORDER BY user_id ASC + LIMIT 10) + UNION ALL +(SELECT '' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE tenant_id='1' + ORDER BY user_id ASC + LIMIT 10) + UNION ALL +(SELECT '' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE tenant_id='101' + ORDER BY user_id ASC + LIMIT 10) +---- +(SELECT '?' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE user_id='?' + ORDER BY user_id ASC + LIMIT ?) + UNION ALL +(SELECT '?' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE tenant_id='?' + ORDER BY user_id ASC + LIMIT ?) + UNION ALL ... +" +`; + +exports[`batched ops 11`] = ` +" +(SELECT '' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE ROW(tenant_id,user_id)=ROW('1','3') + ORDER BY user_id ASC + LIMIT 10) + UNION ALL +(SELECT '' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE ROW(tenant_id,user_id) IN(ROW('1','3'),ROW('1','4')) + ORDER BY user_id ASC + LIMIT 10) +---- +(SELECT '?' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE ROW(tenant_id,user_id)=ROW('?','?') + ORDER BY user_id ASC + LIMIT ?) + UNION ALL +(SELECT '?' AS _key, tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE ROW(tenant_id,user_id) IN(ROW('?','?'), ...) + ORDER BY user_id ASC + LIMIT ?) +" +`; + +exports[`single ops 1`] = ` +" +INSERT INTO "pg-schema.composite-pk""table" (name, tenant_id, user_id) VALUES + ('n1', '1', '1') ON CONFLICT DO NOTHING RETURNING ROW(tenant_id,user_id) AS id +---- +INSERT INTO "pg-schema.composite-pk""table" (name, tenant_id, user_id) VALUES + ('?', '?', '?') ON CONFLICT DO NOTHING RETURNING ROW(tenant_id,user_id) AS id +" +`; + +exports[`single ops 2`] = `""`; + +exports[`single ops 3`] = ` +" +WITH rows(tenant_id, user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id, (NULL::"pg-schema.composite-pk""table").name, ''), + ('1', '101', 'n11', '')), + updates AS ( + UPDATE "pg-schema.composite-pk""table" SET tenant_id=rows.tenant_id, user_id=rows.user_id, name=rows.name + FROM rows WHERE "pg-schema.composite-pk""table".tenant_id=rows.tenant_id AND "pg-schema.composite-pk""table".user_id=rows.user_id + RETURNING rows._key, ROW("pg-schema.composite-pk""table".tenant_id,"pg-schema.composite-pk""table".user_id) AS id), + inserts AS ( + INSERT INTO "pg-schema.composite-pk""table" (tenant_id, user_id, name) + SELECT tenant_id, user_id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (tenant_id, user_id) DO UPDATE SET tenant_id=EXCLUDED.tenant_id, user_id=EXCLUDED.user_id, name=EXCLUDED.name + RETURNING NULL AS _key, ROW(tenant_id,user_id) AS id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(tenant_id, user_id, name, _key) AS (VALUES + ((NULL::"pg-schema.composite-pk""table").tenant_id, (NULL::"pg-schema.composite-pk""table").user_id, (NULL::"pg-schema.composite-pk""table").name, '?'), + ('?', '?', '?', '?')), + updates AS ( + UPDATE "pg-schema.composite-pk""table" SET tenant_id=rows.tenant_id, user_id=rows.user_id, name=rows.name + FROM rows WHERE "pg-schema.composite-pk""table".tenant_id=rows.tenant_id AND "pg-schema.composite-pk""table".user_id=rows.user_id + RETURNING rows._key, ROW("pg-schema.composite-pk""table".tenant_id,"pg-schema.composite-pk""table".user_id) AS id), + inserts AS ( + INSERT INTO "pg-schema.composite-pk""table" (tenant_id, user_id, name) + SELECT tenant_id, user_id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (tenant_id, user_id) DO UPDATE SET tenant_id=EXCLUDED.tenant_id, user_id=EXCLUDED.user_id, name=EXCLUDED.name + RETURNING NULL AS _key, ROW(tenant_id,user_id) AS id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`single ops 4`] = ` +" +UPDATE "pg-schema.composite-pk""table" SET name='absent' WHERE ROW(tenant_id,user_id)=ROW('42','42') RETURNING ROW(tenant_id,user_id) AS id +---- +UPDATE "pg-schema.composite-pk""table" SET name='?' WHERE ROW(tenant_id,user_id)=ROW('?','?') RETURNING ROW(tenant_id,user_id) AS id +" +`; + +exports[`single ops 5`] = ` +" +UPDATE "pg-schema.composite-pk""table" SET name='new-name' WHERE ROW(tenant_id,user_id)=ROW('1','1') RETURNING ROW(tenant_id,user_id) AS id +---- +UPDATE "pg-schema.composite-pk""table" SET name='?' WHERE ROW(tenant_id,user_id)=ROW('?','?') RETURNING ROW(tenant_id,user_id) AS id +" +`; + +exports[`single ops 6`] = ` +" +DELETE FROM "pg-schema.composite-pk""table" WHERE ROW(tenant_id,user_id) IN(ROW('1','1')) RETURNING ROW(tenant_id,user_id) AS id +---- +DELETE FROM "pg-schema.composite-pk""table" WHERE ROW(tenant_id,user_id) IN(ROW('?','?'), ...) RETURNING ROW(tenant_id,user_id) AS id +" +`; + +exports[`single ops 7`] = ` +" +DELETE FROM "pg-schema.composite-pk""table" WHERE ROW(tenant_id,user_id) IN(ROW('1','2')) AND (1=1) RETURNING ROW(tenant_id,user_id) AS id +---- +DELETE FROM "pg-schema.composite-pk""table" WHERE ROW(tenant_id,user_id) IN(ROW('?','?'), ...) AND (?=?) RETURNING ROW(tenant_id,user_id) AS id +" +`; + +exports[`single ops 8`] = ` +" +SELECT COUNT(1) AS count FROM "pg-schema.composite-pk""table" + WHERE user_id='3' +---- +SELECT COUNT(?) AS count FROM "pg-schema.composite-pk""table" + WHERE user_id='?' +" +`; + +exports[`single ops 9`] = ` +" +SELECT EXISTS (SELECT true FROM "pg-schema.composite-pk""table" + WHERE user_id='3') +---- +SELECT EXISTS (SELECT true FROM "pg-schema.composite-pk""table" + WHERE user_id='?') +" +`; + +exports[`single ops 10`] = ` +" +SELECT tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE user_id='3' + LIMIT 10 +---- +SELECT tenant_id, user_id, name, ROW(tenant_id,user_id) AS id FROM "pg-schema.composite-pk""table" + WHERE user_id='?' + LIMIT ? +" +`; diff --git a/src/pg/__tests__/__snapshots__/PgSchema.generic.test.ts.snap b/src/pg/__tests__/__snapshots__/PgSchema.generic.test.ts.snap new file mode 100644 index 0000000..85d9573 --- /dev/null +++ b/src/pg/__tests__/__snapshots__/PgSchema.generic.test.ts.snap @@ -0,0 +1,1457 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`delete batched 1`] = ` +" +DELETE FROM "pg-schema.generic""table" WHERE (id=ANY('{,,42}') OR id IS NULL) RETURNING id +---- +DELETE FROM "pg-schema.generic""table" WHERE (id=ANY('?') OR id IS NULL) RETURNING id +" +`; + +exports[`delete single 1`] = ` +" +DELETE FROM "pg-schema.generic""table" WHERE id='' RETURNING id +---- +DELETE FROM "pg-schema.generic""table" WHERE id='?' RETURNING id + +DELETE FROM "pg-schema.generic""table" WHERE id='42' RETURNING id +---- +DELETE FROM "pg-schema.generic""table" WHERE id='?' RETURNING id + +DELETE FROM "pg-schema.generic""table" WHERE id IS NULL RETURNING id +---- +DELETE FROM "pg-schema.generic""table" WHERE id IS NULL RETURNING id +" +`; + +exports[`delete where 1`] = ` +" +DELETE FROM "pg-schema.generic""table" WHERE id=ANY('{,}') AND (1=1) RETURNING id +---- +DELETE FROM "pg-schema.generic""table" WHERE id=ANY('?') AND (?=?) RETURNING id +" +`; + +exports[`idGen batched 1`] = ` +" +SELECT id_gen() AS k*, id_gen() AS k* +---- +SELECT id_gen() AS k*, id_gen() AS k* +" +`; + +exports[`idGen single 1`] = ` +" +SELECT id_gen() +---- +SELECT id_gen() + +SELECT id_gen() +---- +SELECT id_gen() +" +`; + +exports[`insert batched 1`] = ` +" +SELECT id_gen() +---- +SELECT id_gen() + +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, ''), + ('a', 'u', NULL, NULL, NULL, NULL, NULL, now(), now(), NULL, '', ''), + ('a', 'u_dup', false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), ''), + ('z''b''c', NULL, true, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET 1 + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +---- +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, '?'), + ('?', '?', NULL, NULL, NULL, NULL, NULL, now(), now(), NULL, '?', '?'), + ('?', '?', false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '?'), + ('?', NULL, true, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '?')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET ? + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +" +`; + +exports[`insert de-batches pg error 1`] = ` +" +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, ''), + ('other', NULL, false, NULL, NULL, NULL, NULL, now(), now(), '0', id_gen(), ''), + ('some', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET 1 + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +---- +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, '?'), + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), '?', id_gen(), '?'), + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '?')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET ? + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id + +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('some', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id +---- +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id + +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('other', NULL, false, NULL, NULL, NULL, NULL, now(), now(), '0', id_gen()) ON CONFLICT DO NOTHING RETURNING id +---- +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), '?', id_gen()) ON CONFLICT DO NOTHING RETURNING id +" +`; + +exports[`insert de-batches pg error: data type: bigint 1`] = ` +" +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, ''), + ('other', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, 'not a bigint', ''), + ('some', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET 1 + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +---- +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, '?'), + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, '?', '?'), + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '?')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET ? + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id + +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('some', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id +---- +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id + +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('other', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, 'not a bigint') ON CONFLICT DO NOTHING RETURNING id +---- +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, '?') ON CONFLICT DO NOTHING RETURNING id +" +`; + +exports[`insert de-batches pg error: data type: varchar 1`] = ` +" +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, ''), + ('loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooog', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), ''), + ('some', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET 1 + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +---- +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, '?'), + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '?'), ...) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET ? + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id + +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('some', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id +---- +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id + +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooog', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id +---- +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('?', NULL, false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id +" +`; + +exports[`insert is never dedupped 1`] = ` +" +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, ''), + ('aaa', 'uuu', NULL, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), ''), + ('aaa', 'uuu', NULL, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET 1 + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +---- +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, '?'), + ('?', '?', NULL, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen(), '?'), ...) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET ? + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +" +`; + +exports[`insert pre-generated id with uniq key violation 1`] = ` +" +SELECT id_gen() +---- +SELECT id_gen() + +SELECT id_gen() +---- +SELECT id_gen() + +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, ''), + ('some', 'aaa', false, NULL, NULL, NULL, NULL, now(), now(), NULL, '', ''), + ('some', 'aaa', false, NULL, NULL, NULL, NULL, now(), now(), NULL, '', '')) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET 1 + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +---- +WITH rows(name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").json_text_field, (NULL::"pg-schema.generic""table").json_strongly_typed_field, (NULL::"pg-schema.generic""table").jsonb_field, (NULL::"pg-schema.generic""table").encrypted_field, (NULL::"pg-schema.generic""table").created_at, (NULL::"pg-schema.generic""table").updated_at, (NULL::"pg-schema.generic""table").parent_id, (NULL::"pg-schema.generic""table").id, '?'), + ('?', '?', false, NULL, NULL, NULL, NULL, now(), now(), NULL, '?', '?'), ...) + INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) + SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM rows OFFSET ? + ON CONFLICT DO NOTHING RETURNING (SELECT _key FROM rows WHERE rows.id="pg-schema.generic""table".id), id +" +`; + +exports[`insert single 1`] = ` +" +SELECT id_gen() +---- +SELECT id_gen() + +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('a''b''c', 'aaa', true, '{"a":10,"b":{"c":20}}', '{"a":42}', '{"a":42}', 'encrypted:ufyu', now(), now(), NULL, '') ON CONFLICT DO NOTHING RETURNING id +---- +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('?', '?', true, '?', '?', '?', '?', now(), now(), NULL, '?') ON CONFLICT DO NOTHING RETURNING id + +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('a''b''c', 'aaa_dup', false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id +---- +INSERT INTO "pg-schema.generic""table" (name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id) VALUES + ('?', '?', false, NULL, NULL, NULL, NULL, now(), now(), NULL, id_gen()) ON CONFLICT DO NOTHING RETURNING id +" +`; + +exports[`load batched 1`] = ` +" +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE id=ANY('{,}') +---- +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE id=ANY('?') +" +`; + +exports[`loadBy batched one column 1`] = ` +" +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name=ANY('{"a\\\\a","b\\\\nb","no value"}') +---- +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name=ANY('?') +" +`; + +exports[`loadBy batched two columns 1`] = ` +" +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col" + WHERE (name, url_name) IN(VALUES + ((NULL::"pg-schema.generic""table_2col").name, (NULL::"pg-schema.generic""table_2col").url_name), + ('z', 'z1'), + ('z', 'z,2'), + ('b', 'no\\value'), + ('b', 'b{1}'), + ('c', 'NuLL'), + ('c', '')) +---- +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col" + WHERE (name, url_name) IN(VALUES + ((NULL::"pg-schema.generic""table_2col").name, (NULL::"pg-schema.generic""table_2col").url_name), + ('?', '?'), ...) +" +`; + +exports[`loadBy batched with two columns nullable unique key 1`] = ` +" +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col_nullable_unique_key" + WHERE (name='z' + AND url_name=ANY('{z1,z2}')) + OR (name='no' + AND url_name='Null') + OR (name='b' + AND url_name IS NULL) + OR (name='c' + AND (url_name='c2' + OR url_name IS NULL)) +---- +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col_nullable_unique_key" + WHERE (name='?' + AND url_name=ANY('?')) + OR (name='?' + AND url_name='?') + OR (name='?' + AND url_name IS NULL) + OR (name='?' + AND (url_name='?' + OR url_name IS NULL)) +" +`; + +exports[`loadBy single one column 1`] = ` +" +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='b' +---- +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='?' +" +`; + +exports[`loadBy single two columns 1`] = ` +" +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col" + WHERE (name='b' + AND url_name='bbb') +---- +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col" + WHERE (name='?' + AND url_name='?') +" +`; + +exports[`loadBy single two columns with nullable unique key 1`] = ` +" +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col_nullable_unique_key" + WHERE (name='b' + AND url_name IS NULL) +---- +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col_nullable_unique_key" + WHERE (name='?' + AND url_name IS NULL) + +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col_nullable_unique_key" + WHERE (name IS NULL + AND url_name='a') +---- +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col_nullable_unique_key" + WHERE (name IS NULL + AND url_name='?') + +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col_nullable_unique_key" + WHERE (name IS NULL + AND url_name IS NULL) +---- +SELECT id, name, url_name, created_at, updated_at FROM "pg-schema.generic""table_2col_nullable_unique_key" + WHERE (name IS NULL + AND url_name IS NULL) +" +`; + +exports[`rawPrepend hint 1`] = ` +" +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='a' + LIMIT 10 +-- Hints:""=/*+SeqScan("pg-schema.generic""table")*/ +---- +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='?' + LIMIT ? +-- Hints:""=/*+SeqScan("pg-schema.generic""table")*/ + +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='b' + LIMIT 10 +-- Hints:""=/*+SeqScan("pg-schema.generic""table")*/ +---- +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='?' + LIMIT ? +-- Hints:""=/*+SeqScan("pg-schema.generic""table")*/ +" +`; + +exports[`select and count batched 1`] = ` +" +(SELECT '' AS _key, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name=ANY('{"a +a","a\\\\a"}') + AND some_flag=true + AND (name=ANY('{"a +a","a\\\\a"}') + AND name<>'kk' + AND name IS DISTINCT FROM 'dd' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('kk') + AND url_name IS NOT NULL) + AND true/*empty_NOT_IN*/ + AND ('5' > '2') + AND (name>='a' + AND name<='z')) + AND (name='a +a' + OR name='a\\a' + OR false/*empty_ANY*/ + OR (url_name='zzz' + OR url_name IS NULL)) + AND NOT (name='zz' + AND (5 < '2')) + ORDER BY name ASC, url_name DESC, 1=2 + LIMIT 10) + UNION ALL +(SELECT '' AS _key, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='b' + LIMIT 10) +---- +(SELECT '?' AS _key, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name=ANY('?') + AND some_flag=true + AND (name=ANY('?') + AND name<>'?' + AND name IS DISTINCT FROM '?' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('?') + AND url_name IS NOT NULL) + AND true + AND ('?' > '?') + AND (name>='?' + AND name<='?')) + AND (name='?' + OR name='?' + OR false + OR (url_name='?' + OR url_name IS NULL)) + AND NOT (name='?' + AND (? < '?')) + ORDER BY name ASC, url_name DESC, ?=? + LIMIT ?) + UNION ALL +(SELECT '?' AS _key, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='?' + LIMIT ?) + +SELECT COUNT(1) AS count FROM "pg-schema.generic""table" + WHERE name=ANY('{"a +a","a\\\\a"}') + AND some_flag=true + AND (name=ANY('{"a +a","a\\\\a"}') + AND name<>'kk' + AND name IS DISTINCT FROM 'dd' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('kk') + AND url_name IS NOT NULL) + AND true/*empty_NOT_IN*/ + AND ('5' > '2') + AND (name>='a' + AND name<='z')) + AND (name='a +a' + OR name='a\\a' + OR false/*empty_ANY*/ + OR (url_name='zzz' + OR url_name IS NULL)) + AND NOT (name='zz' + AND (5 < '2')) + UNION ALL +SELECT COUNT(1) AS count FROM "pg-schema.generic""table" + WHERE name=ANY('{"a +a","a\\\\a"}') + AND some_flag=true + AND url_name='a1' + AND (name=ANY('{"a +a","a\\\\a"}') + AND name<>'kk' + AND name IS DISTINCT FROM 'dd' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('kk') + AND url_name IS NOT NULL) + AND true/*empty_NOT_IN*/ + AND ('5' > '2') + AND (name>='a' + AND name<='z')) + AND (name='a +a' + OR name='a\\a' + OR false/*empty_ANY*/ + OR (url_name='zzz' + OR url_name IS NULL)) + AND NOT (name='zz' + AND (5 < '2')) + UNION ALL +SELECT COUNT(1) AS count FROM "pg-schema.generic""table" + WHERE name='b' +---- +SELECT COUNT(?) AS count FROM "pg-schema.generic""table" + WHERE name=ANY('?') + AND some_flag=true + AND (name=ANY('?') + AND name<>'?' + AND name IS DISTINCT FROM '?' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('?') + AND url_name IS NOT NULL) + AND true + AND ('?' > '?') + AND (name>='?' + AND name<='?')) + AND (name='?' + OR name='?' + OR false + OR (url_name='?' + OR url_name IS NULL)) + AND NOT (name='?' + AND (? < '?')) + UNION ALL +SELECT COUNT(?) AS count FROM "pg-schema.generic""table" + WHERE name=ANY('?') + AND some_flag=true + AND url_name='?' + AND (name=ANY('?') + AND name<>'?' + AND name IS DISTINCT FROM '?' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('?') + AND url_name IS NOT NULL) + AND true + AND ('?' > '?') + AND (name>='?' + AND name<='?')) + AND (name='?' + OR name='?' + OR false + OR (url_name='?' + OR url_name IS NULL)) + AND NOT (name='?' + AND (? < '?')) + UNION ALL +SELECT COUNT(?) AS count FROM "pg-schema.generic""table" + WHERE name='?' + +SELECT COUNT(1) AS count FROM "pg-schema.generic""table" + WHERE name=ANY('{"a +a","a\\\\a"}') + AND some_flag=true + AND (name=ANY('{"a +a","a\\\\a"}') + AND name<>'kk' + AND name IS DISTINCT FROM 'dd' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('kk') + AND url_name IS NOT NULL) + AND true/*empty_NOT_IN*/ + AND ('5' > '2') + AND (name>='a' + AND name<='z')) + AND (name='a +a' + OR name='a\\a' + OR false/*empty_ANY*/ + OR (url_name='zzz' + OR url_name IS NULL)) + AND NOT (name='zz' + AND (5 < '2')) +---- +SELECT COUNT(?) AS count FROM "pg-schema.generic""table" + WHERE name=ANY('?') + AND some_flag=true + AND (name=ANY('?') + AND name<>'?' + AND name IS DISTINCT FROM '?' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('?') + AND url_name IS NOT NULL) + AND true + AND ('?' > '?') + AND (name>='?' + AND name<='?')) + AND (name='?' + OR name='?' + OR false + OR (url_name='?' + OR url_name IS NULL)) + AND NOT (name='?' + AND (? < '?')) + +SELECT EXISTS (SELECT true FROM "pg-schema.generic""table" + WHERE name=ANY('{"a +a","a\\\\a"}') + AND some_flag=true + AND (name=ANY('{"a +a","a\\\\a"}') + AND name<>'kk' + AND name IS DISTINCT FROM 'dd' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('kk') + AND url_name IS NOT NULL) + AND true/*empty_NOT_IN*/ + AND ('5' > '2') + AND (name>='a' + AND name<='z')) + AND (name='a +a' + OR name='a\\a' + OR false/*empty_ANY*/ + OR (url_name='zzz' + OR url_name IS NULL)) + AND NOT (name='zz' + AND (5 < '2'))) + UNION ALL +SELECT EXISTS (SELECT true FROM "pg-schema.generic""table" + WHERE name=ANY('{"a +a","a\\\\a"}') + AND some_flag=true + AND url_name='a1' + AND (name=ANY('{"a +a","a\\\\a"}') + AND name<>'kk' + AND name IS DISTINCT FROM 'dd' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('kk') + AND url_name IS NOT NULL) + AND true/*empty_NOT_IN*/ + AND ('5' > '2') + AND (name>='a' + AND name<='z')) + AND (name='a +a' + OR name='a\\a' + OR false/*empty_ANY*/ + OR (url_name='zzz' + OR url_name IS NULL)) + AND NOT (name='zz' + AND (5 < '2'))) + UNION ALL +SELECT EXISTS (SELECT true FROM "pg-schema.generic""table" + WHERE name='b') +-- Hints:enable_seqscan=off +---- +SELECT EXISTS (SELECT true FROM "pg-schema.generic""table" + WHERE name=ANY('?') + AND some_flag=true + AND (name=ANY('?') + AND name<>'?' + AND name IS DISTINCT FROM '?' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('?') + AND url_name IS NOT NULL) + AND true + AND ('?' > '?') + AND (name>='?' + AND name<='?')) + AND (name='?' + OR name='?' + OR false + OR (url_name='?' + OR url_name IS NULL)) + AND NOT (name='?' + AND (? < '?'))) + UNION ALL +SELECT EXISTS (SELECT true FROM "pg-schema.generic""table" + WHERE name=ANY('?') + AND some_flag=true + AND url_name='?' + AND (name=ANY('?') + AND name<>'?' + AND name IS DISTINCT FROM '?' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('?') + AND url_name IS NOT NULL) + AND true + AND ('?' > '?') + AND (name>='?' + AND name<='?')) + AND (name='?' + OR name='?' + OR false + OR (url_name='?' + OR url_name IS NULL)) + AND NOT (name='?' + AND (? < '?'))) + UNION ALL +SELECT EXISTS (SELECT true FROM "pg-schema.generic""table" + WHERE name='?') +-- Hints:enable_seqscan=off + +SELECT EXISTS (SELECT true FROM "pg-schema.generic""table" + WHERE name=ANY('{"a +a","a\\\\a"}') + AND some_flag=true + AND (name=ANY('{"a +a","a\\\\a"}') + AND name<>'kk' + AND name IS DISTINCT FROM 'dd' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('kk') + AND url_name IS NOT NULL) + AND true/*empty_NOT_IN*/ + AND ('5' > '2') + AND (name>='a' + AND name<='z')) + AND (name='a +a' + OR name='a\\a' + OR false/*empty_ANY*/ + OR (url_name='zzz' + OR url_name IS NULL)) + AND NOT (name='zz' + AND (5 < '2'))) +---- +SELECT EXISTS (SELECT true FROM "pg-schema.generic""table" + WHERE name=ANY('?') + AND some_flag=true + AND (name=ANY('?') + AND name<>'?' + AND name IS DISTINCT FROM '?' + AND url_name IS DISTINCT FROM NULL + AND (url_name NOT IN('?') + AND url_name IS NOT NULL) + AND true + AND ('?' > '?') + AND (name>='?' + AND name<='?')) + AND (name='?' + OR name='?' + OR false + OR (url_name='?' + OR url_name IS NULL)) + AND NOT (name='?' + AND (? < '?'))) +" +`; + +exports[`select custom 1`] = ` +" +WITH + cte1 AS (SELECT 1+'1' AS cte1_v FROM generate_series(1,1)), + cte2 AS (SELECT 10+'10' AS cte2_v) +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" +JOIN cte1 ON true +JOIN (SELECT 10+'10' AS join2_v) join2 ON true +WHERE name=ANY('{a,aa}') AND ('5' > '2') ORDER BY name ASC, cte1_v LIMIT 10 +-- Hints:enable_seqscan=off +---- +WITH + cte1 AS (SELECT ?+'?' AS cte1_v FROM generate_series(?,?)), + cte2 AS (SELECT ?+'?' AS cte2_v) +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" +JOIN cte1 ON true +JOIN (SELECT ?+'?' AS join2_v) join2 ON true +WHERE name=ANY('?') AND ('?' > '?') ORDER BY name ASC, cte1_v LIMIT ? +-- Hints:enable_seqscan=off + +(WITH + cte1 AS (SELECT 1+'1' AS cte1_v FROM generate_series(1,1)), + cte2 AS (SELECT 10+'10' AS cte2_v) +SELECT '' AS _key, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" +JOIN cte1 ON true +JOIN (SELECT 10+'10' AS join2_v) join2 ON true +WHERE name=ANY('{a,aa}') AND ('5' > '2') ORDER BY name ASC, cte1_v LIMIT 10) + UNION ALL +(SELECT '' AS _key, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" WHERE name='b' LIMIT 10) +-- Hints:enable_seqscan=off +---- +(WITH + cte1 AS (SELECT ?+'?' AS cte1_v FROM generate_series(?,?)), + cte2 AS (SELECT ?+'?' AS cte2_v) +SELECT '?' AS _key, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" +JOIN cte1 ON true +JOIN (SELECT ?+'?' AS join2_v) join2 ON true +WHERE name=ANY('?') AND ('?' > '?') ORDER BY name ASC, cte1_v LIMIT ?) + UNION ALL +(SELECT '?' AS _key, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" WHERE name='?' LIMIT ?) +-- Hints:enable_seqscan=off + +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='c' + LIMIT 10 +---- +SELECT name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id, id FROM "pg-schema.generic""table" + WHERE name='?' + LIMIT ? +" +`; + +exports[`selectBy batched three columns 1`] = ` +" +SELECT id, type, id1, id2, created_at, updated_at FROM "pg-schema.generic""table_3col" + WHERE (type, id1) IN(VALUES + ((NULL::"pg-schema.generic""table_3col").type, (NULL::"pg-schema.generic""table_3col").id1), + ('a', '1'), + ('a', '2'), + ('b', '1')) +---- +SELECT id, type, id1, id2, created_at, updated_at FROM "pg-schema.generic""table_3col" + WHERE (type, id1) IN(VALUES + ((NULL::"pg-schema.generic""table_3col").type, (NULL::"pg-schema.generic""table_3col").id1), + ('?', '?'), ...) + +SELECT id, type, id1, id2, created_at, updated_at FROM "pg-schema.generic""table_3col" + WHERE type='b' +---- +SELECT id, type, id1, id2, created_at, updated_at FROM "pg-schema.generic""table_3col" + WHERE type='?' +" +`; + +exports[`selectBy single three columns 1`] = ` +" +SELECT id, type, id1, id2, created_at, updated_at FROM "pg-schema.generic""table_3col" + WHERE (type='a' + AND id1='1') +---- +SELECT id, type, id1, id2, created_at, updated_at FROM "pg-schema.generic""table_3col" + WHERE (type='?' + AND id1='?') +" +`; + +exports[`update batched 1`] = ` +" +WITH rows(id, name, some_flag, updated_at, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").id, (NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").updated_at, ''), + ('42', 'absent', true, now(), '42'), + ('', 'a''upd', true, now(), '')) + UPDATE "pg-schema.generic""table" SET name=rows.name, some_flag=rows.some_flag, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.generic""table".id=rows.id RETURNING rows._key +---- +WITH rows(id, name, some_flag, updated_at, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").id, (NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").some_flag, (NULL::"pg-schema.generic""table").updated_at, '?'), + ('?', '?', true, now(), '?'), ...) + UPDATE "pg-schema.generic""table" SET name=rows.name, some_flag=rows.some_flag, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.generic""table".id=rows.id RETURNING rows._key + +WITH rows(id, name, url_name, updated_at, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").id, (NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").updated_at, ''), + ('', 'b''upd', NULL, now(), ''), + ('', 'null''upd', 'n1', now(), '')) + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.generic""table".id=rows.id RETURNING rows._key +---- +WITH rows(id, name, url_name, updated_at, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").id, (NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, (NULL::"pg-schema.generic""table").updated_at, '?'), + ('?', '?', '?', now(), '?'), + ('?', '?', NULL, now(), '?')) + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=rows.updated_at + FROM rows WHERE "pg-schema.generic""table".id=rows.id RETURNING rows._key +" +`; + +exports[`update date 1`] = ` +" +UPDATE "pg-schema.generic""table_date" SET name='bb' WHERE date_id='' RETURNING date_id AS id +---- +UPDATE "pg-schema.generic""table_date" SET name='?' WHERE date_id='?' RETURNING date_id AS id +" +`; + +exports[`update literal 1`] = ` +" +UPDATE "pg-schema.generic""table" SET updated_at=now(), name = name || 42 WHERE id='' RETURNING id +---- +UPDATE "pg-schema.generic""table" SET updated_at=now(), name = name || ? WHERE id='?' RETURNING id + +UPDATE "pg-schema.generic""table" SET updated_at=now(), name = name || 42 WHERE id='' RETURNING id +---- +UPDATE "pg-schema.generic""table" SET updated_at=now(), name = name || ? WHERE id='?' RETURNING id + +UPDATE "pg-schema.generic""table_date" SET name = name || 42 WHERE date_id='42' RETURNING date_id AS id +---- +UPDATE "pg-schema.generic""table_date" SET name = name || ? WHERE date_id='?' RETURNING date_id AS id + +UPDATE "pg-schema.generic""table" SET name='bbb', updated_at=now() WHERE id='' RETURNING id +---- +UPDATE "pg-schema.generic""table" SET name='?', updated_at=now() WHERE id='?' RETURNING id +" +`; + +exports[`update single 1`] = ` +" +UPDATE "pg-schema.generic""table" SET name='a''upd', updated_at=now() WHERE id='' RETURNING id +---- +UPDATE "pg-schema.generic""table" SET name='?', updated_at=now() WHERE id='?' RETURNING id + +UPDATE "pg-schema.generic""table" SET name='b''upd', url_name=NULL, some_flag=true, updated_at=now() WHERE id='' RETURNING id +---- +UPDATE "pg-schema.generic""table" SET name='?', url_name=NULL, some_flag=true, updated_at=now() WHERE id='?' RETURNING id + +UPDATE "pg-schema.generic""table" SET url_name='n', updated_at=now() WHERE id='' RETURNING id +---- +UPDATE "pg-schema.generic""table" SET url_name='?', updated_at=now() WHERE id='?' RETURNING id + +UPDATE "pg-schema.generic""table" SET name='absent', updated_at=now() WHERE id='42' RETURNING id +---- +UPDATE "pg-schema.generic""table" SET name='?', updated_at=now() WHERE id='?' RETURNING id +" +`; + +exports[`update skips if no known fields present 1`] = ` +" +UPDATE "pg-schema.generic""table" SET name='absent', updated_at=now() WHERE id='42' RETURNING id +---- +UPDATE "pg-schema.generic""table" SET name='?', updated_at=now() WHERE id='?' RETURNING id +" +`; + +exports[`upsert batched does not mess up the rows order 1`] = ` +" +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, ''), + ('zzz', 'zzz', '["zzz"]'), + ('bbb', 'bbb', '["bbb"]'), + ('ccc', 'ccc', '["ccc"]')), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, '?'), + ('?', '?', '?'), ...), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert batched normal 1`] = ` +" +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, ''), + ('a''b''c', 'aaa_new', '["a''b''c"]'), + ('bbb', 'bbb_new', '["bbb"]'), + ('zzz', 'zzz_dup', '["zzz"]')), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, '?'), + ('?', '?', '?'), ...), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert batched with nullable unique key must treat nulls as non-equal 1`] = ` +" +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_nullable_unique_key").name, (NULL::"pg-schema.generic""table_nullable_unique_key").url_name, ''), + ('000', '0', '')), + updates AS ( + UPDATE "pg-schema.generic""table_nullable_unique_key" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table_nullable_unique_key".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_nullable_unique_key".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_nullable_unique_key" (id, name, url_name, created_at, updated_at) + SELECT id_gen(), name, url_name, now(), now() + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_nullable_unique_key").name, (NULL::"pg-schema.generic""table_nullable_unique_key").url_name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_nullable_unique_key" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table_nullable_unique_key".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_nullable_unique_key".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_nullable_unique_key" (id, name, url_name, created_at, updated_at) + SELECT id_gen(), name, url_name, now(), now() + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts + +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_nullable_unique_key").name, (NULL::"pg-schema.generic""table_nullable_unique_key").url_name, ''), + ('a''b''c', NULL, '[{"guaranteed_unique_value":"k3"}]'), + ('bbb', NULL, '[{"guaranteed_unique_value":"k6"}]'), + ('ccc', NULL, '[{"guaranteed_unique_value":"k9"}]')), + updates AS ( + UPDATE "pg-schema.generic""table_nullable_unique_key" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table_nullable_unique_key".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_nullable_unique_key".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_nullable_unique_key" (id, name, url_name, created_at, updated_at) + SELECT id_gen(), name, url_name, now(), now() + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_nullable_unique_key").name, (NULL::"pg-schema.generic""table_nullable_unique_key").url_name, '?'), + ('?', NULL, '?'), ...), + updates AS ( + UPDATE "pg-schema.generic""table_nullable_unique_key" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table_nullable_unique_key".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_nullable_unique_key".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_nullable_unique_key" (id, name, url_name, created_at, updated_at) + SELECT id_gen(), name, url_name, now(), now() + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert does not override autoInsert field when it updates 1`] = ` +" +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").name, (NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, ''), + ('my', 'aaa', '')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET name=rows.name, url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), name, url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").name, (NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET name=rows.name, url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), name, url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts + +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, ''), + ('aaa', '')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), 'n', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, '?'), + ('?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), '?', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert does not override autoInsert field when it updates 2`] = ` +" +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").name, (NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, ''), + ('my', 'bbb', '')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET name=rows.name, url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), name, url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").name, (NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET name=rows.name, url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), name, url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts + +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, ''), + ('bbb', '')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), 'n', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, '?'), + ('?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), '?', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert does not override autoInsert field when it updates 3`] = ` +" +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, ''), + ('ccc', '')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), 'n', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, '?'), + ('?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), '?', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts + +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").name, (NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, ''), + ('my', 'ccc', '')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET name=rows.name, url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), name, url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_name_with_auto_insert").name, (NULL::"pg-schema.generic""table_name_with_auto_insert").url_name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_name_with_auto_insert" SET name=rows.name, url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_name_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_name_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_name_with_auto_insert" (id, name, url_name) + SELECT id_gen(), name, url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert single 1`] = ` +" +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, ''), + ('a''b''c', 'aaa', '')), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts + +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, ''), + ('a''b''c', 'aaa_dup', '')), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts + +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, ''), + ('zzz', 'n', '')), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table").name, (NULL::"pg-schema.generic""table").url_name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table" SET name=rows.name, url_name=rows.url_name, updated_at=now() + FROM rows WHERE "pg-schema.generic""table".name=rows.name + RETURNING rows._key, "pg-schema.generic""table".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table" (id, name, url_name, some_flag, json_text_field, json_strongly_typed_field, jsonb_field, encrypted_field, created_at, updated_at, parent_id) + SELECT id_gen(), name, url_name, false, NULL, NULL, NULL, NULL, now(), now(), NULL + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name, updated_at=now() + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert single 2`] = ` +" +WITH rows(id, name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_id_unique_key").id, (NULL::"pg-schema.generic""table_id_unique_key").name, ''), + ('', 'a', '')), + updates AS ( + UPDATE "pg-schema.generic""table_id_unique_key" SET id=rows.id, name=rows.name + FROM rows WHERE "pg-schema.generic""table_id_unique_key".id=rows.id + RETURNING rows._key, "pg-schema.generic""table_id_unique_key".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_id_unique_key" (id, name) + SELECT id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id, name=EXCLUDED.name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(id, name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_id_unique_key").id, (NULL::"pg-schema.generic""table_id_unique_key").name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_id_unique_key" SET id=rows.id, name=rows.name + FROM rows WHERE "pg-schema.generic""table_id_unique_key".id=rows.id + RETURNING rows._key, "pg-schema.generic""table_id_unique_key".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_id_unique_key" (id, name) + SELECT id, name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id, name=EXCLUDED.name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert with autoInsert name creates it 1`] = ` +" +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_unique_key_with_auto_insert").url_name, ''), + ('n', '')), + updates AS ( + UPDATE "pg-schema.generic""table_unique_key_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_unique_key_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_unique_key_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_unique_key_with_auto_insert" (id, name, url_name) + SELECT id_gen(), 'myname', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_unique_key_with_auto_insert").url_name, '?'), + ('?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_unique_key_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_unique_key_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_unique_key_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_unique_key_with_auto_insert" (id, name, url_name) + SELECT id_gen(), '?', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; + +exports[`upsert with autoInsert name does not override on update when name is omitted 1`] = ` +" +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_unique_key_with_auto_insert").name, (NULL::"pg-schema.generic""table_unique_key_with_auto_insert").url_name, ''), + ('000', 'n', '')), + updates AS ( + UPDATE "pg-schema.generic""table_unique_key_with_auto_insert" SET name=rows.name, url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_unique_key_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_unique_key_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_unique_key_with_auto_insert" (id, name, url_name) + SELECT id_gen(), name, url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(name, url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_unique_key_with_auto_insert").name, (NULL::"pg-schema.generic""table_unique_key_with_auto_insert").url_name, '?'), + ('?', '?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_unique_key_with_auto_insert" SET name=rows.name, url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_unique_key_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_unique_key_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_unique_key_with_auto_insert" (id, name, url_name) + SELECT id_gen(), name, url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET name=EXCLUDED.name, url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts + +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_unique_key_with_auto_insert").url_name, ''), + ('n', '')), + updates AS ( + UPDATE "pg-schema.generic""table_unique_key_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_unique_key_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_unique_key_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_unique_key_with_auto_insert" (id, name, url_name) + SELECT id_gen(), 'myname', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET 1 + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +---- +WITH rows(url_name, _key) AS (VALUES + ((NULL::"pg-schema.generic""table_unique_key_with_auto_insert").url_name, '?'), + ('?', '?')), + updates AS ( + UPDATE "pg-schema.generic""table_unique_key_with_auto_insert" SET url_name=rows.url_name + FROM rows WHERE "pg-schema.generic""table_unique_key_with_auto_insert".url_name=rows.url_name + RETURNING rows._key, "pg-schema.generic""table_unique_key_with_auto_insert".id AS id), + inserts AS ( + INSERT INTO "pg-schema.generic""table_unique_key_with_auto_insert" (id, name, url_name) + SELECT id_gen(), '?', url_name + FROM rows WHERE _key NOT IN (SELECT _key FROM updates) OFFSET ? + ON CONFLICT (url_name) DO UPDATE SET url_name=EXCLUDED.url_name + RETURNING NULL AS _key, id) + SELECT _key, id FROM updates UNION ALL SELECT _key, id FROM inserts +" +`; diff --git a/src/pg/__tests__/test-utils.ts b/src/pg/__tests__/test-utils.ts new file mode 100644 index 0000000..9c412c9 --- /dev/null +++ b/src/pg/__tests__/test-utils.ts @@ -0,0 +1,428 @@ +import type { AddressInfo, Server, Socket } from "net"; +import { connect, createServer } from "net"; +import delay from "delay"; +import compact from "lodash/compact"; +import { Pool, type PoolConfig } from "pg"; +import waitForExpect from "wait-for-expect"; +import type { ClientRole } from "../../abstract/Client"; +import { Cluster } from "../../abstract/Cluster"; +import type { Loggers } from "../../abstract/Loggers"; +import type { Query } from "../../abstract/Query"; +import type { STALE_REPLICA, Shard } from "../../abstract/Shard"; +import { MASTER } from "../../abstract/Shard"; +import { Timeline } from "../../abstract/Timeline"; +import { GLOBAL_SHARD } from "../../ent/ShardAffinity"; +import { join, mapJoin, nullthrows, runInVoid } from "../../internal/misc"; +import type { Literal } from "../../types"; +import { buildShape } from "../helpers/buildShape"; +import { escapeIdent } from "../helpers/escapeIdent"; +import { escapeLiteral } from "../helpers/escapeLiteral"; +import { PgClient } from "../PgClient"; +import { PgShardNamer } from "../PgShardNamer"; + +/** + * A proxy for an PgClient which records all the queries passing through and has + * some other helper methods. + */ +export class TestPgClient extends PgClient { + readonly queries: string[] = []; + + override async query( + params: Parameters[0], + ): Promise { + if (params.op !== "SHARD_NOS") { + this.queries.push( + escapeLiteral(params.query) + + (params.hints && Object.keys(params.hints).length > 0 + ? "\n-- Hints:" + + Object.entries(params.hints) + .map(([k, v]) => `${k || '""'}=${v}`) + .join(",") + : ""), + ); + } + + return super.query(params); + } + + toMatchSnapshot(): void { + expect( + this.queries + .map( + (query) => + "\n" + + indentQuery(query) + + "\n----\n" + + indentQuery(buildShape(query)) + + "\n", + ) + .join(""), + ).toMatchSnapshot(); + + this.resetSnapshot(); + } + + resetSnapshot(): void { + this.queries.length = 0; + } + + async rows( + sql: string, + ...values: Literal + ): Promise>> { + sql = sql.replace(/%T/g, (_) => escapeIdent("" + values.shift())); + return nullthrows( + await super.query({ + query: [sql, ...values], + isWrite: true, // because used for BOTH read and write queries in tests + annotations: [], + op: "", + table: "", + }), + ); + } +} + +/** + * A custom type example. + */ +export class EncryptedValue { + static dbValueToJs(dbValue: string): EncryptedValue { + return new this(dbValue); + } + + static stringify(obj: EncryptedValue): string { + return obj.dbValue; + } + + static parse(str: string): EncryptedValue { + return new this(str); + } + + static async encrypt(text: string, delta: number): Promise { + return new this( + "encrypted:" + + text + .split("") + .map((c) => String.fromCharCode(c.charCodeAt(0) + delta)) + .join(""), + ); + } + + async decrypt(delta: number): Promise { + return this.dbValue + .replace("encrypted:", "") + .split("") + .map((c) => String.fromCharCode(c.charCodeAt(0) - delta)) + .join(""); + } + + private constructor(private dbValue: string) {} +} + +/** + * A simple PGBouncer simulation to test connection issues. + */ +export class TCPProxyServer { + private connections = new Set(); + private server: Server; + private usedPort?: number; + + constructor({ + host, + port, + delayOnConnect, + }: { + host: string; + port: number; + delayOnConnect?: number; + }) { + this.server = createServer((socket) => { + this.connections.add(socket); + socket.once("close", () => this.connections.delete(socket)); + runInVoid( + delay(delayOnConnect ?? 0).then(() => + socket.pipe(connect(port, host)).pipe(socket), + ), + ); + }); + } + + async abortConnections(): Promise { + const destroyed = [...this.connections]; + if (destroyed.length > 0) { + // Some new connection may appear while we're waiting for all the existing + // connections to be destroyed - this is fine. We just need to make sure + // that we really closed all of the EXISTING connections. + this.connections.forEach((socket) => socket.destroy()); + await waitForExpect(() => + expect(destroyed.some((s) => this.connections.has(s))).toBeFalsy(), + ); + } + + return destroyed.length; + } + + async destroy(): Promise { + this.server.close(); + await this.abortConnections(); + } + + async resurrectOnSamePort(): Promise { + await this.listen(this.usedPort); + } + + async waitForAtLeastConnections(n: number): Promise { + await waitForExpect(() => + expect(this.connections.size).toBeGreaterThanOrEqual(n), + ); + } + + connectionCount(): number { + return this.connections.size; + } + + async hostPort(): Promise<{ host: string; port: number }> { + if (!this.usedPort) { + await this.listen(); + } + + return { + host: (this.server.address() as AddressInfo).address, + port: this.usedPort!, + }; + } + + private async listen(port?: number): Promise { + return new Promise((resolve) => + this.server.listen(port, "127.0.0.1", () => { + this.usedPort = (this.server.address() as AddressInfo).port; + resolve(); + }), + ); + } +} + +/** + * A node-postgres config we use in tests. + */ +export const TEST_CONFIG: PoolConfig & { + nameSuffix: string | undefined; + isAlwaysLaggingReplica: boolean; + loggers: Loggers; + role?: ClientRole; +} = { + host: process.env["PGHOST"] || process.env["DB_HOST_DEFAULT"], + port: + parseInt(process.env["PGPORT"] || process.env["DB_PORT"] || "0") || + undefined, + database: process.env["PGDATABASE"] || process.env["DB_DATABASE"], + user: process.env["PGUSER"] || process.env["DB_USER"], + password: process.env["PGPASSWORD"] || process.env["DB_PASS"], + idleTimeoutMillis: 30000, + // Additional custom props (tests facilities). + nameSuffix: undefined, + isAlwaysLaggingReplica: false, + loggers: { + swallowedErrorLogger: jest.fn(), + clientQueryLogger: jest.fn(), + runOnShardErrorLogger: jest.fn(), + }, +}; + +/** + * An initial test list of Islands to start from. + */ +export const TEST_ISLANDS = [ + { + no: 0, + nodes: [TEST_CONFIG, { ...TEST_CONFIG, isAlwaysLaggingReplica: true }], + }, +]; + +/** + * A stub value for QueryAnnotation. + */ +export const TEST_ANNOTATION = { + trace: "some-trace", + debugStack: "", + vc: "some-vc", + whyClient: undefined, + attempt: 0, +}; + +/** + * A stub Timeline used in shardRun() helper. + */ +export const TEST_TIMELINE = new Timeline(); + +beforeEach(() => { + TEST_TIMELINE.reset(); +}); + +/** + * A derived class from the default node-postgres Pool. + */ +export class TestPool extends Pool { + some(): string { + return "some"; + } +} + +/** + * Test Cluster backed by the test config. + */ +export const testCluster = new Cluster({ + islands: TEST_ISLANDS, + createClient: ({ + nameSuffix, + isAlwaysLaggingReplica, + loggers, + role, + ...config + }) => + new TestPgClient({ + name: + `test-pool(replica=${isAlwaysLaggingReplica})` + + (nameSuffix ? `-${nameSuffix}` : ""), + loggers, + ...(isAlwaysLaggingReplica + ? { role: "replica", maxReplicationLagMs: 1e10 } + : role + ? { role, maxReplicationLagMs: 10000 } + : {}), + config, + createPool: (config) => new TestPool(config), + }), + loggers: { + swallowedErrorLogger: jest.fn(), + clientQueryLogger: jest.fn(), + runOnShardErrorLogger: jest.fn(), + clientEndLogger: jest.fn(), + clientConnectedLogger: jest.fn(), + }, + shardNamer: new PgShardNamer({ + nameFormat: "sh%04d", + discoverQuery: + "SELECT nspname FROM pg_namespace WHERE nspname ~ 'sh[0-9]+'", + }), + shardsDiscoverIntervalMs: 500, + shardsDiscoverIntervalJitter: 0.01, +}); + +/** + * Recreates the test tables on the test Cluster. + */ +export async function recreateTestTables( + EntClasses: Array<{ + CREATE: string[]; + SCHEMA: { name: string }; + SHARD_AFFINITY: typeof GLOBAL_SHARD | readonly string[]; + }>, + tableInverse?: string, +): Promise { + await mapJoin( + [testCluster.globalShard(), ...(await testCluster.nonGlobalShards())], + async (shard) => { + const master = await shard.client(MASTER); + await mapJoin( + compact([ + tableInverse, + ...EntClasses.map((EntClass) => EntClass.SCHEMA.name), + ]), + async (table) => master.rows("DROP TABLE IF EXISTS %T CASCADE", table), + ); + await join([ + tableInverse && + master.rows( + `CREATE TABLE %T( + id bigint NOT NULL PRIMARY KEY, + id1 bigint, + type varchar(32) NOT NULL, + id2 bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE(type, id1, id2) + )`, + tableInverse, + ), + mapJoin(EntClasses, async (EntClass) => { + if ((EntClass.SHARD_AFFINITY === GLOBAL_SHARD) === (shard.no === 0)) { + for (const sql of EntClass.CREATE) { + await master.rows( + sql, + EntClass.SCHEMA.name, + EntClass.SCHEMA.name, + EntClass.SCHEMA.name, + ); + } + } + }), + ]); + }, + ); +} + +/** + * A shortcut helper to run a query against a Shard. + */ +export async function shardRun( + shard: Shard, + query: Query, + freshness: typeof STALE_REPLICA | null = null, +): Promise { + return shard.run(query, TEST_ANNOTATION, TEST_TIMELINE, freshness); +} + +/** + * Reconfigures the Cluster to have 2 Islands, where both Island 0 and Island 1 + * has 1 master node each. + */ +export async function reconfigureToTwoIslands(): Promise { + // Since we add the same physical host to island 1 as we already have in + // island 0, we force the old Client to discover 0 shards to avoid "Shard + // exists on more than one island" error. + const oldIsland0 = await testCluster.island(0); + const oldMaster0 = oldIsland0.master(); // will be reused + jest.spyOn(oldMaster0, "shardNos").mockResolvedValue([]); + + testCluster.options.islands = () => [ + { no: 0, nodes: [TEST_CONFIG] }, + { no: 1, nodes: [{ ...TEST_CONFIG, nameSuffix: "island1" }] }, + ]; + await testCluster.rediscover(); +} + +function indentQuery(query: string): string { + query = query + .replace(/\d{4}-\d{2}-\d{2}T[^']+/g, "") + .replace(/(?<=[':])[a-f0-9]{40}'/g, "'") + .replace(/'k\d+'/g, "''") + .replace(/\d{16,}/g, (m) => + m === Number.MAX_SAFE_INTEGER.toString() ? m : "", + ) + .replace(/ id AS id/g, " id") + .replace(/( AS k)\d+/g, "$1*"); + + // Beautify single-lined SQL queries. + if (query.match(/^\(?SELECT/)) { + query = query + .replace(/\s+(WHERE|ORDER|LIMIT)/g, "\n $1") + .replace(/\s+(AND|OR)/g, "\n $1"); + } + + // Make the order of rows in UPDATE clause static (to avoid flaky tests); the + // framework orders the rows by ID typically. + const match = query.match(/^(WITH.*\(VALUES\n.*?\n)(.*?)(\)\n\s*UPDATE.*)/s); + if (match) { + query = + match[1] + + match[2] + .split(",\n") + .sort() + .map((s) => `${s}`) + .join(",\n") + + match[3]; + } + + return query; +} diff --git a/src/pg/helpers/buildShape.ts b/src/pg/helpers/buildShape.ts new file mode 100644 index 0000000..db5326d --- /dev/null +++ b/src/pg/helpers/buildShape.ts @@ -0,0 +1,66 @@ +const RE_COMMENT = toRegExp` + /\*(?!\+).*?\*/ +`; + +const RE_NUMBER = toRegExp` + (? + flag === "i" ? escapeID("" + args.shift()) : escapeAny(args.shift()), + ); +} diff --git a/src/pg/index.ts b/src/pg/index.ts new file mode 100644 index 0000000..095036c --- /dev/null +++ b/src/pg/index.ts @@ -0,0 +1,28 @@ +/** + * @file Automatically generated by barrelsby. + */ + +export * from "./PgClient"; +export * from "./PgError"; +export * from "./PgQueryCount"; +export * from "./PgQueryDelete"; +export * from "./PgQueryDeleteWhere"; +export * from "./PgQueryExists"; +export * from "./PgQueryIDGen"; +export * from "./PgQueryInsert"; +export * from "./PgQueryLoad"; +export * from "./PgQueryLoadBy"; +export * from "./PgQuerySelect"; +export * from "./PgQuerySelectBy"; +export * from "./PgQueryUpdate"; +export * from "./PgQueryUpsert"; +export * from "./PgRunner"; +export * from "./PgSchema"; +export * from "./PgShardNamer"; +export * from "./PgTimelineStorage"; +export * from "./helpers/buildShape"; +export * from "./helpers/escapeIdent"; +export * from "./helpers/escapeLiteral"; +export * from "./types/BigIntArrayType"; +export * from "./types/ByteaBufferType"; +export * from "./types/StringArrayType"; diff --git a/src/pg/internal/__tests__/buildHintQueries.test.ts b/src/pg/internal/__tests__/buildHintQueries.test.ts new file mode 100644 index 0000000..c911c83 --- /dev/null +++ b/src/pg/internal/__tests__/buildHintQueries.test.ts @@ -0,0 +1,113 @@ +import { RAW_PREPEND_HINT, buildHintQueries } from "../buildHintQueries"; + +test("no hints", () => { + expect(buildHintQueries({}, {})).toEqual(["", [], []]); + expect(buildHintQueries(undefined, undefined)).toEqual(["", [], []]); +}); + +test("no user hints", () => { + expect( + buildHintQueries( + { statement_timeout: "10000", transaction: "read only" }, + {}, + ), + ).toEqual([ + "", + ["SET LOCAL statement_timeout TO 10000", "SET LOCAL transaction read only"], + [], + ]); +}); + +test("user overrides a hint", () => { + expect( + buildHintQueries( + { statement_timeout: "10000", transaction: "read only" }, + { statement_timeout: "42" }, + ), + ).toEqual([ + "", + ["SET LOCAL transaction read only"], + ["SET LOCAL statement_timeout TO 42"], + ]); +}); + +test("user resets a hint", () => { + expect( + buildHintQueries( + { statement_timeout: "10000", transaction: "read only" }, + { transaction: null }, + ), + ).toEqual(["", ["SET LOCAL statement_timeout TO 10000"], []]); +}); + +test('user passes undefined aka "no key mentioned"', () => { + expect( + buildHintQueries( + { statement_timeout: "10000", transaction: "read only" }, + { transaction: undefined }, + ), + ).toEqual([ + "", + ["SET LOCAL statement_timeout TO 10000", "SET LOCAL transaction read only"], + [], + ]); +}); + +test("raw prepend hint", () => { + expect( + buildHintQueries(undefined, { + statement_timeout: "42", + [RAW_PREPEND_HINT]: "/*+IndexScan(my_table)*/", + }), + ).toEqual([ + "/*+IndexScan(my_table)*/ ", + [], + ["SET LOCAL statement_timeout TO 42"], + ]); +}); + +test('default passes undefined aka "no key mentioned"', () => { + expect(buildHintQueries({ statement_timeout: undefined }, {})).toEqual([ + "", + [], + [], + ]); + expect( + buildHintQueries( + { statement_timeout: undefined }, + { statement_timeout: null }, + ), + ).toEqual(["", [], []]); + expect( + buildHintQueries( + { statement_timeout: undefined }, + { statement_timeout: undefined }, + ), + ).toEqual(["", [], []]); + expect( + buildHintQueries( + { statement_timeout: undefined }, + { statement_timeout: "42" }, + ), + ).toEqual(["", [], ["SET LOCAL statement_timeout TO 42"]]); +}); + +test("default resets a hint", () => { + expect(buildHintQueries({ statement_timeout: null }, {})).toEqual([ + "", + [], + [], + ]); + expect( + buildHintQueries({ statement_timeout: null }, { statement_timeout: null }), + ).toEqual(["", [], []]); + expect( + buildHintQueries( + { statement_timeout: null }, + { statement_timeout: undefined }, + ), + ).toEqual(["", [], []]); + expect( + buildHintQueries({ statement_timeout: null }, { statement_timeout: "42" }), + ).toEqual(["", [], ["SET LOCAL statement_timeout TO 42"]]); +}); diff --git a/src/pg/internal/__tests__/parseCompositeRow.test.ts b/src/pg/internal/__tests__/parseCompositeRow.test.ts new file mode 100644 index 0000000..2d7f3e8 --- /dev/null +++ b/src/pg/internal/__tests__/parseCompositeRow.test.ts @@ -0,0 +1,27 @@ +import { parseCompositeRow } from "../parseCompositeRow"; + +test("parseCompositeRow", async () => { + expect(parseCompositeRow("()")).toEqual([null]); + expect(parseCompositeRow("(,)")).toEqual([null, null]); + expect(parseCompositeRow('("1","2")')).toEqual(["1", "2"]); + expect(parseCompositeRow('(,"aaa")')).toEqual([null, "aaa"]); + expect(parseCompositeRow('("1","2","(""d""""d"",""5"")")')).toEqual([ + "1", + "2", + '("d""d","5")', + ]); + expect(parseCompositeRow('("1","2","{""x"",""y""}")')).toEqual([ + "1", + "2", + '{"x","y"}', + ]); + expect(parseCompositeRow('("aaa",)')).toEqual(["aaa", null]); + expect(parseCompositeRow(String.raw`("a\\b")`)).toEqual([String.raw`a\b`]); + expect(parseCompositeRow("(aa)")).toEqual(["aa"]); + expect(parseCompositeRow('("aaaaa aa","2")')).toEqual(["aaaaa aa", "2"]); + expect(parseCompositeRow("(12345,678)")).toEqual(["12345", "678"]); + + await expect(async () => parseCompositeRow("(")).rejects.toThrow(); + await expect(async () => parseCompositeRow('("aa)')).rejects.toThrow(); + await expect(async () => parseCompositeRow("(aa")).rejects.toThrow(); +}); diff --git a/src/pg/internal/buildHintQueries.ts b/src/pg/internal/buildHintQueries.ts new file mode 100644 index 0000000..7ed6c72 --- /dev/null +++ b/src/pg/internal/buildHintQueries.ts @@ -0,0 +1,62 @@ +import type { Hints } from "../../types"; + +export const RAW_PREPEND_HINT = ""; + +/** + * Builds query prologue queries for the given hints. + * + * In the resulting compound queries, the returned `queries` will become a part + * of the debug query text, and queriesDefault will be prepended and omitted + * from the debug query text. + * + * Also, if there is a special hint with key = "", the its value is appended as + * it is to the very beginning of the compound query sent. You can e.g. pass + * pg_hint_plan extension hints there. + */ +export function buildHintQueries( + hintsDefault: Readonly = {}, + hints: Readonly = {}, +): [rawPrepend: string, queriesDefault: string[], queries: string[]] { + const queriesDefault: string[] = []; + const queries: string[] = []; + + let rawPrepend = hints[RAW_PREPEND_HINT] ?? ""; + if (rawPrepend) { + rawPrepend += " "; + } + + for (const k in hintsDefault) { + const v = hintsDefault[k]; + if (k === RAW_PREPEND_HINT) { + continue; + } else if (v === null || v === undefined) { + // Engine default or non-set. + continue; + } else if (hints[k] !== undefined) { + // User sets this hint to some different value (or resets it). + continue; + } else { + queriesDefault.push(buildHintQuery(k, v)); + } + } + + for (const k in hints) { + const v = hints[k]; + if (k === RAW_PREPEND_HINT) { + continue; + } else if (v === null || v === undefined) { + // Engine default or non-set. + continue; + } else { + queries.push(buildHintQuery(k, v)); + } + } + + return [rawPrepend, queriesDefault, queries]; +} + +function buildHintQuery(k: string, v: string): string { + return k.toLowerCase() === "transaction" + ? `SET LOCAL ${k} ${v}` + : `SET LOCAL ${k} TO ${v}`; +} diff --git a/src/pg/internal/escapeAny.ts b/src/pg/internal/escapeAny.ts new file mode 100644 index 0000000..cdb7b7a --- /dev/null +++ b/src/pg/internal/escapeAny.ts @@ -0,0 +1,25 @@ +import { escapeArray } from "./escapeArray"; +import { escapeBoolean } from "./escapeBoolean"; +import { escapeDate } from "./escapeDate"; +import { escapeString } from "./escapeString"; + +/** + * Tries its best to escape the value according to its type. + * + * Try to not use this function; although it protects against SQL injections, + * it's not aware of the actual field type, so it e.g. cannot prevent a bigint + * overflow SQL error. + */ +export function escapeAny(v: unknown): string { + return v === null || v === undefined + ? "NULL" + : typeof v === "number" + ? v.toString() + : typeof v === "boolean" + ? escapeBoolean(v) + : v instanceof Date + ? escapeDate(v) + : v instanceof Array + ? escapeArray(v) + : escapeString(v as string | null | undefined); +} diff --git a/src/pg/internal/escapeArray.ts b/src/pg/internal/escapeArray.ts new file mode 100644 index 0000000..30d265d --- /dev/null +++ b/src/pg/internal/escapeArray.ts @@ -0,0 +1,22 @@ +import { escapeString } from "./escapeString"; + +/** + * Escapes an array of strings. + */ +export function escapeArray( + obj: Array | null | undefined, +): string { + return obj === null || obj === undefined + ? "NULL" + : escapeString( + "{" + + obj + .map((v) => + v === null + ? "NULL" + : `"${v.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`, + ) + .join(",") + + "}", + ); +} diff --git a/src/pg/internal/escapeBoolean.ts b/src/pg/internal/escapeBoolean.ts new file mode 100644 index 0000000..4903216 --- /dev/null +++ b/src/pg/internal/escapeBoolean.ts @@ -0,0 +1,6 @@ +/** + * Escapes a boolean as PG string literal. + */ +export function escapeBoolean(v: boolean | null | undefined): string { + return v === null || v === undefined ? "NULL" : v ? "true" : "false"; +} diff --git a/src/pg/internal/escapeComposite.ts b/src/pg/internal/escapeComposite.ts new file mode 100644 index 0000000..f408ee4 --- /dev/null +++ b/src/pg/internal/escapeComposite.ts @@ -0,0 +1,27 @@ +import { escapeString } from "./escapeString"; +import { parseCompositeRow } from "./parseCompositeRow"; + +/** + * PostgreSQL doesn't allow comparison like `WHERE (a, b) = '(1,2)'` - it throws + * "Input of anonymous composite types is not implemented" error. So to compare, + * we have to convert the stringified row representation to ROW() notation + * manually: `WHERE (a, b) = ROW('1', '2')` + * + * Notice that we don't work with ROWs consisting of 1 element; instead, we + * treat them as the element itself. I.e. instead of emitting "(123)" or + * "ROW(123)", we always emit just "123". + * + * - "1" => "1" + * - "(1)" => "1" + * - "(1,2)" => "ROW('1','2')" + */ +export function escapeComposite(v: string | null | undefined): string { + if (v === null || v === undefined) { + return "NULL"; + } + + const parts = + v.startsWith("(") && v.endsWith(")") ? parseCompositeRow(v) : [v]; + const list = parts.map((v) => escapeString(v)).join(","); + return parts.length > 1 ? `ROW(${list})` : list; +} diff --git a/src/pg/internal/escapeDate.ts b/src/pg/internal/escapeDate.ts new file mode 100644 index 0000000..5c28bce --- /dev/null +++ b/src/pg/internal/escapeDate.ts @@ -0,0 +1,10 @@ +/** + * Escapes a date as PG string literal. + */ +export function escapeDate(v: Date | null | undefined, field?: string): string { + try { + return v === null || v === undefined ? "NULL" : "'" + v.toISOString() + "'"; + } catch (e: unknown) { + throw Error(`Failed to perform escapeDate for "${field}": ${e}`); + } +} diff --git a/src/pg/internal/escapeID.ts b/src/pg/internal/escapeID.ts new file mode 100644 index 0000000..f9db277 --- /dev/null +++ b/src/pg/internal/escapeID.ts @@ -0,0 +1,19 @@ +import { isBigintStr } from "../../helpers/isBigintStr"; +import { escapeString } from "./escapeString"; + +/** + * Escapes a value implying that it's a PG ID (which is a bigint). This should + * be a preferred way of escaping when we know that the value is a bigint. + */ +export function escapeID(v: string | null | undefined): string { + if (v === null || v === undefined) { + return "NULL"; + } + + const str = "" + v; + if (!isBigintStr(str)) { + return "'-1'/*bad_bigint*/"; + } + + return escapeString(str); +} diff --git a/src/pg/internal/escapeIdentComposite.ts b/src/pg/internal/escapeIdentComposite.ts new file mode 100644 index 0000000..1e92b3d --- /dev/null +++ b/src/pg/internal/escapeIdentComposite.ts @@ -0,0 +1,18 @@ +import { escapeIdent } from "../helpers/escapeIdent"; + +/** + * A pair for escapeComposite(), but works with a list of identifiers (e.g. list + * of unique key fields), not with values. + * + * - fields=["some_id"], table="tbl" => "tbl.some_id" + * - fields=["f1", "f2"], table="tbl" => "(tbl.f1,tbl.f2)" + */ +export function escapeIdentComposite( + fields: readonly string[], + table?: string, +): string { + const list = fields + .map((k) => (table ? `${escapeIdent(table)}.` : "") + escapeIdent(k)) + .join(","); + return fields.length > 1 ? `ROW(${list})` : list; +} diff --git a/src/pg/internal/escapeString.ts b/src/pg/internal/escapeString.ts new file mode 100644 index 0000000..778a9c7 --- /dev/null +++ b/src/pg/internal/escapeString.ts @@ -0,0 +1,10 @@ +/** + * Escapes a string as PG string literal. + */ +export function escapeString(v: string | null | undefined): string { + return v === null || v === undefined + ? "NULL" + : // Postgres doesn't like ASCII NUL character (error message is "unterminated + // quoted string" or "invalid message format"), so we remove it too. + "'" + ("" + v).replace(/\0/g, "").replace(/'/g, "''") + "'"; +} diff --git a/src/pg/internal/escapeStringify.ts b/src/pg/internal/escapeStringify.ts new file mode 100644 index 0000000..d5cc00e --- /dev/null +++ b/src/pg/internal/escapeStringify.ts @@ -0,0 +1,12 @@ +import { escapeString } from "./escapeString"; + +/** + * A helper method which additionally calls to a stringify() function before + * escaping the value as string. + */ +export function escapeStringify( + v: unknown, + stringify: (v: unknown) => string, +): string { + return v === null || v === undefined ? "NULL" : escapeString(stringify(v)); +} diff --git a/src/pg/internal/misc.ts b/src/pg/internal/misc.ts new file mode 100644 index 0000000..38382b2 --- /dev/null +++ b/src/pg/internal/misc.ts @@ -0,0 +1,184 @@ +import type { + ClientErrorKind, + ClientErrorPostAction, +} from "../../abstract/ClientError"; + +/** + * Some errors affect the logic of choosing another Client when a query retry is + * requested (controlled via ClientError). Mostly, those are the situation when + * a PG node goes down. + * + * Notice that, if a master goes down, we always need to find another master + * (and retry hard hoping is a switchover, and it will appear on another node). + */ +export const CLIENT_ERROR_PREDICATES: Array< + (error: { code: string; message: string }) => + | false + | { + abbreviation: string; + postAction: { + ifMaster: ClientErrorPostAction; + ifReplica: ClientErrorPostAction; + reportConnectionIssue: boolean; + }; + kind: ClientErrorKind; + comment: string; + } +> = [ + ({ code }) => + code === "42P01" && { + abbreviation: "undefined_table", + postAction: { + ifMaster: "rediscover-cluster", + ifReplica: "rediscover-cluster", + reportConnectionIssue: false, + }, + kind: "data-on-server-is-unchanged", + comment: + "For single-queries: table doesn't exist or disappeared (e.g. the Shard was relocated to another Island).", + }, + ({ code }) => + code === "42704" && { + abbreviation: "undefined_object", + postAction: { + ifMaster: "rediscover-cluster", + ifReplica: "rediscover-cluster", + reportConnectionIssue: false, + }, + kind: "data-on-server-is-unchanged", + comment: + "For batched queries: table doesn't exist or disappeared (e.g. the Shard was relocated to another Island).", + }, + ({ code }) => + code === "25006" && { + abbreviation: "read_only_sql_transaction", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "rediscover-island", + reportConnectionIssue: false, + }, + kind: "data-on-server-is-unchanged", + comment: + "A write happened in a read-only Client (probably the Client's role was changed from master to replica due to a failover/switchover).", + }, + ({ code }) => + code === "57P01" && { + abbreviation: "admin_shutdown", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "data-on-server-is-unchanged", + comment: "The database is shutting down by an administrator.", + }, + ({ code }) => + code === "57P02" && { + abbreviation: "crash_shutdown", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "data-on-server-is-unchanged", + comment: "The database is crashed and is shutting down.", + }, + ({ message }) => + message === "Connection terminated unexpectedly" && { + abbreviation: "connection_terminated", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "unknown-server-state", // !!! + comment: + "Node-postgres connection terminated unexpectedly (from client.js).", + }, + ({ code }) => + code === "ECONNREFUSED" && { + abbreviation: "ECONNREFUSED", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "data-on-server-is-unchanged", + comment: "Node TCP library connect error.", + }, + ({ code }) => + code === "ECONNRESET" && { + abbreviation: "ECONNRESET", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "unknown-server-state", // !!! + comment: "Node TCP library connection was externally closed mid-way.", + }, + ({ code, message }) => + code === "08P01" && // protocol_violation + message.includes("server conn crashed") && { + abbreviation: "server_conn_crashed", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "unknown-server-state", // !!! + comment: "PG is terminated cruelly (e.g. by SIGKILL or SIGSEGV).", + }, + ({ code, message }) => + code === "08P01" && // protocol_violation + message === "query_wait_timeout" && { + abbreviation: "query_wait_timeout", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "data-on-server-is-unchanged", + comment: + "PG went down a few seconds ago, but a PgBouncer connection is still open, so the query sent to it times out waiting. After some time, PgBouncer will stop emitting this error and start emitting server_login_retry on new connections instead.", + }, + ({ code, message }) => + code === "08P01" && // protocol_violation + message.includes("server_login_retry") && { + abbreviation: "server_login_retry", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "data-on-server-is-unchanged", + comment: + "PG has been down for quite some, so PgBouncer emits this error on each new connection immediately. Or, PG server has just been shut down, and PgBouncer's server pool is empty (fast-fail use case).", + }, + ({ code, message }) => + code === "08P01" && // protocol_violation + message === "pgbouncer cannot connect to server" && { + abbreviation: "cannot_connect_to_server", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "data-on-server-is-unchanged", + comment: + "Older PgBouncer versions emit this error instead of server_login_retry.", + }, + ({ code }) => + code === "08P01" && { + // protocol_violation + abbreviation: "protocol_violation_catch_all", + postAction: { + ifMaster: "rediscover-island", + ifReplica: "choose-another-client", + reportConnectionIssue: true, + }, + kind: "data-on-server-is-unchanged", + comment: + 'Sometimes PgBouncer emit unrelated error codes when PG server crashes (e.g. "invalid server parameter").', + }, +]; diff --git a/src/pg/internal/parseCompositeRow.ts b/src/pg/internal/parseCompositeRow.ts new file mode 100644 index 0000000..dfb2579 --- /dev/null +++ b/src/pg/internal/parseCompositeRow.ts @@ -0,0 +1,93 @@ +/** + * Parses composite row input into parts. See references at: + * - https://www.postgresql.org/docs/current/rowtypes.html + * - https://github.com/DmitryKoterov/db_type/blob/master/lib/DB/Type/Pgsql/Row.php + * - unit tests of this function + */ +export function parseCompositeRow(str: string): Array { + let p = 0; + let c = ""; + + function readCharAfterSpaces(): string { + while (p < str.length) { + const c = str[p]; + if (c !== " " && c !== "\t" && c !== "\r" && c !== "\n") { + break; + } + } + + return p < str.length ? str[p] : ""; + } + + function throwError(message: string): void { + throw Error(`${message} at position ${p} of "${str}"`); + } + + // Leading "(". + c = readCharAfterSpaces(); + if (c !== "(") { + throwError('An anonymous composite type row must start with "("'); + } + + p++; + + // Check for immediate trailing ')': by convention, "()" translates to + // ROW(NULL) tuple and not to an empty tuple (because ROW() and ROW(NULL) have + // the same stringified representation which is "()" unfortunately). + c = readCharAfterSpaces(); + if (c === ")") { + return [null]; + } + + // Row may contain: + // - "-quoted strings (escaping: ["] is doubled) + // - unquoted strings (before first "," or ")") + // - empty string (it is treated as NULL) + // Nested rows and all other things are represented as strings. + + const reTillNext = /[^,)]*/sy; + const reQuoted = /"((?:[^"]+|"")*)"/sy; + + const result: Array = []; + while (true) { + // We read a value in this iteration, then - delimiter. + c = readCharAfterSpaces(); + + // Always read a next element value. + if (c === "," || c === ")") { + // Comma or end of row instead of value: treat as NULL. + result.push(null); + } else if (c !== '"') { + // Unquoted string. Notice that NULL here is treated as "NULL" string, but + // NOT as a null value! This is how composite values are encoded by PG. + reTillNext.lastIndex = p; + const matches = reTillNext.exec(str)!; + result.push(matches[0]); + p += matches[0].length; + } else { + reQuoted.lastIndex = p; + const matches = reQuoted.exec(str); + if (matches) { + // Quoted string. + result.push(matches[1].replace(/""/g, '"').replace(/\\\\/g, "\\")); + p += matches[0].length; + } else { + throwError("Expected a balanced quoted string"); + } + } + + // Delimiter or the end of row. + c = readCharAfterSpaces(); + if (c === ",") { + p++; + continue; + } else if (c === ")") { + p++; + break; + } else { + throwError('Expected a delimiter "," or ")"'); + } + } + + return result; +} diff --git a/src/pg/internal/parseLsn.ts b/src/pg/internal/parseLsn.ts new file mode 100644 index 0000000..af77e47 --- /dev/null +++ b/src/pg/internal/parseLsn.ts @@ -0,0 +1,7 @@ +/** + * Parses a WAL LSN number into a JS bigint. + */ +export function parseLsn(lsn: string): bigint { + const [a, b] = lsn.split("/").map((x) => BigInt(parseInt(x, 16))); + return (a << BigInt(32)) + b; +} diff --git a/src/pg/types/BigIntArrayType.ts b/src/pg/types/BigIntArrayType.ts new file mode 100644 index 0000000..c98bb3e --- /dev/null +++ b/src/pg/types/BigIntArrayType.ts @@ -0,0 +1,35 @@ +import { types } from "pg"; +import type { DesperateAny } from "../../internal/misc"; + +/** + * It is in pg-types/lib/textParsers.js (1016::regtype is "bigint[]"), just not + * exported to TS types. + */ +const BIGINT_ARRAY_OID: DesperateAny = 1016; + +/** + * An array of IDs. Notice that: + * 1. Node-postgres natively supports this type on read path, but on write path, + * we have to stringify by ourselves. + * 2. GIN index doesn't support NULL, because PG's "&&" operator (intersection + * check) doesn't work with NULLs. But we still allow NULLs in + * BigIntArrayType, since to query such values, the user could use a separate + * partial index. + */ +export function BigIntArrayType(): { + dbValueToJs: (dbValue: T[]) => T[]; + stringify: (jsValue: T[]) => string; + parse: (str: string) => T[]; +} { + return { + dbValueToJs: (dbValue) => dbValue, + + // PG's representation: '{123,NULL,NULL,223}'. Notice that the unquoted NULL + // must be of a capital case to denote a null value, otherwise it's treated + // as a 4-char string "null". This is a part of the protocol. + stringify: (jsValue) => + "{" + jsValue.map((v) => v ?? "NULL").join(",") + "}", + + parse: (str) => types.getTypeParser(BIGINT_ARRAY_OID)(str) as T[], + }; +} diff --git a/src/pg/types/ByteaBufferType.ts b/src/pg/types/ByteaBufferType.ts new file mode 100644 index 0000000..a7edf8a --- /dev/null +++ b/src/pg/types/ByteaBufferType.ts @@ -0,0 +1,20 @@ +import { types } from "pg"; + +/** + * A value stored in the DB as a bytea buffer. + */ +export function ByteaBufferType(): { + dbValueToJs: (dbValue: Buffer) => Buffer; + stringify: (jsValue: Buffer) => string; + parse: (str: string) => Buffer; +} { + return { + // Node-postgres returns bytea values as Buffer already. + dbValueToJs: (dbValue) => dbValue, + + // PG's representation: '\xDEADBEEF' + stringify: (jsValue) => "\\x" + jsValue.toString("hex"), + + parse: (str) => types.getTypeParser(types.builtins.BYTEA)(str) as Buffer, + }; +} diff --git a/src/pg/types/StringArrayType.ts b/src/pg/types/StringArrayType.ts new file mode 100644 index 0000000..11cf07a --- /dev/null +++ b/src/pg/types/StringArrayType.ts @@ -0,0 +1,35 @@ +import { types } from "pg"; +import type { DesperateAny } from "../../internal/misc"; + +/** + * It is in pg-types/lib/textParsers.js (1015::regtype is "character + * varying[]"), just not exported to TS types. + */ +const VARCHAR_ARRAY_OID: DesperateAny = 1015; + +/** + * An array of Strings. Note: node-postgres natively supports this type on read + * path, but on write path, we have to stringify by ourselves. + */ +export function StringArrayType(): { + dbValueToJs: (dbValue: T[]) => T[]; + stringify: (jsValue: T[]) => string; + parse: (str: string) => T[]; +} { + return { + dbValueToJs: (dbValue) => dbValue, + + stringify: (jsValue) => + "{" + + jsValue + .map((v) => + v === null + ? "NULL" + : `"${v.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`, + ) + .join(",") + + "}", + + parse: (str) => types.getTypeParser(VARCHAR_ARRAY_OID)(str) as T[], + }; +} diff --git a/src/pg/types/__tests__/BigIntArrayType.test.ts b/src/pg/types/__tests__/BigIntArrayType.test.ts new file mode 100644 index 0000000..6fcf259 --- /dev/null +++ b/src/pg/types/__tests__/BigIntArrayType.test.ts @@ -0,0 +1,52 @@ +import { testSpecTypeIntegrity } from "../../../helpers/testSpecTypeIntegrity"; +import { BigIntArrayType } from "../BigIntArrayType"; + +test("sanity", () => { + expect( + testSpecTypeIntegrity(BigIntArrayType(), ["5012413060896574870", "123"]), + ).toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + "5012413060896574870", + "123", + ], + "stringifiedBack": "{5012413060896574870,123}", + } + `); +}); + +test("nulls", () => { + expect(testSpecTypeIntegrity(BigIntArrayType(), ["123", null, null, "234"])) + .toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + "123", + null, + null, + "234", + ], + "stringifiedBack": "{123,NULL,NULL,234}", + } + `); + expect(testSpecTypeIntegrity(BigIntArrayType(), [null])) + .toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + null, + ], + "stringifiedBack": "{NULL}", + } + `); +}); + +test("non-nullable type", () => { + expect(testSpecTypeIntegrity(BigIntArrayType(), ["123"])) + .toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + "123", + ], + "stringifiedBack": "{123}", + } + `); +}); diff --git a/src/pg/types/__tests__/ByteaBufferType.test.ts b/src/pg/types/__tests__/ByteaBufferType.test.ts new file mode 100644 index 0000000..47353de --- /dev/null +++ b/src/pg/types/__tests__/ByteaBufferType.test.ts @@ -0,0 +1,21 @@ +import { testSpecTypeIntegrity } from "../../../helpers/testSpecTypeIntegrity"; +import { ByteaBufferType } from "../ByteaBufferType"; + +test("sanity", () => { + expect( + testSpecTypeIntegrity(ByteaBufferType(), Buffer.from("DEADBEEF", "hex")), + ).toMatchInlineSnapshot(` + { + "jsValueDecoded": { + "data": [ + 222, + 173, + 190, + 239, + ], + "type": "Buffer", + }, + "stringifiedBack": "\\xdeadbeef", + } + `); +}); diff --git a/src/pg/types/__tests__/StringArrayType.test.ts b/src/pg/types/__tests__/StringArrayType.test.ts new file mode 100644 index 0000000..4be6fd0 --- /dev/null +++ b/src/pg/types/__tests__/StringArrayType.test.ts @@ -0,0 +1,97 @@ +import { testSpecTypeIntegrity } from "../../../helpers/testSpecTypeIntegrity"; +import { StringArrayType } from "../StringArrayType"; + +test("sanity", () => { + expect( + testSpecTypeIntegrity(StringArrayType(), [ + "a", + null, + 'a"b"c', + "a\\n", + "a\nb\nc", + ]), + ).toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + "a", + null, + "a"b"c", + "a\\n", + "a + b + c", + ], + "stringifiedBack": "{"a",NULL,"a\\"b\\"c","a\\\\n","a^nb^nc"}", + } + `); +}); + +test("nulls", () => { + expect( + testSpecTypeIntegrity(StringArrayType(), [ + "abc", + null, + null, + "null", + "NULL", + ]), + ).toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + "abc", + null, + null, + "null", + "NULL", + ], + "stringifiedBack": "{"abc",NULL,NULL,"null","NULL"}", + } + `); + expect(testSpecTypeIntegrity(StringArrayType(), [null])) + .toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + null, + ], + "stringifiedBack": "{NULL}", + } + `); +}); + +test("non-nullable type", () => { + expect(testSpecTypeIntegrity(StringArrayType(), ["abc"])) + .toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + "abc", + ], + "stringifiedBack": "{"abc"}", + } + `); +}); + +test("enum and literal type", () => { + enum MyEnum { + A = "a", + } + + expect(testSpecTypeIntegrity(StringArrayType(), [MyEnum.A])) + .toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + "a", + ], + "stringifiedBack": "{"a"}", + } + `); + + expect(testSpecTypeIntegrity(StringArrayType<"a" | "b">(), ["a"])) + .toMatchInlineSnapshot(` + { + "jsValueDecoded": [ + "a", + ], + "stringifiedBack": "{"a"}", + } + `); +}); diff --git a/src/tools/ToolPing.ts b/src/tools/ToolPing.ts new file mode 100644 index 0000000..0c6cc67 --- /dev/null +++ b/src/tools/ToolPing.ts @@ -0,0 +1,196 @@ +import { Readable } from "stream"; +import chalk from "chalk"; +import delay from "delay"; +import compact from "lodash/compact"; +import defaults from "lodash/defaults"; +import first from "lodash/first"; +import type { Client } from "../abstract/Client"; +import type { Cluster } from "../abstract/Cluster"; +import { OP_PING, OP_SHARD_NOS } from "../abstract/internal/misc"; +import { QueryPing } from "../abstract/QueryPing"; +import type { Shard } from "../abstract/Shard"; +import { Timeline } from "../abstract/Timeline"; +import { indent, nullthrows } from "../internal/misc"; +import type { MaybeError, PickPartial } from "../internal/misc"; +import { formatTimeWithMs } from "./internal/formatTimeWithMs"; +import { highlightIf } from "./internal/highlightIf"; + +/** + * Ping tool constructor options. + */ +export interface ToolPingOptions { + cluster: Cluster; + shard?: number; + pingExecTimeMs?: number; + pingPollMs?: number; + pingIsWrite?: boolean; +} + +/** + * A tool which plays the role of Linux `ping` command, but for master() or + * replica() Client of a Shard. Allows to verify that there is no downtime + * happening when a PG node goes down or experiences a failover/switchover. + */ +export class ToolPing { + /** Default values for the constructor options. */ + static readonly DEFAULT_OPTIONS: Required> = { + shard: 0, + pingExecTimeMs: 0, + pingPollMs: 500, + pingIsWrite: false, + }; + + /** Options of this tool. */ + readonly options: Required; + + /** + * Initializes the instance. + */ + constructor(options: ToolPingOptions) { + this.options = defaults({}, options, ToolPing.DEFAULT_OPTIONS); + } + + /** + * Runs an endless loop that pings a master() or replica() Client of the + * passed Island. Yields the colored output line by line. + */ + async *[Symbol.asyncIterator](): AsyncGenerator { + const cluster = this.options.cluster; + const stream = new Readable({ objectMode: true, read() {} }); + + const oldLoggers = { ...cluster.options.loggers }; + + cluster.options.loggers.clientQueryLogger = ({ + annotations, + op, + elapsed, + error, + backend, + role, + connStats, + }) => { + const annotation = first(annotations); + if (op === OP_PING || op === OP_SHARD_NOS) { + stream.push( + chalk[ + error + ? op === OP_PING + ? "yellowBright" + : "yellow" + : op === OP_PING + ? "whiteBright" + : "white" + ]( + compact([ + `[${formatTimeWithMs(new Date(), true)}]`, + "clientQueryLogger", + op, + `backend=${backend}`, + highlightIf( + `elapsed=${Math.round(elapsed.total).toString().padEnd(2)}`, + "bgBlue", + () => elapsed.total > 3000, + ), + `role=${role}`, + `conn.id=${connStats.id}`, + `conn.queriesSent=${connStats.queriesSent}`, + ...(annotation + ? [ + highlightIf( + `attempt=${annotation.attempt}`, + "bgGrey", + () => annotation.attempt > 0, + ), + `whyClient=${annotation.whyClient}`, + `|| ${error ?? "ok"}`, + ] + : [`|| ${error?.replace(/\s*\[\S+\]$/s, "") ?? "ok"}`]), + ]).join(" "), + ), + ); + } + }; + + cluster.options.loggers.swallowedErrorLogger = (props) => { + if (props.importance === "low") { + return; + } + + stream.push( + chalk.gray( + [ + `[${formatTimeWithMs(new Date(), true)}]`, + "swallowedErrorLogger", + highlightIf( + `elapsed=${Math.round(props.elapsed ?? 0) + .toString() + .padEnd(2)}`, + "bgBlue", + () => (props.elapsed ?? 0) > 3000, + ), + `where="${props.where}"`, + "||", + (props.error as MaybeError)?.message?.replace(/\n.*/s, ""), + ].join(" "), + ), + ); + }; + + let stop = false; + const promise = (async () => { + let shard: Shard | undefined = undefined; + try { + const shards = [ + cluster.globalShard(), + ...(await cluster.nonGlobalShards()), + ]; + shard = nullthrows( + shards.find((s) => s.no === this.options.shard), + "No such Shard", + ); + } catch (e: unknown) { + stream.push(indent(chalk.redBright(`Shard selection threw ${e}`))); + return; + } + + while (!stop) { + try { + const startTime = performance.now(); + await shard!.run( + new QueryPing({ + execTimeMs: this.options.pingExecTimeMs, + isWrite: this.options.pingIsWrite, + }), + { + trace: "ping-trace", + vc: "ping-vc", + debugStack: "", + whyClient: undefined, + attempt: 0, + }, + new Timeline(), + null, + ); + const duration = Math.round(performance.now() - startTime); + stream.push( + indent(chalk.green(`ping() succeeded in ${duration} ms`)), + ); + } catch (e: unknown) { + stream.push(indent(chalk.redBright(`ping() threw ${e}`))); + } + + await delay(this.options.pingPollMs); + } + })(); + + try { + for await (const item of stream) { + yield item; + } + } finally { + stop = true; + await promise; + Object.assign(cluster.options.loggers, oldLoggers); + } + } +} diff --git a/src/tools/ToolScoreboard.ts b/src/tools/ToolScoreboard.ts new file mode 100644 index 0000000..a1c2678 --- /dev/null +++ b/src/tools/ToolScoreboard.ts @@ -0,0 +1,592 @@ +import chalk from "chalk"; +import delay from "delay"; +import compact from "lodash/compact"; +import defaults from "lodash/defaults"; +import first from "lodash/first"; +import range from "lodash/range"; +import sortBy from "lodash/sortBy"; +import takeWhile from "lodash/takeWhile"; +import uniqBy from "lodash/uniqBy"; +import pDefer from "p-defer"; +import { table } from "table"; +import type { ClientPingInput } from "../abstract/Client"; +import { Client } from "../abstract/Client"; +import type { Cluster } from "../abstract/Cluster"; +import { OP_SHARD_NOS } from "../abstract/internal/misc"; +import type { ClientQueryLoggerProps } from "../abstract/Loggers"; +import { QueryPing } from "../abstract/QueryPing"; +import { MASTER, STALE_REPLICA } from "../abstract/Shard"; +import { Timeline } from "../abstract/Timeline"; +import { DefaultMap } from "../internal/DefaultMap"; +import { + firstLine, + indent, + join, + mapJoin, + nullthrows, + runInVoid, +} from "../internal/misc"; +import type { MaybeError, PickPartial } from "../internal/misc"; +import { formatTimeWithMs } from "./internal/formatTimeWithMs"; + +/** + * Scoreboard tool constructor options. + */ +export interface ToolScoreboardOptions { + cluster: Cluster; + refreshMs?: number; + pingExecTimeMs?: number; + pingParallelism?: number; + pingPollMs?: number; + tickMs?: number; + maxQueries?: number; + maxErrors?: number; +} + +type ClientIdent = number | "master" | "replica"; + +interface ToolScoreboardQuery { + timestamp: number; + elapsed: number | null; + op: "tick" | "discovery" | "query"; + error: string | null; +} + +interface ToolScoreboardSwallowedError { + timestamp: number; + elapsed: number | null; + message: string; +} + +interface ToolScoreboardQueryError { + timestamp: number; + elapsed: number | null; + message: string; + clientIdent: ClientIdent; +} + +const ROTATING_CHARS = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"; +const FACES = ["➀", "➁", "➂", "➃", "➄", "➅", "➆", "➇", "➈"]; + +/** + * A tool which plays the role of Linux `top` command, but for the Cluster. + * Tracks the state of the Cluster and Clients health. + */ +export class ToolScoreboard { + /** Default values for the constructor options. */ + static readonly DEFAULT_OPTIONS: Required< + PickPartial + > = { + refreshMs: 100, + pingExecTimeMs: 0, + pingParallelism: 1, + pingPollMs: 200, + tickMs: 0, + maxQueries: 30, + maxErrors: 6, + }; + + private launchedPollers: Set = new Set(); + private renderCallCount = 0; + private renderCallFirstAt?: number; + private queryPollDefers: Array> = []; + + /** Options of this tool. */ + readonly options: Required; + + /** Registry of all Islands with Clients. */ + islands: Map< + number, + { + shards: number; + clients: Map; + } + > = new Map(); + + /** Log of queries sent (ping, discovery, tick). */ + queries = new DefaultMap< + number, + DefaultMap + >(); + + /** Pool stats of Clients. */ + poolStats = new DefaultMap< + number, + Map + >(); + + /** Registry of the recent swallowed errors (pings-independent). */ + swallowedErrors: ToolScoreboardSwallowedError[] = []; + + /** Errors extracted from the queries log. */ + queryErrors: ToolScoreboardQueryError[] = []; + + /** + * Initializes the instance. + */ + constructor(options: ToolScoreboardOptions) { + this.options = defaults({}, options, ToolScoreboard.DEFAULT_OPTIONS); + } + + /** + * Runs an endless loop that updates the Scoreboard with the current state of + * the Cluster and yields back on every refreshMs tick. + */ + async *[Symbol.asyncIterator](): AsyncGenerator { + const cluster = this.options.cluster; + + const oldLoggers = { ...cluster.options.loggers }; + + cluster.options.loggers.clientQueryLogger = (props) => { + for (const { islandNo, clientIdent } of this.findClients(props.address)) { + this.poolStats + .getOrAdd(islandNo, Map) + .set(clientIdent, props.poolStats); + if (props.op === OP_SHARD_NOS) { + this.addQuery(islandNo, clientIdent, { + timestamp: Date.now(), + elapsed: props.elapsed.total, + op: "discovery", + error: props.error ? `clientQueryLogger: ${props.error}` : null, + }); + } + } + }; + + cluster.options.loggers.swallowedErrorLogger = ({ error, elapsed }) => + this.addSwallowedError({ + timestamp: Date.now(), + elapsed, + message: `swallowedErrorLogger: ${error}`, + }); + + const tickInterval = this.options.tickMs + ? setInterval(() => { + for (const [islandNo, { clients }] of this.islands) { + for (const clientIdent of clients.keys()) { + this.addQuery(islandNo, clientIdent, { + timestamp: Date.now(), + elapsed: null, + op: "tick", + error: null, + }); + } + } + }, this.options.tickMs).unref() + : undefined; + + const queryPollTimeouts: NodeJS.Timeout[] = []; + for (const i of range(this.options.pingParallelism)) { + this.queryPollDefers[i] = pDefer(); + queryPollTimeouts.push( + setTimeout( + () => + queryPollTimeouts.push( + setInterval(() => { + nullthrows(this.queryPollDefers.at(i)).resolve(); + this.queryPollDefers[i] = pDefer(); + }, this.options.pingPollMs).unref(), + ), + i * (this.options.pingPollMs / this.options.pingParallelism), + ).unref(), + ); + } + + try { + while (true) { + this.islands = new Map( + await mapJoin(cluster.islands(), async (island) => { + const res = await join({ + islandNo: island.no, + shards: island.shards().length, + clients: join([ + island.clients, + island.master(), + island.replica(), + ] as const).then( + ([clients, master, replica]) => + new Map([ + ["master", master], + ["replica", replica], + ...clients.entries(), + ]), + ), + }); + for (const clientIdent of res.clients.keys()) { + for (const i of range(this.options.pingParallelism)) { + const key = `${island.no}:${clientIdent}:${i}`; + if (!this.launchedPollers.has(key)) { + this.launchedPollers.add(key); + runInVoid( + this.pollerLoop(island.no, clientIdent, i).finally(() => + this.launchedPollers.delete(key), + ), + ); + } + } + } + + return [island.no, res]; + }), + ); + + for (const [islandNo, statsByClientIdent] of this.poolStats.entries()) { + for (const clientIdent of statsByClientIdent.keys()) { + if (!this.islands.get(islandNo)?.clients.has(clientIdent)) { + statsByClientIdent.delete(clientIdent); + } + } + + if (statsByClientIdent.size === 0) { + this.poolStats.delete(islandNo); + } + } + + this.swallowedErrors = this.swallowedErrors.filter( + (e) => + e.timestamp > + Date.now() - this.options.pingPollMs * this.options.maxQueries, + ); + + this.queryErrors = []; + + for (const [islandNo, queriesByClientIdent] of this.queries.entries()) { + for (const [clientIdent, queries] of queriesByClientIdent.entries()) { + if (this.islands.get(islandNo)?.clients.has(clientIdent)) { + for (const { timestamp, elapsed, error } of queries) { + if (error) { + this.queryErrors.push({ + timestamp, + elapsed, + message: error, + clientIdent, + }); + } + } + } else { + queriesByClientIdent.delete(clientIdent); + } + } + + if (queriesByClientIdent.size === 0) { + this.queries.delete(islandNo); + } + } + + this.queryErrors = uniqBy( + sortBy( + this.queryErrors, + ({ clientIdent }) => (typeof clientIdent === "string" ? 0 : 1), + ({ message }) => message, + ), + ({ message, clientIdent }) => message + clientIdent, + ).slice(0, this.options.maxErrors); + + yield this; + + await delay(this.options.refreshMs); + } + } finally { + this.queryPollDefers.forEach((defer) => defer.resolve()); + this.islands.clear(); + this.queries.clear(); + this.swallowedErrors = []; + this.queryErrors = []; + Object.assign(cluster.options.loggers, oldLoggers); + queryPollTimeouts.forEach((interval) => clearTimeout(interval)); + clearInterval(tickInterval); + } + } + + /** + * Renders the current state of the Scoreboard as a string. + */ + render(): string { + this.renderCallCount++; + this.renderCallFirstAt ??= Date.now(); + const queriesWidth = this.options.maxQueries * 2; + + const rows: string[][] = []; + rows.push([ + "Island", + "#", + "Client", + "Role", + "Pool Conns", + "Queries (ms or Ⓝ ×10 - pings; D - discovery; red - error)", + "Health", + ]); + for (const [islandNo, { clients }] of this.islands) { + let i = 0; + for (const [clientIdent, client] of clients.entries()) { + let curQueriesWidth = 0; + const primaryColor = typeof clientIdent !== "string" ? "gray" : "white"; + const connectionIssue = client.connectionIssue(); + const queries = this.queries.get(islandNo)?.get(clientIdent) ?? []; + const poolStats = this.poolStats.get(islandNo)?.get(clientIdent); + rows.push([ + // Island + i === 0 ? `${islandNo}` : "", + // # + chalk[primaryColor]( + typeof clientIdent !== "string" + ? `#${clientIdent}` + : `${clientIdent}()`, + ), + // Client + chalk[primaryColor](client.options.name), + // Role + chalk[primaryColor](client.role()), + // Pool Conns + typeof clientIdent !== "string" && poolStats + ? chalk[primaryColor]( + `${poolStats.totalConns} (${poolStats.totalConns - poolStats.idleConns} busy)`.padEnd( + 13, + " ", + ), + ) + : "", + // Queries + takeWhile( + compact( + queries.map((query) => this.renderQuery(clientIdent, query)), + ), + ([str]) => { + if (curQueriesWidth + str.length < queriesWidth) { + curQueriesWidth += str.length + 1; + return true; + } else { + return false; + } + }, + ) + .map(([str, color]) => chalk[color](str)) + .join(" ") + " ".repeat(queriesWidth - curQueriesWidth), + // Health + connectionIssue + ? chalk[clientIdentErrorColor(clientIdent)]( + "UNHEALTHY: " + + firstLine((connectionIssue.cause as MaybeError)?.message), + ) + : chalk.green("healthy") + + (clientIdent === "replica" && client.role() === "master" + ? " (but fallback to master)" + : ""), + ]); + i++; + } + } + + const lines: string[] = []; + + lines.push( + "[" + + formatTimeWithMs(new Date()) + + (this.renderCallFirstAt + ? `, ${((Date.now() - this.renderCallFirstAt) / 1000).toFixed(1)} sec elapsed` + : "") + + "]", + ); + + lines.push( + table(rows, { + drawHorizontalLine: (i, rowCount) => + i === 0 || i === 1 || i === rowCount, + spanningCells: [], + }), + ); + + lines.push( + ...[ + ...this.queryErrors, + ...this.swallowedErrors.map((e) => ({ ...e, clientIdent: null })), + ].map(({ timestamp, message, clientIdent }) => + chalk[clientIdentErrorColor(clientIdent)]( + indent( + "- " + + (clientIdent === null + ? "" + : typeof clientIdent !== "string" + ? `[pinging client ${clientIdent}] ` + : `[pinging ${clientIdent}()] `) + + firstLine(message) + + ` [${formatTimeWithMs(new Date(timestamp))}] `, + ).substring(2), + ), + ), + ); + + return lines.join("\n"); + } + + /** + * Renders a colorful cell corresponding to one query. + */ + private renderQuery( + clientIdent: ClientIdent, + { timestamp, elapsed, op, error }: ToolScoreboardQuery, + ): [string, typeof chalk.Color] | null { + const lag = Math.round(Date.now() - timestamp); + const rot = ROTATING_CHARS[this.renderCallCount % ROTATING_CHARS.length]; + return op === "tick" + ? [".", "gray"] + : op === "discovery" + ? ["D", error ? clientIdentErrorColor(null) : "white"] + : elapsed === null && lag < this.options.pingExecTimeMs + ? null + : elapsed === null && lag > this.options.pingExecTimeMs + 500 + ? [ + rot + this.renderElapsed(lag) + rot, + error ? clientIdentErrorColor(clientIdent) : "magentaBright", + ] + : [ + this.renderElapsed(elapsed ?? lag), + error ? clientIdentErrorColor(clientIdent) : "green", + ]; + } + + /** + * Renders the text value of a cell corresponding to a query. + */ + private renderElapsed(elapsed: number): string { + return elapsed > 1000 + ? `${Math.trunc(elapsed / 1000)}s` + : elapsed >= 10 * (FACES.length + 1) + ? Math.trunc(elapsed).toString() + : elapsed < 10 + ? Math.trunc(elapsed).toString() + : FACES[Math.min(Math.trunc(elapsed / 10), FACES.length) - 1]; + } + + /** + * Runs an endless polling loop for the provided Client. The loop terminates + * if the Client disappears from the cluster. + */ + private async pollerLoop( + islandNo: number, + clientIdent: ClientIdent, + i: number, + ): Promise { + const cluster = this.options.cluster; + + while (true) { + const island = this.islands.get(islandNo); + if (!island) { + return; + } + + const client = + typeof clientIdent !== "string" + ? island.clients.get(clientIdent) + : clientIdent === "master" + ? MASTER + : STALE_REPLICA; + if (!client) { + return; + } + + // Sync pings in all pollerLoop() loops among each other. + await nullthrows(this.queryPollDefers.at(i)).promise; + + const timestamp = Date.now(); + const query = this.addQuery(islandNo, clientIdent, { + timestamp, + elapsed: null, + op: "query", + error: null, + }); + const input: ClientPingInput = { + execTimeMs: this.options.pingExecTimeMs, + isWrite: clientIdent === "master", + annotation: { + trace: "scoreboard-trace", + vc: "scoreboard-vc", + debugStack: "", + whyClient: undefined, + attempt: 0, + }, + }; + + let error: string | null = null; + try { + if (client instanceof Client) { + await client.ping(input); + } else { + const island = await cluster.island(islandNo); + const shard = first(island.shards()); + if (!shard) { + throw Error( + "No Shards on this Island, or Shards discovery never succeeded.", + ); + } + + // For master() and replica(), we run queries through Shard, to + // benefit from its zero-downtime retries logic. + await shard.run( + new QueryPing(input), + input.annotation, + new Timeline(), + client, + ); + } + } catch (e: unknown) { + error = "thrown: " + e; + } finally { + query.elapsed = Date.now() - timestamp; + query.error = error; + } + } + } + + /** + * Adds a query (probably running right now) to the Scoreboard. + */ + private addQuery( + islandNo: number, + clientIdent: ClientIdent, + query: ToolScoreboardQuery, + ): ToolScoreboardQuery { + const slot = this.queries + .getOrAdd(islandNo, DefaultMap) + .getOrAdd(clientIdent, Array); + slot.unshift(query); + slot.splice(this.options.maxQueries); + return query; + } + + /** + * Adds an error to the Scoreboard. + */ + private addSwallowedError(error: ToolScoreboardSwallowedError): void { + this.swallowedErrors = [ + error, + ...this.swallowedErrors.filter((e) => e.message !== error.message), + ]; + this.swallowedErrors.splice(this.options.maxErrors); + } + + /** + * Finds all existing Client by matching their addresses to the passed one. + */ + private *findClients( + address: string, + ): Generator<{ islandNo: number; clientIdent: ClientIdent; client: Client }> { + for (const [islandNo, { clients }] of this.islands) { + for (const [clientIdent, client] of clients.entries()) { + if (client.address() === address) { + yield { islandNo, clientIdent, client }; + } + } + } + } +} + +function clientIdentErrorColor( + clientIdent: ClientIdent | null, +): typeof chalk.Color { + return clientIdent === null + ? "cyan" + : typeof clientIdent !== "string" + ? "yellow" + : "red"; +} diff --git a/src/tools/index.ts b/src/tools/index.ts new file mode 100644 index 0000000..46defc8 --- /dev/null +++ b/src/tools/index.ts @@ -0,0 +1,6 @@ +/** + * @file Automatically generated by barrelsby. + */ + +export * from "./ToolPing"; +export * from "./ToolScoreboard"; diff --git a/src/tools/internal/formatTimeWithMs.ts b/src/tools/internal/formatTimeWithMs.ts new file mode 100644 index 0000000..cd0276c --- /dev/null +++ b/src/tools/internal/formatTimeWithMs.ts @@ -0,0 +1,19 @@ +let prevDate: Date | null = null; + +export function formatTimeWithMs(date: Date, withDelta = false): string { + const h = date.getHours().toString(); + const m = date.getMinutes().toString(); + const s = date.getSeconds().toString(); + const ms = date.getMilliseconds().toString(); + + let deltaMs: number | null = null; + if (withDelta) { + deltaMs = prevDate ? date.getTime() - prevDate.getTime() : 0; + prevDate = date; + } + + return ( + `${h.padStart(2, "0")}:${m.padStart(2, "0")}:${s.padStart(2, "0")}.${ms.padStart(3, "0")}` + + (deltaMs !== null ? " in " + `${deltaMs}ms`.padStart(7) : "") + ); +} diff --git a/src/tools/internal/highlightIf.ts b/src/tools/internal/highlightIf.ts new file mode 100644 index 0000000..b3cd2a7 --- /dev/null +++ b/src/tools/internal/highlightIf.ts @@ -0,0 +1,9 @@ +import chalk from "chalk"; + +export function highlightIf( + message: string, + color: typeof chalk.Color | typeof chalk.Modifiers, + condition: () => boolean, +): string { + return condition() ? chalk[color](message) : message; +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..ff9da7f --- /dev/null +++ b/src/types.ts @@ -0,0 +1,428 @@ +import type { DesperateAny, TuplePrefixes } from "./internal/misc"; + +/** + * Primary key field's name is currently hardcoded for simplicity. It's a + * convention to have it named as "id". + */ +export const ID = "id"; + +/** + * Literal operation with placeholders. We don't use a tuple type here (like + * `[string, ...T[]]`), because it would force us to use `as const` everywhere, + * which we don't want to do. + */ +export type Literal = Array< + | string + | number + | boolean + | Date + | null + | Array +>; + +// - Table: an object of Field:Spec +// - Field: name of the column in the table (and element of a Row) +// - Spec: detailed description of the column (type, allowNull etc.) +// - Row: an object with { Column: Value } data in it +// - Input: a subset of Row suitable for a particular write operation +// - Value: the actual data type (string, number, Date) + +/** + * { id: string } + */ +export type RowWithID = { + [ID]: string; +}; + +/** + * Spec (metadata) of some field. + */ +export type SpecType = + | typeof Boolean + | typeof Date + | typeof ID + | typeof Number + | typeof String + | { + /** Converts a value of some field returned by the low-level DB engine to + * its Client representation, which can be reacher (e.g. support + * encryption/decryption). Notice that some DB engines already do some + * conversions internally: e.g. for node-postgres and an array field, + * dbValue returned by the engine is already an array of things, so + * dbValueToJs for it will likely do nothing. */ + dbValueToJs: (dbValue: DesperateAny) => unknown; + /** Converts a Client value to the internal stringified representation of + * the low-level DB engine, which is suitable for injecting it into a + * plaintext query (with e.g. ?-placeholders). + * - Notice that this is intentionally NOT the opposite to dbValueToJs, + * because it always needs to convert the value to a string, not to the + * DB engine's row field type. + * - Example: node-postgres natively understands json/jsonb PG types and + * can unescape them (called "PG type parsers" and mainly lives in + * pg-types module; notice that there are no "PG type stringifiers + * though"). The problem is that the low-level library's facilities for + * escaping data is poor or doesn't exist (we do escaping by ourselves + * for various reasons, like batching queries and better logging). So we + * trust the library on the dbValueToJs path, but must manually + * serialize on stringify path. */ + stringify: (jsValue: DesperateAny) => string; + /** The opposite to stringify function. Generally, it is not used on the + * read path (because the low level engine returns the rows suitable for + * dbValueToJs), but it's still here for completeness of the interface. */ + parse: (str: string) => unknown; + }; + +/** + * { type: ..., ... } - one attribute spec. + */ +export type Spec = { + type: SpecType; + allowNull?: true; + autoInsert?: string; + autoUpdate?: string; +}; + +/** + * { id: Spec, name: Spec, ... } - table columns. + */ +export type Table = { + [K: string | symbol]: Spec; +}; + +/** + * A database table's field (no symbols). In regards to some table structure, + * there can be 3 options: + * 1. Field: only DB-stored attributes, no ephemeral symbols + * 2. keyof TTable: both real and ephemeral attributes + * 3. keyof TTable & symbol: only "ephemeral" attributes available to triggers + * + * By doing `& string`, we ensure that we select only regular (non-symbol) + * fields. + */ +export type Field = keyof TTable & string; + +/** + * Same as Field, but may optionally hold information about of "alias value + * source" for a field name (e.g. `{ field: "abc", alias: "$cas.abc" }`). + */ +export type FieldAliased = + | Field + | { field: Field; alias: string }; + +/** + * (Table) -> "field1" | "field2" | ... where the union contains only fields + * which can potentially be used as a part of unique key. + */ +export type FieldOfPotentialUniqueKey = { + [K in Field]: TTable[K] extends { + type: + | typeof Number + | typeof String + | typeof Boolean + | typeof ID + | typeof Date + | { dbValueToJs: (dbValue: never) => string | number }; + // allows nullable fields too! + } + ? K + : never; +}[Field]; + +/** + * Table -> "user_id" | "some_id" | ... + */ +export type FieldOfIDType = { + [K in Field]: K extends string + ? TTable[K] extends { type: typeof ID } + ? K + : never + : never; +}[Field]; + +/** + * Table -> "user_id" | "some_id" | ... + */ +export type FieldOfIDTypeRequired = + InsertFieldsRequired & FieldOfIDType; + +/** + * SpecType -> Value deduction (always deduces non-nullable type). + */ +export type ValueRequired = + TSpec["type"] extends typeof Number + ? number + : TSpec["type"] extends typeof String + ? string + : TSpec["type"] extends typeof Boolean + ? boolean + : TSpec["type"] extends typeof ID + ? string + : TSpec["type"] extends typeof Date + ? Date + : TSpec["type"] extends { + dbValueToJs: (dbValue: never) => infer TJSValue; + } + ? TSpec["type"] extends { + stringify: (jsValue: TJSValue) => string; + parse: (str: string) => TJSValue; + } + ? TJSValue + : never + : never; + +/** + * Spec -> nullable Value or non-nullable Value. + */ +export type Value = TSpec extends { allowNull: true } + ? ValueRequired | null + : ValueRequired; + +/** + * Table -> Row deduction (no symbols). + */ +export type Row = RowWithID & { + [K in Field]: Value; +}; + +/** + * Insert: Table -> "field1" | "field2" | ... deduction (required). + */ +export type InsertFieldsRequired = { + [K in keyof TTable]: TTable[K] extends { autoInsert: unknown } + ? never + : TTable[K] extends { autoUpdate: unknown } + ? never + : K; +}[keyof TTable]; + +/** + * Insert: Table -> "created_at" | "field2" | ... deduction (optional fields). + */ +export type InsertFieldsOptional = { + [K in keyof TTable]: TTable[K] extends { autoInsert: unknown } + ? K + : TTable[K] extends { autoUpdate: unknown } + ? K + : never; +}[keyof TTable]; + +/** + * Insert: Table -> { field: string, updated_at?: Date, created_at?: Date... }. + * Excludes id Spec entirely and makes autoInsert/autoUpdate Specs optional. + */ +export type InsertInput = { + [K in InsertFieldsRequired]: Value; +} & { + [K in InsertFieldsOptional]?: Value; +}; + +/** + * Update: Table -> "field1" | "created_at" | "updated_at" | ... deduction. + */ +export type UpdateField = Exclude< + keyof TTable, + keyof RowWithID +>; + +/** + * Update: Table -> { field?: string, created_at?: Date, updated_at?: Date }. + * - Excludes id Spec entirely and makes all fields optional. + * - If $literal is passed, it will be appended to the list of updating fields + * (engine specific). + * - If $cas is passed, only the rows whose fields match the exact values in + * $cas will be updated; the non-matching rows will be skipped. + */ +export type UpdateInput = { + [K in UpdateField]?: Value; +} & { + $literal?: Literal; + $cas?: { [K in UpdateField]?: Value }; +}; + +/** + * Table -> ["field1", "field2", ...], list of fields allowed to compose an + * unique key on the table; fields must be allowed in insert/upsert. + */ +export type UniqueKey = + | [] + | [ + FieldOfPotentialUniqueKey, + ...Array>, + ]; + +/** + * (Table, UniqueKey) -> { field1: number, field2: number, field3: number }. + * loadBy operation is allowed for exact unique key attributes only. + */ +export type LoadByInput< + TTable extends Table, + TUniqueKey extends UniqueKey, +> = TUniqueKey extends [] + ? never + : { [K in TUniqueKey[number]]: Value }; + +/** + * (Table, UniqueKey) -> { field1: number [, field2: number [, ...] ] }. + * selectBy operation is allowed for unique key PREFIX attributes only. + */ +export type SelectByInput< + TTable extends Table, + TUniqueKey extends UniqueKey, +> = LoadByInput>; + +/** + * Table -> { f: 10, [$or]: [ { f2: "a }, { f3: "b""} ], $literal: ["x=?", 1] } + */ +export type Where = { + // Block operators for SelectInput. We MUST use string versions of them and + // start with $, because we calculate the query key by doing JSON + // serializations, and Symbol doesn't serialize. This is a step back in some + // sense (in comparison to e.g. Sequelize). + // + // Example: { $op: { field: "value" } } + $and?: ReadonlyArray>; + $or?: ReadonlyArray>; + $not?: Where; + $literal?: Literal; + $shardOfID?: string; +} & { + [ID]?: TTable extends { [ID]: unknown } ? unknown : string | string[]; +} & { + [K in Field]?: + | Value + | ReadonlyArray> + // Field comparison operators for SelectInput. + // + // Example: { field: { $op: "value" } } + | { $lte: NonNullable> } + | { $lt: NonNullable> } + | { $gte: NonNullable> } + | { $gt: NonNullable> } + | { $overlap: NonNullable> } + | { $ne: Value | ReadonlyArray> } + | { $isDistinctFrom: Value }; +}; + +/** + * Table -> [["f1", "ASC"], ["f2", "DESC"]] or [ [{[$literal]: ["a=?", 10]}, + * "ASC"], ["b", "DESC"] ] + */ +export type Order = ReadonlyArray< + { [K in Field]?: string } & { $literal?: Literal } +>; + +/** + * Table -> { where: ..., order?: ..., ... } + */ +export type SelectInput = { + where: Where; + order?: Order; + custom?: {}; // custom engine-defined parameters/modifiers + limit: number; // required - for safety +}; + +/** + * Table -> { f: 10, [$or]: [ { f2: "a }, { f3: "b""} ], $literal: ["x=?", 1] } + */ +export type CountInput = Where; + +/** + * Table -> { f: 10, [$or]: [ { f2: "a }, { f3: "b""} ], $literal: ["x=?", 1] } + */ +export type ExistsInput = Where; + +/** + * Table -> { id: ["1", "2", "3"], ... } + */ +export type DeleteWhereInput = { [ID]: string[] } & Omit< + Where, + typeof ID +>; + +/** + * Planner hints. Null means "reset to the engine's default", and "undefined" + * means the same as "no key mentioned at all". + */ +export type Hints = Record; + +/** + * A wrapper for literal union types, suitable for the following Spec: + * - { type: EnumType<"a" | "b" | "c">() } + * - { type: EnumType<1 | 2 | 3>() } + */ +export function EnumType(): { + dbValueToJs: (dbValue: string | number) => TValue; + stringify: (jsValue: TValue) => string; + parse: (str: string) => TValue; +}; + +/** + * A wrapper for literal union types, suitable for the following Spec: + * ``` + * enum MyEnum { + * A = "a", + * B = "b", + * } + * ... + * { type: EnumType() } + * ``` + */ +export function EnumType>(): { + dbValueToJs: (dbValue: string | number) => TEnum[keyof TEnum]; + stringify: (jsValue: TEnum[keyof TEnum]) => string; + parse: (str: string) => TEnum[keyof TEnum]; +}; + +/** @ignore */ +export function EnumType(): unknown { + return { + dbValueToJs: (dbValue: string | number) => dbValue, + stringify: (jsValue: never) => jsValue as string, + parse: (str: string) => str, + } as const; +} + +/** + * A value stored in the DB as a base64 encoded binary buffer. Actually, DB + * engines (like PostgreSQL) support native binary data fields (and store binary + * data efficiently), but sometimes (especially for small things, like + * public/private keys), it's easier to store the binary data as base64 encoded + * strings rather than dealing with the native binary data type. + */ +export function Base64BufferType(): { + dbValueToJs: (dbValue: string) => Buffer; + stringify: (jsValue: Buffer) => string; + parse: (str: string) => Buffer; +} { + return { + dbValueToJs: (dbValue) => Buffer.from(dbValue, "base64"), + stringify: (jsValue) => jsValue.toString("base64"), + parse: (str) => Buffer.from(str, "base64"), + }; +} + +/** + * A JSON-serializable value. + */ +export type JSONValue = + | null + | string + | number + | boolean + | JSONValue[] + | { [k in string]?: JSONValue }; + +/** + * An arbitrary JSON field type. + */ +export function JSONType(): { + dbValueToJs: (dbValue: TCurrent) => TCurrent; + stringify: (jsValue: TCurrent) => string; + parse: (str: string) => TCurrent; +} { + return { + dbValueToJs: (dbValue) => dbValue, + stringify: (jsValue) => JSON.stringify(jsValue), + parse: (str) => JSON.parse(str), + }; +} diff --git a/tsconfig.base.json b/tsconfig.base.json new file mode 100644 index 0000000..d5ee298 --- /dev/null +++ b/tsconfig.base.json @@ -0,0 +1,32 @@ +{ + "include": ["src/**/*"], + "compilerOptions": { + "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo", + "allowJs": true, + "composite": true, + "declaration": true, + "declarationMap": true, + "disableReferencedProjectLoad": true, + "disableSourceOfProjectReferenceRedirect": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "incremental": true, + "lib": ["ES2019"], + "module": "Node16", + "noEmitOnError": true, + "noErrorTruncation": true, + "noImplicitOverride": true, + "noImplicitReturns": true, + "noPropertyAccessFromIndexSignature": true, + "outDir": "dist", + "pretty": true, + "removeComments": false, + "resolveJsonModule": true, + "rootDir": "src", + "skipLibCheck": true, + "sourceMap": true, + "strict": true, + "target": "ES2019", + "types": ["node", "jest"] + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..68ebd74 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.base.json", + "include": ["src/**/*"], + "compilerOptions": { + "lib": ["ES2021", "ES2021.WeakRef"], + "target": "ES2021" + } +} diff --git a/typedoc.config.js b/typedoc.config.js new file mode 100644 index 0000000..d2e96b7 --- /dev/null +++ b/typedoc.config.js @@ -0,0 +1,35 @@ +"use strict"; +const { basename } = require("path"); + +module.exports = { + entryPoints: ["src"], + exclude: [ + "**/internal/**", + "**/__tests__/**", + "**/__benchmarks__/**", + "**/node_modules/**", + ], + entryPointStrategy: "expand", + mergeModulesMergeMode: "project", + sort: ["source-order"], + out: "docs", + parametersFormat: "table", + interfacePropertiesFormat: "table", + classPropertiesFormat: "table", + tableColumnSettings: { + hideInherited: true, + hideModifiers: true, + hideOverrides: true, + hideSources: true, + }, + logLevel: "Warn", + hideGenerator: true, + excludeInternal: true, + excludePrivate: true, + categorizeByGroup: true, + blockTags: ["@file", "@deprecated"], // added by e.g. barrelsby + excludeTags: ["@file"], // added by e.g. barrelsby + gitRevision: "master", + sourceLinkTemplate: `https://github.com/clickup/${basename(__dirname)}/blob/master/{path}#L{line}`, + basePath: ".", +};