From b56bb2f9e621469c7c8b9ad88ae552f26d3f7ee4 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Tue, 16 Jun 2026 15:36:45 -0700 Subject: [PATCH 1/8] Add BYOVPC doc --- modules/ROOT/nav.adoc | 1 + .../cluster-types/byoc/aws/vpc-byo-aws.adoc | 1 + .../pages/get-started/deploy-sql-cluster.adoc | 2 +- .../get-started/enable-sql-byovpc-aws.adoc | 140 ++++++++++++++++++ 4 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index fe305c72c..a126bb681 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -333,6 +333,7 @@ ** xref:sql:get-started/index.adoc[Get Started] *** xref:sql:get-started/sql-quickstart.adoc[Quickstart] *** xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL] +*** xref:sql:get-started/enable-sql-byovpc-aws.adoc[Enable Redpanda SQL on a BYOVPC Cluster on AWS] *** xref:sql:get-started/overview.adoc[] **** xref:sql:get-started/oltp-vs-olap.adoc[] **** xref:sql:get-started/redpanda-sql-vs-postgresql.adoc[] diff --git a/modules/get-started/pages/cluster-types/byoc/aws/vpc-byo-aws.adoc b/modules/get-started/pages/cluster-types/byoc/aws/vpc-byo-aws.adoc index 32df78da8..d630ec54a 100644 --- a/modules/get-started/pages/cluster-types/byoc/aws/vpc-byo-aws.adoc +++ b/modules/get-started/pages/cluster-types/byoc/aws/vpc-byo-aws.adoc @@ -324,3 +324,4 @@ module "redpanda_byovpc" { * xref:networking:aws-privatelink.adoc[Configure AWS PrivateLink] * xref:security:authorization/cloud-iam-policies.adoc[Review AWS IAM policies] * xref:reference:rpk/index.adoc[Learn about `rpk` commands] +* xref:sql:get-started/enable-sql-byovpc-aws.adoc[] diff --git a/modules/sql/pages/get-started/deploy-sql-cluster.adoc b/modules/sql/pages/get-started/deploy-sql-cluster.adoc index fa4ba47f4..78d211ffc 100644 --- a/modules/sql/pages/get-started/deploy-sql-cluster.adoc +++ b/modules/sql/pages/get-started/deploy-sql-cluster.adoc @@ -14,7 +14,7 @@ After reading this page, you will be able to: * [ ] {learning-objective-2} * [ ] {learning-objective-3} -NOTE: Redpanda SQL is currently available only on BYOC clusters running on AWS. +NOTE: Redpanda SQL is available on BYOC clusters running on AWS. For BYOVPC clusters, see xref:sql:get-started/enable-sql-byovpc-aws.adoc[]. == Prerequisites diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc new file mode 100644 index 000000000..e97797f5a --- /dev/null +++ b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc @@ -0,0 +1,140 @@ += Enable Redpanda SQL on a BYOVPC Cluster +:description: Enable the Redpanda SQL engine on an existing BYOVPC cluster on AWS by provisioning SQL-specific resources with Terraform and supplying them as customer-managed resources. +:page-topic-type: how-to +:personas: platform_admin +:learning-objective-1: Provision SQL-specific AWS resources using the Redpanda BYOVPC Terraform module +:learning-objective-2: Enable Redpanda SQL on a BYOVPC cluster by supplying customer-managed resources + +Enable Redpanda SQL on a Bring Your Own Virtual Private Cloud (BYOVPC) cluster on AWS so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. + +Unlike standard BYOC clusters, Redpanda does not create resources in your AWS account for BYOVPC clusters. You must provision the SQL-specific AWS resources yourself and supply them to the cluster as customer-managed resources before enabling the SQL engine. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} + +== Prerequisites + +* An existing xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[BYOVPC cluster on AWS]. +* The https://registry.terraform.io/modules/redpanda-data/redpanda-byovpc/aws/latest[Redpanda BYOVPC Terraform module^] and https://developer.hashicorp.com/terraform[Terraform^] version 1.8.5 or later. +* Admin permissions in your Redpanda Cloud organization on xref:billing:billing.adoc[usage-based billing]. +* If using the Cloud API, a valid glossterm:bearer token[]. See link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. + +[NOTE] +==== +The Redpanda Terraform provider does not support SQL customer-managed resource fields. Use the Cloud API or Cloud Console to supply these resources and enable SQL on a BYOVPC cluster. +==== + +== Provision SQL resources with Terraform + +In your `terraform-aws-redpanda-byovpc` module configuration, set `enable_redpanda_sql = true`: + +[,hcl] +---- +module "redpanda_byovpc" { + source = "redpanda-data/redpanda-byovpc/aws" + # ... existing configuration ... + enable_redpanda_sql = true +} +---- + +Apply the updated configuration: + +[,bash] +---- +terraform apply +---- + +This creates three SQL-specific resources in your AWS account: + +* An IAM role and instance profile for SQL compute nodes +* An S3 bucket for SQL data storage (versioning disabled) +* A security group for SQL nodes + +After applying, note the following module outputs. You supply these ARNs to the cluster in the next step. + +[cols="1,3",options="header"] +|=== +| Module output | Description +| `rpsql_node_group_instance_profile_arn` | ARN of the IAM instance profile for SQL nodes +| `rpsql_cloud_storage_bucket_arn` | ARN of the S3 bucket for SQL data storage +| `rpsql_security_group_arn` | ARN of the security group for SQL nodes +|=== + +== Supply SQL resources and enable SQL + +[tabs] +===== +Cloud Console:: ++ +-- +. Log in to https://cloud.redpanda.com[Redpanda Cloud^] and open your BYOVPC cluster. +. From the left navigation, select *Dataplane settings*. +. On the *Cluster* tab, find the *RP SQL* row and click *Edit*. +. In the dialog, enter the ARNs from the Terraform module outputs, then use the *RPU* slider to set the compute size. +. Click *Enable Redpanda SQL engine*. +-- + +Cloud API:: ++ +-- +. link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. +. Locate the cluster ID in the *Details* section of the cluster overview in the Cloud Console. +. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecluster[`PATCH /v1/clusters/{cluster.id}`] request with the SQL customer-managed resources and `rpsql.enabled` set to `true`. Replace `{cluster.id}` with your cluster ID: ++ +[,bash] +---- +curl -X PATCH "https://api.redpanda.com/v1/clusters/{cluster.id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "customer_managed_resources": { + "aws": { + "rpsql_node_group_instance_profile": {"arn": ""}, + "rpsql_cloud_storage_bucket": {"arn": ""}, + "rpsql_security_group": {"arn": ""} + } + }, + "rpsql": { + "enabled": true, + "replicas": , + "zones": [""] + } + }' +---- ++ +Replace the placeholders with your own values: ++ +* ``, ``, ``: The corresponding outputs from the Terraform module. +* ``: Initial number of SQL compute nodes (minimum 1, maximum 9). +* `` (optional): One of the cluster's glossterm:availability zones[]. The SQL engine deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. ++ +. The request returns the ID of a long-running operation. Poll the link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] endpoint until the operation completes: ++ +[,bash] +---- +curl -X GET "https://api.redpanda.com/v1/operations/{operation.id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" +---- ++ +When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. +-- +===== + +[IMPORTANT] +==== +The SQL customer-managed resources (`rpsql_node_group_instance_profile`, `rpsql_cloud_storage_bucket`, `rpsql_security_group`) are immutable while SQL is enabled. To change any of these resources, disable SQL first. See xref:sql:get-started/deploy-sql-cluster.adoc#disable-redpanda-sql[Disable Redpanda SQL]. +==== + +[NOTE] +==== +The SQL-specific AWS resources (IAM instance profile, Amazon S3 bucket, and security group) are not deleted when you disable SQL. They remain in your account and can be reused if you re-enable SQL. To remove them, set `enable_redpanda_sql = false` in the Terraform module and run `terraform apply`, or run `terraform destroy` to remove all BYOVPC resources. +==== + +== Next steps + +* xref:sql:get-started/deploy-sql-cluster.adoc[Scale, verify, and disable the SQL engine]: Adjust compute size, check engine status, and connect with `psql`. +* xref:sql:get-started/sql-quickstart.adoc[Quickstart]: Connect to Redpanda SQL with `psql` and run your first query. +* xref:sql:connect-to-sql/authenticate.adoc[Authenticate to Redpanda SQL]: Connect with an OIDC bearer token, OIDC client credentials, or a SCRAM password. From 91fd17fd3d9986fac23f3766dd5f33061b168f7f Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Tue, 16 Jun 2026 16:23:32 -0700 Subject: [PATCH 2/8] Disable SQL --- .../get-started/enable-sql-byovpc-aws.adoc | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc index e97797f5a..ef0d76ab9 100644 --- a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc +++ b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc @@ -4,6 +4,7 @@ :personas: platform_admin :learning-objective-1: Provision SQL-specific AWS resources using the Redpanda BYOVPC Terraform module :learning-objective-2: Enable Redpanda SQL on a BYOVPC cluster by supplying customer-managed resources +:learning-objective-3: Verify that the SQL engine is running and ready to accept connections Enable Redpanda SQL on a Bring Your Own Virtual Private Cloud (BYOVPC) cluster on AWS so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. @@ -13,6 +14,7 @@ After reading this page, you will be able to: * [ ] {learning-objective-1} * [ ] {learning-objective-2} +* [ ] {learning-objective-3} == Prerequisites @@ -125,16 +127,63 @@ When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. [IMPORTANT] ==== -The SQL customer-managed resources (`rpsql_node_group_instance_profile`, `rpsql_cloud_storage_bucket`, `rpsql_security_group`) are immutable while SQL is enabled. To change any of these resources, disable SQL first. See xref:sql:get-started/deploy-sql-cluster.adoc#disable-redpanda-sql[Disable Redpanda SQL]. +The SQL customer-managed resources (`rpsql_node_group_instance_profile`, `rpsql_cloud_storage_bucket`, `rpsql_security_group`) are immutable while SQL is enabled. To change any of these resources, disable SQL first. ==== +== Verify the SQL engine is running + +After you enable Redpanda SQL, a *Redpanda SQL* tile appears in the cluster overview under *Products*. The tile shows the engine status, active sessions, queries per minute, and node count. When the *Nodes* indicator shows all nodes as ready (for example, *1/1*), the engine is provisioned and accepting connections. Provisioning can take up to 30 minutes. + +For the API flow, poll the long-running operation until it returns `STATE_COMPLETED`. + +To verify the SQL engine is running, use the connection details on the *SQL* tab under *Connection details* to connect with a PostgreSQL client, such as `psql` (v16 or later required). You can also query data directly using the xref:sql:query-data/sql-editor.adoc[SQL editor] in the left navigation. + +== Disable Redpanda SQL + +[WARNING] +==== +Disabling Redpanda SQL tears down the SQL compute engine and clears its catalog state (catalog metadata, table mappings, and role/grant data). In-flight queries fail when SQL is disabled. +==== + +Redpanda topic data, Schema Registry subjects, and any Iceberg-committed history for Iceberg-enabled topics are not affected. The Redpanda cluster itself continues to run normally; only the SQL engine and its associated state are removed. + +Re-enabling SQL on the same cluster provisions a fresh engine: no prior catalog state, table mappings, or grants are restored. You must re-create catalogs, tables, and grants after re-enabling. + [NOTE] ==== The SQL-specific AWS resources (IAM instance profile, Amazon S3 bucket, and security group) are not deleted when you disable SQL. They remain in your account and can be reused if you re-enable SQL. To remove them, set `enable_redpanda_sql = false` in the Terraform module and run `terraform apply`, or run `terraform destroy` to remove all BYOVPC resources. ==== +[tabs] +===== +Cloud Console:: ++ +-- +. Log in to https://cloud.redpanda.com[Redpanda Cloud^] and open your cluster. +. From the left navigation, select *Dataplane settings*. +. On the *Cluster* tab, find the *RP SQL* row and click *Edit*. +. In the *Edit Redpanda SQL engine* dialog, click *Disable*. +-- + +Cloud API:: ++ +-- +Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecluster[`PATCH /v1/clusters/{cluster.id}`] request with `rpsql.enabled` set to `false`. Replace `{cluster.id}` with your cluster ID: + +[,bash] +---- +curl -X PATCH "https://api.redpanda.com/v1/clusters/{cluster.id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"rpsql":{"enabled":false}}' +---- + +The request returns the ID of a long-running operation. Poll link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] until the operation completes. +-- +===== + == Next steps -* xref:sql:get-started/deploy-sql-cluster.adoc[Scale, verify, and disable the SQL engine]: Adjust compute size, check engine status, and connect with `psql`. +* xref:sql:get-started/deploy-sql-cluster.adoc[Scale the SQL engine]: Adjust compute size after enabling SQL. * xref:sql:get-started/sql-quickstart.adoc[Quickstart]: Connect to Redpanda SQL with `psql` and run your first query. * xref:sql:connect-to-sql/authenticate.adoc[Authenticate to Redpanda SQL]: Connect with an OIDC bearer token, OIDC client credentials, or a SCRAM password. From 8985a75f721b194205e5c42e464d623c0ecf045e Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Tue, 16 Jun 2026 16:27:55 -0700 Subject: [PATCH 3/8] Minor edit --- modules/sql/pages/get-started/deploy-sql-cluster.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sql/pages/get-started/deploy-sql-cluster.adoc b/modules/sql/pages/get-started/deploy-sql-cluster.adoc index 78d211ffc..cf738af66 100644 --- a/modules/sql/pages/get-started/deploy-sql-cluster.adoc +++ b/modules/sql/pages/get-started/deploy-sql-cluster.adoc @@ -14,7 +14,7 @@ After reading this page, you will be able to: * [ ] {learning-objective-2} * [ ] {learning-objective-3} -NOTE: Redpanda SQL is available on BYOC clusters running on AWS. For BYOVPC clusters, see xref:sql:get-started/enable-sql-byovpc-aws.adoc[]. +NOTE: For BYOVPC clusters, see xref:sql:get-started/enable-sql-byovpc-aws.adoc[]. == Prerequisites From 2dcb2e542cd1ed472ff330a42177d782c180b183 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 17 Jun 2026 09:05:21 -0700 Subject: [PATCH 4/8] Minor nav tree update --- modules/ROOT/nav.adoc | 4 ++-- modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index a126bb681..0e7136017 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -332,8 +332,8 @@ * xref:sql:index.adoc[Redpanda SQL] ** xref:sql:get-started/index.adoc[Get Started] *** xref:sql:get-started/sql-quickstart.adoc[Quickstart] -*** xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL] -*** xref:sql:get-started/enable-sql-byovpc-aws.adoc[Enable Redpanda SQL on a BYOVPC Cluster on AWS] +*** xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL on BYOC] +*** xref:sql:get-started/enable-sql-byovpc-aws.adoc[Enable Redpanda SQL on BYOVPC] *** xref:sql:get-started/overview.adoc[] **** xref:sql:get-started/oltp-vs-olap.adoc[] **** xref:sql:get-started/redpanda-sql-vs-postgresql.adoc[] diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc index ef0d76ab9..d9ab3187e 100644 --- a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc +++ b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc @@ -1,4 +1,4 @@ -= Enable Redpanda SQL on a BYOVPC Cluster += Enable Redpanda SQL on a BYOVPC Cluster on AWS :description: Enable the Redpanda SQL engine on an existing BYOVPC cluster on AWS by provisioning SQL-specific resources with Terraform and supplying them as customer-managed resources. :page-topic-type: how-to :personas: platform_admin From ccc2700af87163facad70814de59e29396c77473 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 17 Jun 2026 09:22:26 -0700 Subject: [PATCH 5/8] Add new cluster section --- .../get-started/enable-sql-byovpc-aws.adoc | 68 ++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc index d9ab3187e..c0a9a73aa 100644 --- a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc +++ b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc @@ -3,10 +3,10 @@ :page-topic-type: how-to :personas: platform_admin :learning-objective-1: Provision SQL-specific AWS resources using the Redpanda BYOVPC Terraform module -:learning-objective-2: Enable Redpanda SQL on a BYOVPC cluster by supplying customer-managed resources +:learning-objective-2: Enable Redpanda SQL on a new or existing BYOVPC cluster by supplying customer-managed resources :learning-objective-3: Verify that the SQL engine is running and ready to accept connections -Enable Redpanda SQL on a Bring Your Own Virtual Private Cloud (BYOVPC) cluster on AWS so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. +Enable Redpanda SQL on a Bring Your Own Virtual Private Cloud (BYOVPC) cluster on AWS so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. You can enable SQL when creating a new BYOVPC cluster or on an existing one. Unlike standard BYOC clusters, Redpanda does not create resources in your AWS account for BYOVPC clusters. You must provision the SQL-specific AWS resources yourself and supply them to the cluster as customer-managed resources before enabling the SQL engine. @@ -64,7 +64,69 @@ After applying, note the following module outputs. You supply these ARNs to the | `rpsql_security_group_arn` | ARN of the security group for SQL nodes |=== -== Supply SQL resources and enable SQL +== Enable SQL on a new cluster + +When creating a new BYOVPC cluster, you can enable SQL in the same request by including the SQL CMR ARNs and the `rpsql` configuration in the link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] payload alongside your existing BYOVPC CMRs. There is no Cloud Console path for this; BYOVPC cluster creation requires the Cloud API. + +[NOTE] +==== +Run the Terraform module with `enable_redpanda_sql = true` and collect the three SQL ARN outputs before creating the cluster. +==== + +. link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. +. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] request. Add the SQL CMR fields and `rpsql` block to your existing BYOVPC cluster spec: ++ +[,bash] +---- +curl -X POST "https://api.redpanda.com/v1/clusters" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "cluster": { + "name": "", + "type": "TYPE_BYOC", + "cloud_provider": "CLOUD_PROVIDER_AWS", + "region": "", + "zones": ["", "", ""], + "network_id": "", + "throughput_tier": "", + "resource_group_id": "", + "customer_managed_resources": { + "aws": { + "": "...", + "rpsql_node_group_instance_profile": {"arn": ""}, + "rpsql_cloud_storage_bucket": {"arn": ""}, + "rpsql_security_group": {"arn": ""} + } + }, + "rpsql": { + "enabled": true, + "replicas": , + "zones": [""] + } + } + }' +---- ++ +Replace the placeholders with your own values: ++ +* ``: Your cluster's existing BYOVPC customer-managed resource fields. See xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[Create a BYOVPC Cluster on AWS]. +* ``, ``, ``: The corresponding outputs from the Terraform module. +* ``: Initial number of SQL compute nodes (minimum 1, maximum 9). +* `` (optional): One of the cluster's glossterm:availability zones[]. The SQL engine deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. ++ +. The request returns the ID of a long-running operation. Poll the link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] endpoint until the operation completes: ++ +[,bash] +---- +curl -X GET "https://api.redpanda.com/v1/operations/{operation.id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" +---- ++ +When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. + +== Enable SQL on an existing cluster [tabs] ===== From a373878e27781b12fb26b1197b2f6011115815bd Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 17 Jun 2026 11:16:54 -0700 Subject: [PATCH 6/8] Clarify enable paths --- .../get-started/enable-sql-byovpc-aws.adoc | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc index c0a9a73aa..2396b68de 100644 --- a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc +++ b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc @@ -1,14 +1,16 @@ = Enable Redpanda SQL on a BYOVPC Cluster on AWS -:description: Enable the Redpanda SQL engine on an existing BYOVPC cluster on AWS by provisioning SQL-specific resources with Terraform and supplying them as customer-managed resources. +:description: Provision SQL-specific AWS resources with Terraform and enable the Redpanda SQL engine on a new or existing BYOVPC cluster on AWS. :page-topic-type: how-to :personas: platform_admin :learning-objective-1: Provision SQL-specific AWS resources using the Redpanda BYOVPC Terraform module :learning-objective-2: Enable Redpanda SQL on a new or existing BYOVPC cluster by supplying customer-managed resources :learning-objective-3: Verify that the SQL engine is running and ready to accept connections +include::shared:partial$feature-flag-rpcn.adoc[] + Enable Redpanda SQL on a Bring Your Own Virtual Private Cloud (BYOVPC) cluster on AWS so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. You can enable SQL when creating a new BYOVPC cluster or on an existing one. -Unlike standard BYOC clusters, Redpanda does not create resources in your AWS account for BYOVPC clusters. You must provision the SQL-specific AWS resources yourself and supply them to the cluster as customer-managed resources before enabling the SQL engine. +Unlike standard BYOC clusters, Redpanda does not create resources in your AWS account for BYOVPC clusters. You must provision the SQL-specific AWS resources yourself and supply them to the cluster as customer-managed resources (CMRs) before enabling the SQL engine. After reading this page, you will be able to: @@ -18,7 +20,7 @@ After reading this page, you will be able to: == Prerequisites -* An existing xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[BYOVPC cluster on AWS]. +* For a new cluster: a BYOVPC network on AWS. For an existing cluster: a deployed xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[BYOVPC cluster on AWS]. * The https://registry.terraform.io/modules/redpanda-data/redpanda-byovpc/aws/latest[Redpanda BYOVPC Terraform module^] and https://developer.hashicorp.com/terraform[Terraform^] version 1.8.5 or later. * Admin permissions in your Redpanda Cloud organization on xref:billing:billing.adoc[usage-based billing]. * If using the Cloud API, a valid glossterm:bearer token[]. See link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. @@ -54,7 +56,7 @@ This creates three SQL-specific resources in your AWS account: * An S3 bucket for SQL data storage (versioning disabled) * A security group for SQL nodes -After applying, note the following module outputs. You supply these ARNs to the cluster in the next step. +After applying, note the following module outputs. You supply these ARNs to the cluster when you enable SQL. [cols="1,3",options="header"] |=== @@ -64,17 +66,21 @@ After applying, note the following module outputs. You supply these ARNs to the | `rpsql_security_group_arn` | ARN of the security group for SQL nodes |=== -== Enable SQL on a new cluster +== Enable Redpanda SQL -When creating a new BYOVPC cluster, you can enable SQL in the same request by including the SQL CMR ARNs and the `rpsql` configuration in the link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] payload alongside your existing BYOVPC CMRs. There is no Cloud Console path for this; BYOVPC cluster creation requires the Cloud API. +=== On a new cluster [NOTE] ==== -Run the Terraform module with `enable_redpanda_sql = true` and collect the three SQL ARN outputs before creating the cluster. +BYOVPC cluster creation requires the Cloud API. + +If you created your BYOVPC cluster using the Redpanda Terraform provider (as described in xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[Create a BYOVPC Cluster on AWS]), your cluster already exists. Skip to <>. ==== +When creating a new BYOVPC cluster via the Cloud API, include the SQL customer-managed resource ARNs and the `rpsql` configuration in the link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] payload alongside your existing BYOVPC customer-managed resources. + . link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. -. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] request. Add the SQL CMR fields and `rpsql` block to your existing BYOVPC cluster spec: +. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] request. Add the SQL customer-managed resource fields and `rpsql` block to your BYOVPC cluster creation payload: + [,bash] ---- @@ -93,7 +99,7 @@ curl -X POST "https://api.redpanda.com/v1/clusters" \ "resource_group_id": "", "customer_managed_resources": { "aws": { - "": "...", + // ... existing BYOVPC CMR fields ... "rpsql_node_group_instance_profile": {"arn": ""}, "rpsql_cloud_storage_bucket": {"arn": ""}, "rpsql_security_group": {"arn": ""} @@ -126,7 +132,7 @@ curl -X GET "https://api.redpanda.com/v1/operations/{operation.id}" \ + When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. -== Enable SQL on an existing cluster +=== On an existing cluster [tabs] ===== @@ -207,7 +213,7 @@ To verify the SQL engine is running, use the connection details on the *SQL* tab Disabling Redpanda SQL tears down the SQL compute engine and clears its catalog state (catalog metadata, table mappings, and role/grant data). In-flight queries fail when SQL is disabled. ==== -Redpanda topic data, Schema Registry subjects, and any Iceberg-committed history for Iceberg-enabled topics are not affected. The Redpanda cluster itself continues to run normally; only the SQL engine and its associated state are removed. +If you disable Redpanda SQL, Redpanda topic data, Schema Registry subjects, and any Iceberg-committed history for Iceberg-enabled topics are not affected. The Redpanda cluster itself continues to run normally and only the SQL engine and its associated state are removed. Re-enabling SQL on the same cluster provisions a fresh engine: no prior catalog state, table mappings, or grants are restored. You must re-create catalogs, tables, and grants after re-enabling. From e91e97cf88863338ff691d526634b84d930bc69f Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Mon, 22 Jun 2026 13:31:12 -0700 Subject: [PATCH 7/8] Add customer managed resources in Terraform (https://github.com/redpanda-data/terraform-provider-redpanda/pull/353) --- .../get-started/enable-sql-byovpc-aws.adoc | 157 +++++++++++++++--- 1 file changed, 134 insertions(+), 23 deletions(-) diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc index 2396b68de..8180d9d69 100644 --- a/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc +++ b/modules/sql/pages/get-started/enable-sql-byovpc-aws.adoc @@ -25,11 +25,6 @@ After reading this page, you will be able to: * Admin permissions in your Redpanda Cloud organization on xref:billing:billing.adoc[usage-based billing]. * If using the Cloud API, a valid glossterm:bearer token[]. See link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. -[NOTE] -==== -The Redpanda Terraform provider does not support SQL customer-managed resource fields. Use the Cloud API or Cloud Console to supply these resources and enable SQL on a BYOVPC cluster. -==== - == Provision SQL resources with Terraform In your `terraform-aws-redpanda-byovpc` module configuration, set `enable_redpanda_sql = true`: @@ -56,7 +51,7 @@ This creates three SQL-specific resources in your AWS account: * An S3 bucket for SQL data storage (versioning disabled) * A security group for SQL nodes -After applying, note the following module outputs. You supply these ARNs to the cluster when you enable SQL. +After applying, Terraform outputs the following values. Supply these ARNs to the cluster when you enable SQL. [cols="1,3",options="header"] |=== @@ -68,19 +63,75 @@ After applying, note the following module outputs. You supply these ARNs to the == Enable Redpanda SQL +Use the Terraform provider, Cloud API, or Cloud Console to enable the SQL engine. The steps differ depending on whether you are creating a new cluster or updating an existing one. + === On a new cluster [NOTE] ==== -BYOVPC cluster creation requires the Cloud API. +To enable SQL when creating a new BYOVPC cluster, use the Terraform provider or the Cloud API. -If you created your BYOVPC cluster using the Redpanda Terraform provider (as described in xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[Create a BYOVPC Cluster on AWS]), your cluster already exists. Skip to <>. +If your cluster already exists, skip to <>. ==== +[IMPORTANT] +==== +The SQL customer-managed resources (`rpsql_node_group_instance_profile`, `rpsql_cloud_storage_bucket`, `rpsql_security_group`) are immutable while SQL is enabled. To change any of these resources, disable SQL first. +==== + +[tabs] +===== +Terraform:: ++ +-- +Add the SQL customer-managed resource fields and the `rpsql` block to the `redpanda_cluster` resource in your BYOVPC Terraform configuration. For the base `redpanda_cluster` configuration, see xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[Create a BYOVPC Cluster on AWS]. + +[,hcl] +---- +resource "redpanda_cluster" "cluster" { + # ... existing BYOVPC cluster configuration ... + customer_managed_resources = { + aws = { + # ... existing BYOVPC customer-managed resources ... + rpsql_node_group_instance_profile = { + arn = module.redpanda_byovpc.rpsql_node_group_instance_profile_arn + } + rpsql_cloud_storage_bucket = { + arn = module.redpanda_byovpc.rpsql_cloud_storage_bucket_arn + } + rpsql_security_group = { + arn = module.redpanda_byovpc.rpsql_security_group_arn + } + } + } + rpsql = { + enabled = true + replicas = 1 + zones = [""] + } +} +---- + +Replace the placeholders with your own values: + +* `replicas`: Initial number of SQL compute nodes (minimum 1, maximum 9). +* `` (optional): One of the cluster's glossterm:availability zones[]. The SQL engine deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. + +Apply the configuration: + +[,bash] +---- +terraform apply +---- +-- + +Cloud API:: ++ +-- When creating a new BYOVPC cluster via the Cloud API, include the SQL customer-managed resource ARNs and the `rpsql` configuration in the link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] payload alongside your existing BYOVPC customer-managed resources. . link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. -. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] request. Add the SQL customer-managed resource fields and `rpsql` block to your BYOVPC cluster creation payload: +. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] request. Include your existing BYOVPC customer-managed resource fields (see xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[Create a BYOVPC Cluster on AWS]) and add the SQL customer-managed resource fields and `rpsql` block: + [,bash] ---- @@ -99,7 +150,6 @@ curl -X POST "https://api.redpanda.com/v1/clusters" \ "resource_group_id": "", "customer_managed_resources": { "aws": { - // ... existing BYOVPC CMR fields ... "rpsql_node_group_instance_profile": {"arn": ""}, "rpsql_cloud_storage_bucket": {"arn": ""}, "rpsql_security_group": {"arn": ""} @@ -116,7 +166,6 @@ curl -X POST "https://api.redpanda.com/v1/clusters" \ + Replace the placeholders with your own values: + -* ``: Your cluster's existing BYOVPC customer-managed resource fields. See xref:get-started:cluster-types/byoc/aws/vpc-byo-aws.adoc[Create a BYOVPC Cluster on AWS]. * ``, ``, ``: The corresponding outputs from the Terraform module. * ``: Initial number of SQL compute nodes (minimum 1, maximum 9). * `` (optional): One of the cluster's glossterm:availability zones[]. The SQL engine deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. @@ -131,6 +180,8 @@ curl -X GET "https://api.redpanda.com/v1/operations/{operation.id}" \ ---- + When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. +-- +===== === On an existing cluster @@ -140,12 +191,56 @@ Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^] and open your BYOVPC cluster. -. From the left navigation, select *Dataplane settings*. -. On the *Cluster* tab, find the *RP SQL* row and click *Edit*. +. From the navigation menu, select *Dataplane settings*. +. On the *Cluster* tab, find the *Redpanda SQL* row and click *Edit*. . In the dialog, enter the ARNs from the Terraform module outputs, then use the *RPU* slider to set the compute size. . Click *Enable Redpanda SQL engine*. -- +Terraform:: ++ +-- +Add the SQL customer-managed resource fields and the `rpsql` block to the `redpanda_cluster` resource in your BYOVPC Terraform configuration: + +[,hcl] +---- +resource "redpanda_cluster" "cluster" { + # ... existing BYOVPC cluster configuration ... + customer_managed_resources = { + aws = { + # ... existing BYOVPC customer-managed resources ... + rpsql_node_group_instance_profile = { + arn = module.redpanda_byovpc.rpsql_node_group_instance_profile_arn + } + rpsql_cloud_storage_bucket = { + arn = module.redpanda_byovpc.rpsql_cloud_storage_bucket_arn + } + rpsql_security_group = { + arn = module.redpanda_byovpc.rpsql_security_group_arn + } + } + } + rpsql = { + enabled = true + replicas = 1 + zones = [""] + } +} +---- + +Replace the placeholders with your own values: + +* `replicas`: Initial number of SQL compute nodes (minimum 1, maximum 9). +* `` (optional): One of the cluster's glossterm:availability zones[]. The SQL engine deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. + +Apply the configuration: + +[,bash] +---- +terraform apply +---- +-- + Cloud API:: + -- @@ -193,18 +288,13 @@ When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. -- ===== -[IMPORTANT] -==== -The SQL customer-managed resources (`rpsql_node_group_instance_profile`, `rpsql_cloud_storage_bucket`, `rpsql_security_group`) are immutable while SQL is enabled. To change any of these resources, disable SQL first. -==== - == Verify the SQL engine is running After you enable Redpanda SQL, a *Redpanda SQL* tile appears in the cluster overview under *Products*. The tile shows the engine status, active sessions, queries per minute, and node count. When the *Nodes* indicator shows all nodes as ready (for example, *1/1*), the engine is provisioned and accepting connections. Provisioning can take up to 30 minutes. For the API flow, poll the long-running operation until it returns `STATE_COMPLETED`. -To verify the SQL engine is running, use the connection details on the *SQL* tab under *Connection details* to connect with a PostgreSQL client, such as `psql` (v16 or later required). You can also query data directly using the xref:sql:query-data/sql-editor.adoc[SQL editor] in the left navigation. +To verify the SQL engine is running, connect with `psql` version 16 or later, or another PostgreSQL client, using the connection details on the *SQL* tab under *Connection details*. You can also query data directly using the xref:sql:query-data/sql-editor.adoc[SQL editor] in the navigation menu. == Disable Redpanda SQL @@ -215,11 +305,11 @@ Disabling Redpanda SQL tears down the SQL compute engine and clears its catalog If you disable Redpanda SQL, Redpanda topic data, Schema Registry subjects, and any Iceberg-committed history for Iceberg-enabled topics are not affected. The Redpanda cluster itself continues to run normally and only the SQL engine and its associated state are removed. -Re-enabling SQL on the same cluster provisions a fresh engine: no prior catalog state, table mappings, or grants are restored. You must re-create catalogs, tables, and grants after re-enabling. +Re-enabling SQL on the same cluster provisions a fresh engine. Redpanda does not restore prior catalog state, table mappings, or grants. You must re-create catalogs, tables, and grants after re-enabling. [NOTE] ==== -The SQL-specific AWS resources (IAM instance profile, Amazon S3 bucket, and security group) are not deleted when you disable SQL. They remain in your account and can be reused if you re-enable SQL. To remove them, set `enable_redpanda_sql = false` in the Terraform module and run `terraform apply`, or run `terraform destroy` to remove all BYOVPC resources. +The SQL-specific AWS resources (IAM instance profile, S3 bucket, and security group) are not deleted when you disable SQL. They remain in your account and can be reused if you re-enable SQL. To remove them, set `enable_redpanda_sql = false` in the Terraform module and run `terraform apply`, or run `terraform destroy` to remove all BYOVPC resources. ==== [tabs] @@ -228,11 +318,32 @@ Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^] and open your cluster. -. From the left navigation, select *Dataplane settings*. -. On the *Cluster* tab, find the *RP SQL* row and click *Edit*. +. From the navigation menu, select *Dataplane settings*. +. On the *Cluster* tab, find the *Redpanda SQL* row and click *Edit*. . In the *Edit Redpanda SQL engine* dialog, click *Disable*. -- +Terraform:: ++ +-- +Set `rpsql.enabled` to `false` in the `redpanda_cluster` resource and apply: + +[,hcl] +---- +resource "redpanda_cluster" "cluster" { + # ... existing BYOVPC cluster configuration ... + rpsql = { + enabled = false + } +} +---- + +[,bash] +---- +terraform apply +---- +-- + Cloud API:: + -- From b98fe30c0eaf1eef99730a34525f8fd01309f10d Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Mon, 22 Jun 2026 14:10:18 -0700 Subject: [PATCH 8/8] Add combined what's new entry for DOC-2227 and DOC-2228 --- modules/get-started/pages/whats-new-cloud.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/get-started/pages/whats-new-cloud.adoc b/modules/get-started/pages/whats-new-cloud.adoc index ccedee6ac..0be6adab4 100644 --- a/modules/get-started/pages/whats-new-cloud.adoc +++ b/modules/get-started/pages/whats-new-cloud.adoc @@ -8,6 +8,11 @@ This page lists new features added to Redpanda Cloud. == June 2026 +=== Redpanda SQL updates + +* Redpanda SQL is now available on BYOVPC clusters on AWS. Provision the required SQL-specific AWS resources using the Redpanda BYOVPC Terraform module, then supply them as customer-managed resources when enabling the SQL engine via the Cloud Console, Cloud API, or Terraform provider. See xref:sql:get-started/enable-sql-byovpc-aws.adoc[Enable Redpanda SQL on a BYOVPC Cluster on AWS]. +* You can now enable and manage Redpanda SQL on BYOC clusters using the Redpanda Terraform provider (version 2.0.0 or later). Configure the `rpsql` block on a `redpanda_cluster` resource to enable SQL, set the replica count, and specify an availability zone. See xref:manage:terraform-provider.adoc#enable-redpanda-sql-on-a-byoc-cluster[Redpanda Terraform Provider]. + === GCP Lakehouse catalog for Iceberg topics BYOC clusters on GCP can now use GCP Lakehouse as an Iceberg REST catalog. See xref:manage:iceberg/iceberg-topics-gcp-biglake.adoc[].