From 1c878ceb755b4fabd3664c289a74f05f47d1bafd Mon Sep 17 00:00:00 2001 From: lentil32 Date: Thu, 25 Jun 2026 16:22:31 +0900 Subject: [PATCH] feat(data-sources): add Cloudflare R2 SQL source --- .../src/components/provider-icons.tsx | 1 + .../docs/docs/integrations/databases.mdx | 2 +- .../docs/docs/integrations/warehouses.mdx | 3 +- .../docs/docs/reference/source-providers.mdx | 18 +- apps/landing/src/shared/icons/brands.tsx | 1 + .../service/source/credentials.test.ts | 23 ++ packages/cli-server/src/source/model.test.ts | 4 + .../__snapshots__/credentials.test.ts.snap | 1 + packages/db/src/connection-guide.ts | 19 ++ packages/db/src/credentials.test.ts | 72 +++++ packages/db/src/credentials.ts | 16 + .../__snapshots__/data-sources.test.ts.snap | 2 + packages/db/src/source-providers.ts | 32 ++ .../data-source-query/core/registry.ts | 2 + .../data-source-query/execute-query.test.ts | 78 +++++ .../data-source-query/execute-query.ts | 1 + .../providers/cloudflare-r2-sql/driver.ts | 300 ++++++++++++++++++ .../data-source-query/validate-sql.test.ts | 16 + .../data-source-query/validate-sql.ts | 1 + .../connection-guides/cloudflare-r2-sql.ts | 113 +++++++ .../data-sources/connection-guides/index.ts | 2 + .../data-sources/connection-guides/types.ts | 1 + 22 files changed, 697 insertions(+), 11 deletions(-) create mode 100644 packages/server/src/services/data-source-query/providers/cloudflare-r2-sql/driver.ts create mode 100644 packages/ui/src/data-sources/connection-guides/cloudflare-r2-sql.ts diff --git a/apps/dashboard/src/components/provider-icons.tsx b/apps/dashboard/src/components/provider-icons.tsx index 2c9faa67..c7de68a7 100644 --- a/apps/dashboard/src/components/provider-icons.tsx +++ b/apps/dashboard/src/components/provider-icons.tsx @@ -314,6 +314,7 @@ export const ProviderIcons = { bigquery: createSimpleProviderIcon(siGooglebigquery), cal: createSimpleProviderIcon(siCaldotcom), cloudflare_d1: CloudflareD1Icon, + cloudflare_r2_sql: createSimpleProviderIcon(siCloudflareworkers), cloudflare_web_analytics: createSimpleProviderIcon(siCloudflareworkers), cloudflare_workers_observability: createSimpleProviderIcon(siCloudflareworkers), diff --git a/apps/landing/src/content/docs/docs/integrations/databases.mdx b/apps/landing/src/content/docs/docs/integrations/databases.mdx index 716120a1..896132fe 100644 --- a/apps/landing/src/content/docs/docs/integrations/databases.mdx +++ b/apps/landing/src/content/docs/docs/integrations/databases.mdx @@ -42,4 +42,4 @@ Use the session pooler host from the Supabase Connect panel. Keep SSL enabled an - Limit schemas and tables at the database permission layer. - Use explicit columns and `limit` clauses in agent queries. -Cloudflare D1 is covered with warehouse-style SQL sources because its provider ID, `cloudflare_d1`, is grouped with OneQuery warehouse providers. +Cloudflare D1 is covered with warehouse-style SQL sources because its provider ID, `cloudflare_d1`, is grouped with OneQuery warehouse providers. Cloudflare R2 SQL uses the separate `cloudflare_r2_sql` warehouse provider for querying R2 Data Catalog Iceberg tables. diff --git a/apps/landing/src/content/docs/docs/integrations/warehouses.mdx b/apps/landing/src/content/docs/docs/integrations/warehouses.mdx index f1555a87..775c6775 100644 --- a/apps/landing/src/content/docs/docs/integrations/warehouses.mdx +++ b/apps/landing/src/content/docs/docs/integrations/warehouses.mdx @@ -1,6 +1,6 @@ --- title: Warehouses -description: Connect Snowflake, BigQuery, Athena, MotherDuck, and related warehouse systems to OneQuery. +description: Connect Snowflake, BigQuery, Athena, MotherDuck, Cloudflare R2 SQL, and related warehouse systems to OneQuery. sidebar: order: 3 --- @@ -16,6 +16,7 @@ Warehouse sources are useful for analytics, operations review, and agent-assiste | AWS Athena Connector | `aws_athena_connector` | Uses a registered OneQuery connector and Athena database details. | | MotherDuck | `motherduck` | PostgreSQL wire protocol endpoint with service token. | | Cloudflare D1 | `cloudflare_d1` | Serverless SQL source for edge application data. | +| Cloudflare R2 SQL | `cloudflare_r2_sql` | Serverless analytics SQL over R2 Data Catalog Iceberg tables. | ## Snowflake Setup Notes diff --git a/apps/landing/src/content/docs/docs/reference/source-providers.mdx b/apps/landing/src/content/docs/docs/reference/source-providers.mdx index 054d35ac..b5997a06 100644 --- a/apps/landing/src/content/docs/docs/reference/source-providers.mdx +++ b/apps/landing/src/content/docs/docs/reference/source-providers.mdx @@ -9,14 +9,14 @@ Use the provider ID with `onequery source connect --source `. The s ## Provider Families -| Family | Provider IDs | -| ------------------ | ----------------------------------------------------------------------------------------- | -| Databases | `postgres`, `supabase`, `mysql`, `mongodb` | -| Warehouses | `snowflake`, `bigquery`, `aws_athena_connector`, `motherduck`, `cloudflare_d1` | -| Observability | `sentry`, `laminar`, `cloudflare_workers_observability` | -| Developer workflow | `github`, `linear`, `jira`, `confluence`, `vercel`, `discord`, `onepassword` | -| Product analytics | `ga`, `amplitude`, `mixpanel`, `posthog`, `microsoft_clarity`, `cloudflare_web_analytics` | -| Marketing | `google_search_console`, `amazon_ads`, `linkedin_ads`, `tiktok_marketing`, `sendgrid` | -| Productivity | `airtable`, `cal`, `granola` | +| Family | Provider IDs | +| ------------------ | --------------------------------------------------------------------------------------------------- | +| Databases | `postgres`, `supabase`, `mysql`, `mongodb` | +| Warehouses | `snowflake`, `bigquery`, `aws_athena_connector`, `motherduck`, `cloudflare_d1`, `cloudflare_r2_sql` | +| Observability | `sentry`, `laminar`, `cloudflare_workers_observability` | +| Developer workflow | `github`, `linear`, `jira`, `confluence`, `vercel`, `discord`, `onepassword` | +| Product analytics | `ga`, `amplitude`, `mixpanel`, `posthog`, `microsoft_clarity`, `cloudflare_web_analytics` | +| Marketing | `google_search_console`, `amazon_ads`, `linkedin_ads`, `tiktok_marketing`, `sendgrid` | +| Productivity | `airtable`, `cal`, `granola` | For the live list used by the site, see [Connectors](/connectors/). diff --git a/apps/landing/src/shared/icons/brands.tsx b/apps/landing/src/shared/icons/brands.tsx index fc99a77b..9f118e0e 100644 --- a/apps/landing/src/shared/icons/brands.tsx +++ b/apps/landing/src/shared/icons/brands.tsx @@ -281,6 +281,7 @@ const BRAND_ICONS = { bun: createSimpleBrandIcon(siBun), cal: createSimpleBrandIcon(siCaldotcom), cloudflare_d1: CloudflareD1Icon, + cloudflare_r2_sql: createSimpleBrandIcon(siCloudflareworkers), cloudflare_web_analytics: createSimpleBrandIcon(siCloudflareworkers), cloudflare_workers_observability: createSimpleBrandIcon(siCloudflareworkers), confluence: createSimpleBrandIcon(siConfluence), diff --git a/packages/cli-server/src/connect/service/source/credentials.test.ts b/packages/cli-server/src/connect/service/source/credentials.test.ts index a9936e64..7b8905a7 100644 --- a/packages/cli-server/src/connect/service/source/credentials.test.ts +++ b/packages/cli-server/src/connect/service/source/credentials.test.ts @@ -173,6 +173,29 @@ describe("parseConnectSourceCredentials", () => { }); }); + it("injects the Cloudflare R2 SQL credential type", () => { + const result = parseConnectSourceCredentials("cloudflare_r2_sql", { + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + apiToken: "cf_r2_sql_token", + bucketName: "analytics-events", + }); + + expect(result.isOk()).toBe(true); + if (result.isErr()) { + throw result.error; + } + + expect(result.value).toEqual({ + provider: "cloudflare_r2_sql", + credentials: { + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + apiToken: "cf_r2_sql_token", + bucketName: "analytics-events", + type: "cloudflare_r2_sql", + }, + }); + }); + it("infers Google Analytics service-account auth mode from guide-shaped credentials", () => { const result = parseConnectSourceCredentials("ga", { propertyId: "123456789", diff --git a/packages/cli-server/src/source/model.test.ts b/packages/cli-server/src/source/model.test.ts index db3b0667..5d88856d 100644 --- a/packages/cli-server/src/source/model.test.ts +++ b/packages/cli-server/src/source/model.test.ts @@ -51,6 +51,10 @@ describe("source model", () => { expect(getCliSourceInterfaceTypes("cloudflare_d1")).toEqual(["query"]); }); + it("exposes Cloudflare R2 SQL as query capable", () => { + expect(getCliSourceInterfaceTypes("cloudflare_r2_sql")).toEqual(["query"]); + }); + it("keeps provider interfaces visible when a source is unhealthy", () => { expect(getCliSourceInterfaceTypes("postgres")).toEqual(["query"]); }); diff --git a/packages/db/src/__snapshots__/credentials.test.ts.snap b/packages/db/src/__snapshots__/credentials.test.ts.snap index 4ef3f575..d2b209d3 100644 --- a/packages/db/src/__snapshots__/credentials.test.ts.snap +++ b/packages/db/src/__snapshots__/credentials.test.ts.snap @@ -9,6 +9,7 @@ exports[`credentials schemas > credentialSchemaMap > matches supported provider "bigquery", "cal", "cloudflare_d1", + "cloudflare_r2_sql", "cloudflare_web_analytics", "cloudflare_workers_observability", "confluence", diff --git a/packages/db/src/connection-guide.ts b/packages/db/src/connection-guide.ts index 2f6f2d2f..cdb29084 100644 --- a/packages/db/src/connection-guide.ts +++ b/packages/db/src/connection-guide.ts @@ -128,6 +128,25 @@ export const SOURCE_CONNECT_PROVIDER_GUIDES: SourceConnectProviderGuide[] = [ }, }, }, + { + provider: "cloudflare_r2_sql", + summary: + "Connect Cloudflare R2 SQL with an account ID, R2 bucket name, and API token for querying R2 Data Catalog Iceberg tables.", + steps: [ + "Enable R2 Data Catalog for the bucket that owns the Iceberg tables OneQuery should query.", + "Copy the Cloudflare Account ID and the R2 bucket name from the Cloudflare dashboard or Wrangler.", + "Create a Cloudflare API token with R2 SQL read access plus R2 Data Catalog and R2 storage permissions for the target bucket.", + "Only include `apiBaseUrl` when you need a non-default R2 SQL API origin.", + ], + exampleInput: { + sourceKey: "cloudflare_r2_sql_prod", + credentials: { + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + bucketName: "analytics-events", + apiToken: "cloudflare_r2_sql_token", + }, + }, + }, { provider: "laminar", summary: diff --git a/packages/db/src/credentials.test.ts b/packages/db/src/credentials.test.ts index d13235e4..dd96f731 100644 --- a/packages/db/src/credentials.test.ts +++ b/packages/db/src/credentials.test.ts @@ -7,6 +7,7 @@ import { BigQueryCredentialsSchema, CalCredentialsSchema, CloudflareD1CredentialsSchema, + CloudflareR2SqlCredentialsSchema, CloudflareWebAnalyticsCredentialsSchema, ConfluenceCredentialsSchema, ConnectorCredentialsSchema, @@ -56,6 +57,7 @@ import type { BigQueryCredentials, CalCredentials, CloudflareD1Credentials, + CloudflareR2SqlCredentials, CloudflareWebAnalyticsCredentials, ConfluenceCredentials, ConnectorCredentials, @@ -589,6 +591,51 @@ describe("credentials schemas", () => { }); }); + describe("CloudflareR2SqlCredentialsSchema", () => { + it("should validate valid Cloudflare R2 SQL credentials", () => { + const credentials: CloudflareR2SqlCredentials = { + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + apiToken: "cf_r2_sql_token", + bucketName: "analytics-events", + type: "cloudflare_r2_sql", + }; + + const result = CloudflareR2SqlCredentialsSchema.safeParse(credentials); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).toEqual(credentials); + } + }); + + it("should accept optional apiBaseUrl", () => { + const credentials: CloudflareR2SqlCredentials = { + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + apiBaseUrl: "https://api.sql.cloudflarestorage.com/api/v1", + apiToken: "cf_r2_sql_token", + bucketName: "analytics-events", + type: "cloudflare_r2_sql", + }; + + const result = CloudflareR2SqlCredentialsSchema.safeParse(credentials); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.apiBaseUrl).toBe( + "https://api.sql.cloudflarestorage.com/api/v1" + ); + } + }); + + it("should reject missing bucketName", () => { + const result = CloudflareR2SqlCredentialsSchema.safeParse({ + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + apiToken: "cf_r2_sql_token", + type: "cloudflare_r2_sql", + }); + + expect(result.success).toBe(false); + }); + }); + describe("LaminarCredentialsSchema", () => { it("should validate valid Laminar credentials", () => { const credentials: LaminarCredentials = { @@ -2022,6 +2069,12 @@ describe("credentials schemas", () => { ); }); + it("should map cloudflare_r2_sql to CloudflareR2SqlCredentialsSchema", () => { + expect(credentialSchemaMap.cloudflare_r2_sql).toBe( + CloudflareR2SqlCredentialsSchema + ); + }); + it("should map laminar to LaminarCredentialsSchema", () => { expect(credentialSchemaMap.laminar).toBe(LaminarCredentialsSchema); }); @@ -2289,6 +2342,18 @@ describe("credentials schemas", () => { expect(result.type).toBe("cloudflare_d1"); }); + it("should validate Cloudflare R2 SQL credentials", () => { + const credentials = { + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + apiToken: "cf_r2_sql_token", + bucketName: "analytics-events", + type: "cloudflare_r2_sql", + }; + + const result = validateCredentials(credentials); + expect(result.type).toBe("cloudflare_r2_sql"); + }); + it("should validate Sentry credentials", () => { const credentials = { authToken: "sntrys_123", @@ -2448,6 +2513,12 @@ describe("credentials schemas", () => { databaseId: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", type: "cloudflare_d1", }, + { + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + apiToken: "cf_r2_sql_token", + bucketName: "analytics-events", + type: "cloudflare_r2_sql", + }, { apiKey: "lmnr_project_key_123", type: "laminar", @@ -2533,6 +2604,7 @@ describe("credentials schemas", () => { "bigquery", "aws_athena_connector", "cloudflare_d1", + "cloudflare_r2_sql", "laminar", "motherduck", "mysql", diff --git a/packages/db/src/credentials.ts b/packages/db/src/credentials.ts index bb41375c..f1f3f2c3 100644 --- a/packages/db/src/credentials.ts +++ b/packages/db/src/credentials.ts @@ -520,6 +520,18 @@ export type CloudflareD1Credentials = z.infer< typeof CloudflareD1CredentialsSchema >; +export const CloudflareR2SqlCredentialsSchema = z.object({ + accountId: trimmedString("Account ID is required"), + apiBaseUrl: optionalTrimmedUrl("API base URL must be a valid URL"), + apiToken: requiredOpaqueString("API token is required"), + bucketName: trimmedString("Bucket name is required"), + type: z.literal("cloudflare_r2_sql"), +}); + +export type CloudflareR2SqlCredentials = z.infer< + typeof CloudflareR2SqlCredentialsSchema +>; + export const CloudflareWorkersObservabilityCredentialsSchema = z.object({ accountId: trimmedString("Account ID is required"), apiBaseUrl: optionalTrimmedUrl("API base URL must be a valid URL"), @@ -603,6 +615,7 @@ export const CredentialsSchema = z.union([ MicrosoftClarityCredentialsSchema, OnePasswordCredentialsSchema, CloudflareD1CredentialsSchema, + CloudflareR2SqlCredentialsSchema, CloudflareWorkersObservabilityCredentialsSchema, CloudflareWebAnalyticsCredentialsSchema, LinearCredentialsSchema, @@ -615,6 +628,7 @@ export type CredentialProviderType = Credentials["type"]; export const DATABASE_CREDENTIAL_PROVIDER = { BIGQUERY: "bigquery", CLOUDFLARE_D1: "cloudflare_d1", + CLOUDFLARE_R2_SQL: "cloudflare_r2_sql", CONNECTOR: "aws_athena_connector", LAMINAR: "laminar", MOTHERDUCK: "motherduck", @@ -632,6 +646,7 @@ export const DATABASE_CREDENTIAL_PROVIDER_TYPES = [ DATABASE_CREDENTIAL_PROVIDER.SNOWFLAKE, DATABASE_CREDENTIAL_PROVIDER.BIGQUERY, DATABASE_CREDENTIAL_PROVIDER.CLOUDFLARE_D1, + DATABASE_CREDENTIAL_PROVIDER.CLOUDFLARE_R2_SQL, DATABASE_CREDENTIAL_PROVIDER.LAMINAR, DATABASE_CREDENTIAL_PROVIDER.MOTHERDUCK, DATABASE_CREDENTIAL_PROVIDER.CONNECTOR, @@ -650,6 +665,7 @@ export const credentialSchemaMap = { bigquery: BigQueryCredentialsSchema, cal: CalCredentialsSchema, cloudflare_d1: CloudflareD1CredentialsSchema, + cloudflare_r2_sql: CloudflareR2SqlCredentialsSchema, confluence: ConfluenceCredentialsSchema, discord: DiscordCredentialsSchema, e2b: E2BCredentialsSchema, diff --git a/packages/db/src/schema/__snapshots__/data-sources.test.ts.snap b/packages/db/src/schema/__snapshots__/data-sources.test.ts.snap index 1d94e7c8..331e2536 100644 --- a/packages/db/src/schema/__snapshots__/data-sources.test.ts.snap +++ b/packages/db/src/schema/__snapshots__/data-sources.test.ts.snap @@ -10,6 +10,7 @@ exports[`data-sources schema > matches provider type snapshots 1`] = ` "mongodb", "bigquery", "cloudflare_d1", + "cloudflare_r2_sql", "laminar", "motherduck", "aws_athena_connector", @@ -48,6 +49,7 @@ exports[`data-sources schema > matches provider type snapshots 1`] = ` "mongodb", "bigquery", "cloudflare_d1", + "cloudflare_r2_sql", "laminar", "motherduck", "aws_athena_connector", diff --git a/packages/db/src/source-providers.ts b/packages/db/src/source-providers.ts index 079c6f11..4f2f3e18 100644 --- a/packages/db/src/source-providers.ts +++ b/packages/db/src/source-providers.ts @@ -7,6 +7,7 @@ import { BigQueryCredentialsSchema, CalCredentialsSchema, CloudflareD1CredentialsSchema, + CloudflareR2SqlCredentialsSchema, CloudflareWebAnalyticsCredentialsSchema, CloudflareWorkersObservabilityCredentialsSchema, ConfluenceCredentialsSchema, @@ -299,6 +300,37 @@ export const SOURCE_PROVIDER_REGISTRY = { }, }, }, + cloudflare_r2_sql: { + label: "Cloudflare R2 SQL", + credentialSchema: CloudflareR2SqlCredentialsSchema, + credentialType: "cloudflare_r2_sql", + connectable: true, + analysisSource: true, + queryInterface: true, + sourceApiInterface: false, + testable: true, + dashboardConnectable: true, + dashboardCredentialForm: "json", + publicCategory: "Warehouses", + guide: { + summary: + "Connect Cloudflare R2 SQL with an account ID, R2 bucket name, and API token for querying R2 Data Catalog Iceberg tables.", + steps: [ + "Enable R2 Data Catalog for the bucket that owns the Iceberg tables OneQuery should query.", + "Copy the Cloudflare Account ID and the R2 bucket name from the Cloudflare dashboard or Wrangler.", + "Create a Cloudflare API token with R2 SQL read access plus R2 Data Catalog and R2 storage permissions for the target bucket.", + "Only include `apiBaseUrl` when you need a non-default R2 SQL API origin.", + ], + exampleInput: { + sourceKey: "cloudflare_r2_sql_prod", + credentials: { + accountId: "023e105f4ecef8ad9ca31a8372d0c353", + bucketName: "analytics-events", + apiToken: "cloudflare_r2_sql_token", + }, + }, + }, + }, laminar: { label: "Laminar", credentialSchema: LaminarCredentialsSchema, diff --git a/packages/server/src/services/data-source-query/core/registry.ts b/packages/server/src/services/data-source-query/core/registry.ts index 121eb110..6542f5df 100644 --- a/packages/server/src/services/data-source-query/core/registry.ts +++ b/packages/server/src/services/data-source-query/core/registry.ts @@ -1,6 +1,7 @@ import { athenaConnectorQueryDriver } from "../providers/athena-connector/driver"; import { bigQueryQueryDriver } from "../providers/bigquery/driver"; import { cloudflareD1QueryDriver } from "../providers/cloudflare-d1/driver"; +import { cloudflareR2SqlQueryDriver } from "../providers/cloudflare-r2-sql/driver"; import { laminarQueryDriver } from "../providers/laminar/driver"; import { motherDuckQueryDriver } from "../providers/motherduck/driver"; import { mysqlQueryDriver } from "../providers/mysql/driver"; @@ -12,6 +13,7 @@ export const queryDriverRegistry = { aws_athena_connector: athenaConnectorQueryDriver, bigquery: bigQueryQueryDriver, cloudflare_d1: cloudflareD1QueryDriver, + cloudflare_r2_sql: cloudflareR2SqlQueryDriver, laminar: laminarQueryDriver, motherduck: motherDuckQueryDriver, mysql: mysqlQueryDriver, diff --git a/packages/server/src/services/data-source-query/execute-query.test.ts b/packages/server/src/services/data-source-query/execute-query.test.ts index f88368c6..db0671ee 100644 --- a/packages/server/src/services/data-source-query/execute-query.test.ts +++ b/packages/server/src/services/data-source-query/execute-query.test.ts @@ -9,6 +9,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { executeBigQueryQuery, executeCloudflareD1Query, + executeCloudflareR2SqlQuery, executeDatabaseQuery, executeMotherDuckQuery, executeSnowflakeQuery, @@ -244,6 +245,83 @@ describe("data source query execution", () => { } }); + it("executes Cloudflare R2 SQL queries through the R2 SQL REST API", async () => { + const fetchSpy = vi.fn(async (_url: string | URL, _init?: RequestInit) => ({ + json: async () => ({ + errors: [], + messages: [], + result: { + metrics: { + bytes_scanned: 1024, + files_scanned: 2, + r2_requests_count: 3, + }, + rows: [{ one: 1 }], + schema: [{ name: "one", type: "Int64" }], + }, + success: true, + }), + ok: true, + status: 200, + text: async () => "", + })); + globalThis.fetch = fetchSpy as unknown as typeof fetch; + + const rows = unwrapQueryResult( + await executeCloudflareR2SqlQuery( + { + accountId: "acct_123", + apiToken: "cf-r2-sql-token", + bucketName: "analytics-events", + type: "cloudflare_r2_sql", + }, + "SELECT 1" + ) + ); + + expect(rows).toEqual([{ one: 1 }]); + expect(fetchSpy).toHaveBeenCalledTimes(1); + const [url, init] = fetchSpy.mock.calls[0] ?? []; + expect(String(url)).toBe( + "https://api.sql.cloudflarestorage.com/api/v1/accounts/acct_123/r2-sql/query/analytics-events" + ); + expect(init?.method).toBe("POST"); + expect(init?.headers).toMatchObject({ + Authorization: "Bearer cf-r2-sql-token", + "Content-Type": "application/json", + }); + expect(JSON.parse(init?.body as string)).toEqual({ + query: "SELECT 1", + warehouse: "acct_123_analytics-events", + }); + }); + + it("sanitizes Cloudflare R2 SQL error text", async () => { + const fetchSpy = vi.fn(async (_url: string | URL, _init?: RequestInit) => ({ + json: async () => ({}), + ok: false, + status: 403, + text: async () => "Bearer cf-r2-sql-token cannot access cf-r2-sql-token", + })); + globalThis.fetch = fetchSpy as unknown as typeof fetch; + + const result = await executeCloudflareR2SqlQuery( + { + accountId: "acct_123", + apiToken: "cf-r2-sql-token", + bucketName: "analytics-events", + type: "cloudflare_r2_sql", + }, + "SELECT 1" + ); + expect(result.isErr()).toBe(true); + if (result.isErr()) { + expect(result.error.message).toBe( + "Cloudflare R2 SQL query failed: 403 Bearer [REDACTED] cannot access ***" + ); + } + }); + it("executes Snowflake queries with the created connection", async () => { const statement = { cancel: vi.fn(), diff --git a/packages/server/src/services/data-source-query/execute-query.ts b/packages/server/src/services/data-source-query/execute-query.ts index 598f9ccb..12e3d5d5 100644 --- a/packages/server/src/services/data-source-query/execute-query.ts +++ b/packages/server/src/services/data-source-query/execute-query.ts @@ -48,6 +48,7 @@ export { executeBigQueryQueryWithStats, } from "./providers/bigquery/driver"; export { executeCloudflareD1Query } from "./providers/cloudflare-d1/driver"; +export { executeCloudflareR2SqlQuery } from "./providers/cloudflare-r2-sql/driver"; export { executeLaminarQuery } from "./providers/laminar/driver"; export { executeMotherDuckQuery } from "./providers/motherduck/driver"; export { executeMySQLQuery } from "./providers/mysql/driver"; diff --git a/packages/server/src/services/data-source-query/providers/cloudflare-r2-sql/driver.ts b/packages/server/src/services/data-source-query/providers/cloudflare-r2-sql/driver.ts new file mode 100644 index 00000000..d358be59 --- /dev/null +++ b/packages/server/src/services/data-source-query/providers/cloudflare-r2-sql/driver.ts @@ -0,0 +1,300 @@ +import { isRecord } from "@onequery/base"; +import type { CloudflareR2SqlCredentials } from "@onequery/db/server"; +import { Result } from "better-result"; + +import { + createFailedConnectionTest, + runProviderConnectionTest, +} from "../../core/connection-test"; +import type { ProviderQueryDriver } from "../../core/driver"; +import type { QueryErrorClassification } from "../../core/errors"; +import { + ProviderResponseFailure, + QueryInputFailure, + toErrorMessage, + toQueryFailure, +} from "../../core/errors"; +import { normalizeRecordRows } from "../../core/rows"; +import { + hasControlCharacters, + sanitizeProviderErrorText, +} from "../../core/security"; +import { QUERY_TIMEOUT_MS, createQueryDeadline } from "../../core/timeout"; +import type { QueryDeadline } from "../../core/timeout"; +import type { DatabaseQueryResult } from "../../core/types"; +import { validateReadOnlySql } from "../../core/validation"; + +const DEFAULT_CLOUDFLARE_R2_SQL_API_BASE_URL = + "https://api.sql.cloudflarestorage.com/api/v1"; +const TRANSIENT_CLOUDFLARE_R2_SQL_STATUS_CODES = new Set([ + 429, 500, 502, 503, 504, +]); +const CONNECTION_TEST_QUERY = "SELECT 1 AS onequery_connection_test"; + +export async function executeCloudflareR2SqlQuery( + creds: CloudflareR2SqlCredentials, + query: string, + timeoutMs = QUERY_TIMEOUT_MS +): Promise[]>> { + return Result.gen(async function* executeCloudflareR2SqlQueryFlow() { + const apiToken = yield* normalizeCloudflareR2SqlApiToken(creds.apiToken); + const response = yield* Result.await( + Result.tryPromise({ + try: () => + fetch(resolveCloudflareR2SqlQueryUrl(creds), { + body: JSON.stringify({ + query, + warehouse: resolveCloudflareR2SqlWarehouseName(creds), + }), + headers: { + Authorization: `Bearer ${apiToken}`, + "Content-Type": "application/json", + }, + method: "POST", + signal: createQueryDeadline(timeoutMs).createAbortSignal(), + }), + catch: (error) => + toQueryFailure({ + classifier: classifyCloudflareR2SqlError, + error, + provider: "cloudflare_r2_sql", + }), + }) + ); + + if (!response.ok) { + const errorText = await response.text().catch(() => "Unknown error"); + return Result.err( + new ProviderResponseFailure({ + message: `Cloudflare R2 SQL query failed: ${response.status} ${sanitizeCloudflareR2SqlErrorText(errorText, apiToken)}`, + provider: "cloudflare_r2_sql", + retryable: TRANSIENT_CLOUDFLARE_R2_SQL_STATUS_CODES.has( + response.status + ), + timedOut: response.status === 504, + }) + ); + } + + const payload = yield* Result.await( + Result.tryPromise({ + try: () => response.json(), + catch: (cause) => + new ProviderResponseFailure({ + cause, + message: `Cloudflare R2 SQL query returned invalid JSON: ${toErrorMessage(cause)}`, + provider: "cloudflare_r2_sql", + retryable: false, + timedOut: false, + }), + }) + ); + const rows = yield* extractCloudflareR2SqlRows(payload, apiToken); + return Result.ok(rows); + }); +} + +export const cloudflareR2SqlQueryDriver = { + provider: "cloudflare_r2_sql", + capabilities: { + cancellation: "best_effort", + connectionTest: true, + dryRun: false, + stats: false, + }, + validateSql: async ({ sql }) => + validateReadOnlySql({ + provider: "cloudflare_r2_sql", + sql, + }), + execute: async ({ credentials, deadline, sql }) => + ( + await executeCloudflareR2SqlQuery(credentials, sql, deadline.timeoutMs) + ).map((rows) => ({ rows })), + classifyError: classifyCloudflareR2SqlError, + testConnection: async ({ credentials, deadline }) => + runCloudflareR2SqlConnectionTest(credentials, deadline), +} satisfies ProviderQueryDriver; + +function resolveCloudflareR2SqlQueryUrl( + credentials: CloudflareR2SqlCredentials +) { + const trimmedBaseUrl = credentials.apiBaseUrl?.trim() ?? ""; + const configuredBaseUrl = + trimmedBaseUrl.length > 0 + ? trimmedBaseUrl + : DEFAULT_CLOUDFLARE_R2_SQL_API_BASE_URL; + const baseUrl = configuredBaseUrl.replace(/\/+$/, ""); + const accountId = encodeURIComponent(credentials.accountId.trim()); + const bucketName = encodeURIComponent(credentials.bucketName.trim()); + + return `${baseUrl}/accounts/${accountId}/r2-sql/query/${bucketName}`; +} + +function resolveCloudflareR2SqlWarehouseName( + credentials: CloudflareR2SqlCredentials +) { + return `${credentials.accountId.trim()}_${credentials.bucketName.trim()}`; +} + +function normalizeCloudflareR2SqlApiToken( + apiToken: string +): DatabaseQueryResult { + const normalized = apiToken.trim(); + if (normalized.length === 0 || hasControlCharacters(normalized)) { + return Result.err( + new QueryInputFailure({ + message: "Cloudflare R2 SQL API token is required", + provider: "cloudflare_r2_sql", + }) + ); + } + return Result.ok(normalized); +} + +function extractCloudflareR2SqlRows( + payload: unknown, + apiToken: string +): DatabaseQueryResult[]> { + if (!isRecord(payload)) { + return Result.err( + new ProviderResponseFailure({ + message: "Cloudflare R2 SQL query response was not an object.", + provider: "cloudflare_r2_sql", + retryable: false, + timedOut: false, + }) + ); + } + + if (payload.success === false) { + return Result.err( + new ProviderResponseFailure({ + message: `Cloudflare R2 SQL query failed: ${sanitizeCloudflareR2SqlErrorText(readCloudflareR2SqlErrorText(payload), apiToken)}`, + provider: "cloudflare_r2_sql", + retryable: false, + timedOut: false, + }) + ); + } + + const result = payload.result; + if (!isRecord(result)) { + return Result.err( + new ProviderResponseFailure({ + message: "Cloudflare R2 SQL query response did not include a result.", + provider: "cloudflare_r2_sql", + retryable: false, + timedOut: false, + }) + ); + } + + return Result.try({ + try: () => normalizeRecordRows("Cloudflare R2 SQL", result.rows), + catch: (cause) => + new ProviderResponseFailure({ + cause, + message: toErrorMessage(cause), + provider: "cloudflare_r2_sql", + retryable: false, + timedOut: false, + }), + }); +} + +function readCloudflareR2SqlErrorText( + payload: Record +): string { + const errors = payload.errors; + if (!Array.isArray(errors) || errors.length === 0) { + return "Unknown error"; + } + + return errors + .map((entry) => { + if (!isRecord(entry)) { + return String(entry); + } + const code = + typeof entry.code === "number" || typeof entry.code === "string" + ? `${entry.code}: ` + : ""; + const message = + typeof entry.message === "string" + ? entry.message + : JSON.stringify(entry); + return `${code}${message}`; + }) + .join("; "); +} + +function sanitizeCloudflareR2SqlErrorText( + text: string, + apiToken: string +): string { + return sanitizeProviderErrorText( + text.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/giu, "Bearer [REDACTED]"), + apiToken + ); +} + +async function runCloudflareR2SqlConnectionTest( + credentials: CloudflareR2SqlCredentials, + deadline: QueryDeadline = createQueryDeadline(QUERY_TIMEOUT_MS) +) { + return runProviderConnectionTest({ + deadline, + execute: () => + executeCloudflareR2SqlQuery( + credentials, + CONNECTION_TEST_QUERY, + deadline.timeoutMs + ), + mapError: (error, latencyMs) => { + const statusCode = readCloudflareR2SqlStatusCode(error); + if (statusCode === 401) { + return createFailedConnectionTest({ + detail: "Invalid or expired Cloudflare R2 SQL credentials", + latencyMs, + message: "Authentication failed", + }); + } + if (statusCode === 403) { + return createFailedConnectionTest({ + detail: + "Cloudflare credentials do not have access to this R2 SQL warehouse", + latencyMs, + message: "Access denied", + }); + } + + return null; + }, + }); +} + +function classifyCloudflareR2SqlError( + error: unknown +): QueryErrorClassification | null { + const statusCode = readCloudflareR2SqlStatusCode(error); + if (statusCode === null) { + return null; + } + + return { + retryable: TRANSIENT_CLOUDFLARE_R2_SQL_STATUS_CODES.has(statusCode), + timedOut: statusCode === 504, + }; +} + +function readCloudflareR2SqlStatusCode(error: unknown): number | null { + const message = error instanceof Error ? error.message : String(error); + const match = /Cloudflare R2 SQL query failed: (\d{3})\b/u.exec(message); + if (!match) { + return null; + } + + const statusCode = Number(match[1]); + return Number.isInteger(statusCode) ? statusCode : null; +} diff --git a/packages/server/src/services/data-source-query/validate-sql.test.ts b/packages/server/src/services/data-source-query/validate-sql.test.ts index d40c1e0b..0f489461 100644 --- a/packages/server/src/services/data-source-query/validate-sql.test.ts +++ b/packages/server/src/services/data-source-query/validate-sql.test.ts @@ -87,6 +87,22 @@ describe("validateAndNormalizeReadOnlyQuery", () => { ); }); + it("validates Cloudflare R2 SQL queries with the Athena dialect", async () => { + await expectValidQuery( + "SELECT * FROM default.transactions LIMIT 10", + { + sql: "SELECT * FROM default.transactions LIMIT 10", + }, + "cloudflare_r2_sql" + ); + + await expectValidationError( + "CREATE TABLE copied AS SELECT 1", + "Only SELECT queries are allowed. Got: create_table", + "cloudflare_r2_sql" + ); + }); + it("validates Snowflake queries with the Snowflake dialect", async () => { await expectValidQuery( "SELECT id FROM events QUALIFY ROW_NUMBER() OVER (ORDER BY created_at DESC) <= 10", diff --git a/packages/server/src/services/data-source-query/validate-sql.ts b/packages/server/src/services/data-source-query/validate-sql.ts index c4213b82..b0e2c7bd 100644 --- a/packages/server/src/services/data-source-query/validate-sql.ts +++ b/packages/server/src/services/data-source-query/validate-sql.ts @@ -63,6 +63,7 @@ const DIALECT_MAP = { aws_athena_connector: Dialect.Athena, bigquery: Dialect.BigQuery, cloudflare_d1: Dialect.SQLite, + cloudflare_r2_sql: Dialect.Athena, laminar: Dialect.ClickHouse, // MotherDuck executes DuckDB SQL through a PostgreSQL wire endpoint. DuckDB's // dialect intentionally follows PostgreSQL closely enough for our read-only diff --git a/packages/ui/src/data-sources/connection-guides/cloudflare-r2-sql.ts b/packages/ui/src/data-sources/connection-guides/cloudflare-r2-sql.ts new file mode 100644 index 00000000..3d95b5c2 --- /dev/null +++ b/packages/ui/src/data-sources/connection-guides/cloudflare-r2-sql.ts @@ -0,0 +1,113 @@ +import type { GuideContent } from "./types"; + +export const cloudflareR2SqlGuideContent = { + providerLabel: "Cloudflare R2 SQL", + ko: { + title: "Cloudflare R2 SQL 연결 가이드", + description: + "OneQuery의 Cloudflare R2 SQL 연결은 `accountId`, `bucketName`, `apiToken`, optional `apiBaseUrl`를 받습니다. R2 Data Catalog가 활성화된 bucket과 R2 SQL 조회 권한이 있는 API Token을 준비해 주세요.", + steps: [ + { + title: "R2 Data Catalog 확인", + paragraphs: [ + "Cloudflare Dashboard에서 대상 R2 bucket을 열고 R2 Data Catalog가 활성화되어 있는지 확인해 주세요.", + "OneQuery는 R2 SQL REST API를 통해 이 catalog의 Apache Iceberg table을 조회합니다.", + ], + }, + { + title: "Account ID와 Bucket 이름 확인", + paragraphs: [ + "Cloudflare Dashboard에서 대상 계정의 Account ID를 복사해 주세요.", + "R2 bucket 이름을 `bucketName`으로 입력합니다. Wrangler의 warehouse 이름은 보통 `{accountId}_{bucketName}` 형식이지만, OneQuery에는 두 값을 분리해서 입력합니다.", + ], + }, + { + title: "API Token 준비", + paragraphs: [ + "Cloudflare API Tokens 화면에서 OneQuery 전용 token을 생성해 주세요.", + "R2 SQL read, R2 Data Catalog read, R2 storage access 권한을 대상 bucket 범위로 제한하는 것을 권장합니다.", + ], + note: "API Token 값은 비밀값입니다. 스크린샷, 이슈, 채팅에 노출하지 마세요.", + }, + { + title: "OneQuery에 입력", + paragraphs: [ + "OneQuery 웹 폼 또는 CLI JSON에 Account ID, Bucket name, API Token을 입력해 주세요.", + "Cloudflare 기본 R2 SQL API를 사용할 때 `apiBaseUrl`은 비워 두면 됩니다. 프록시나 테스트용 엔드포인트가 있을 때만 설정해 주세요.", + ], + bullets: [ + "`accountId`: Cloudflare Account ID", + "`bucketName`: R2 Data Catalog가 연결된 R2 bucket 이름", + "`apiToken`: R2 SQL 조회용 Cloudflare API Token", + "`apiBaseUrl` (optional): 기본값은 `https://api.sql.cloudflarestorage.com/api/v1`", + ], + code: `{ + "name": "cloudflare_r2_sql_prod", + "credentials": { + "type": "cloudflare_r2_sql", + "accountId": "023e105f4ecef8ad9ca31a8372d0c353", + "bucketName": "analytics-events", + "apiToken": "cf_r2_sql_token" + } +}`, + }, + ], + closingTitle: "연결 준비 완료", + closingDescription: + "연결 후 SQL query 도구에서 `namespace.table_name` 형식으로 R2 Data Catalog의 Iceberg table을 read-only 조회할 수 있습니다.", + }, + en: { + title: "Cloudflare R2 SQL Connection Guide", + description: + "The OneQuery Cloudflare R2 SQL connection accepts `accountId`, `bucketName`, `apiToken`, and optional `apiBaseUrl`. Prepare an R2 bucket with R2 Data Catalog enabled and an API token that can run R2 SQL queries.", + steps: [ + { + title: "Check R2 Data Catalog", + paragraphs: [ + "Open the target R2 bucket in the Cloudflare Dashboard and confirm that R2 Data Catalog is enabled.", + "OneQuery queries Apache Iceberg tables in that catalog through the R2 SQL REST API.", + ], + }, + { + title: "Find the Account ID and bucket name", + paragraphs: [ + "Copy the Account ID from the target Cloudflare account.", + "Enter the R2 bucket name as `bucketName`. Wrangler's warehouse name is usually `{accountId}_{bucketName}`, but OneQuery stores the two fields separately.", + ], + }, + { + title: "Prepare an API token", + paragraphs: [ + "Create a dedicated token for OneQuery from the Cloudflare API Tokens page.", + "Prefer a token scoped to the target bucket with R2 SQL read, R2 Data Catalog read, and R2 storage access.", + ], + note: "The API token is secret. Do not include it in screenshots, issues, or chat.", + }, + { + title: "Enter the fields in OneQuery", + paragraphs: [ + "Paste the Account ID, bucket name, and API token into the OneQuery form or CLI JSON payload.", + "Leave `apiBaseUrl` empty for Cloudflare's default R2 SQL API. Set it only for a proxy or test endpoint.", + ], + bullets: [ + "`accountId`: Cloudflare Account ID", + "`bucketName`: R2 bucket name with R2 Data Catalog enabled", + "`apiToken`: Cloudflare API token for R2 SQL queries", + "`apiBaseUrl` (optional): defaults to `https://api.sql.cloudflarestorage.com/api/v1`", + ], + code: `{ + "name": "cloudflare_r2_sql_prod", + "credentials": { + "type": "cloudflare_r2_sql", + "accountId": "023e105f4ecef8ad9ca31a8372d0c353", + "bucketName": "analytics-events", + "apiToken": "cf_r2_sql_token" + } +}`, + }, + ], + closingTitle: "Ready to Connect", + closingDescription: + "After connecting, SQL query tools can read R2 Data Catalog Iceberg tables with `namespace.table_name` references.", + }, +} satisfies GuideContent; diff --git a/packages/ui/src/data-sources/connection-guides/index.ts b/packages/ui/src/data-sources/connection-guides/index.ts index 7187dded..ba873a4f 100644 --- a/packages/ui/src/data-sources/connection-guides/index.ts +++ b/packages/ui/src/data-sources/connection-guides/index.ts @@ -2,6 +2,7 @@ import { amplitudeGuideContent } from "./amplitude"; import { awsAthenaConnectorGuideContent } from "./aws-athena-connector"; import { bigqueryGuideContent } from "./bigquery"; import { cloudflareD1GuideContent } from "./cloudflare-d1"; +import { cloudflareR2SqlGuideContent } from "./cloudflare-r2-sql"; import { cloudflareWorkersObservabilityGuideContent } from "./cloudflare-workers-observability"; import { gaGuideContent } from "./ga"; import { githubGuideContent } from "./github"; @@ -39,6 +40,7 @@ export const GUIDE_CONTENT: Record< aws_athena_connector: awsAthenaConnectorGuideContent, bigquery: bigqueryGuideContent, cloudflare_d1: cloudflareD1GuideContent, + cloudflare_r2_sql: cloudflareR2SqlGuideContent, cloudflare_workers_observability: cloudflareWorkersObservabilityGuideContent, ga: gaGuideContent, google_search_console: googleSearchConsoleGuideContent, diff --git a/packages/ui/src/data-sources/connection-guides/types.ts b/packages/ui/src/data-sources/connection-guides/types.ts index 07d3d5b4..a65289ea 100644 --- a/packages/ui/src/data-sources/connection-guides/types.ts +++ b/packages/ui/src/data-sources/connection-guides/types.ts @@ -5,6 +5,7 @@ export type DataSourceConnectionGuideProvider = | "aws_athena_connector" | "bigquery" | "cloudflare_d1" + | "cloudflare_r2_sql" | "cloudflare_workers_observability" | "ga" | "google_search_console"