diff --git a/workspaces/boost/.changeset/connector-schema-versioning.md b/workspaces/boost/.changeset/connector-schema-versioning.md new file mode 100644 index 00000000000..4c429731fa9 --- /dev/null +++ b/workspaces/boost/.changeset/connector-schema-versioning.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-boost-backend': minor +--- + +Add per-connector `__schemaVersion` leaf with `db-only` scope and startup migration infrastructure. Registers `boost.connectors..__schemaVersion` metadata keys for jira, github, and gitlab connectors. Introduces `BOOST_CONNECTOR_SCHEMA_VERSION`, `CONNECTOR_IDS`, `ConnectorId` type, `ConnectorMigrationFn`, `ConnectorMigrationRegistry`, and `RuntimeConfigResolver.migrateConnectorSchemas()` which runs on plugin startup. diff --git a/workspaces/boost/AGENTS.md b/workspaces/boost/AGENTS.md index 3a5e3144e79..5aa83d774a0 100644 --- a/workspaces/boost/AGENTS.md +++ b/workspaces/boost/AGENTS.md @@ -82,7 +82,10 @@ failures or config-surface drift. 2. Register the field in `src/config/schemas.ts` under `boostConfigFields` with a Zod schema, `configScope`, and `description` -3. Bump `BOOST_CONFIG_SCHEMA_VERSION` in `src/config/schemas.ts` +3. Bump `BOOST_CONFIG_SCHEMA_VERSION` in `src/config/schemas.ts`. + Per-connector `__schemaVersion` leaves (`configScope: db-only`) are + the versioning machinery itself and do not require bumping this + constant. 4. Add example usage in `examples/app-config.connectors.yaml` (or the appropriate `app-config.*.yaml` example file) 5. Run `yarn tsc:full && yarn build:api-reports:only` and commit the diff --git a/workspaces/boost/openspec/changes/connector-config-hot-reload/design.md b/workspaces/boost/openspec/changes/connector-config-hot-reload/design.md index 7f362484c86..631b4ebd711 100644 --- a/workspaces/boost/openspec/changes/connector-config-hot-reload/design.md +++ b/workspaces/boost/openspec/changes/connector-config-hot-reload/design.md @@ -72,7 +72,7 @@ await this.syncClient.connect(endpoint); ### Decision 2: configScope annotation strategy -Each `boost.connectors..*` field is `configScope: db-overridable` — these are the runtime-tunable fields. Deployment-time fields (`tls.caFile`, `credentials.*`, `namespace`) live under `ai-catalog.providers..*` and are not part of this schema (see Goals namespace table above). +Each user-facing `boost.connectors..*` field is `configScope: db-overridable` — these are the runtime-tunable fields. The per-connector `__schemaVersion` leaf is internal metadata with `configScope: db-only`. Deployment-time fields (`tls.caFile`, `credentials.*`, `namespace`) live under `ai-catalog.providers..*` and are not part of this schema (see Goals namespace table above). | Field | configScope | Rationale | | ---------------------- | ---------------- | -------------------------------------------------- | @@ -82,10 +82,11 @@ Each `boost.connectors..*` field is `configScope: db-overridable` — these | `schedule.cron` | `db-overridable` | Admin can change cron schedule at runtime | | `batchSize` | `db-overridable` | Admin can tune performance at runtime | | `timeout.connectionMs` | `db-overridable` | Admin can adjust for network conditions at runtime | +| `__schemaVersion` | `db-only` | Internal migration metadata; not admin-editable | **Runtime state lives in the health store, not the config resolver:** Fields like `lastSyncTimestamp` and `lastSyncOutcome` are pure runtime state owned by the `boost_sync_attempts` table (see ingestion-health-dashboard Decision 1). They are not config — they are operational state written by providers after each sync. Run status (running/idle) is derived from these fields, not stored as a separate column. Querying them goes through the health API (`GET /api/boost/ingestion-health`), not `RuntimeConfigResolver`. -**Why all fields are db-overridable:** The `boost.connectors` schema only contains runtime-tunable fields by design. Deployment-time fields (mount paths, Secret references, namespace) belong to `ai-catalog.providers` — they can't change at runtime without a pod restart, so they are excluded from this schema entirely rather than marked `yaml-only`. +**Why user-facing fields are db-overridable:** The `boost.connectors` schema contains runtime-tunable fields by design, plus `__schemaVersion` as `db-only` internal metadata. Deployment-time fields (mount paths, Secret references, namespace) belong to `ai-catalog.providers` — they can't change at runtime without a pod restart, so they are excluded from this schema entirely rather than marked `yaml-only`. ### Decision 3: Propagation mechanism — polling-based via reconciliation cycles diff --git a/workspaces/boost/openspec/changes/connector-config-hot-reload/proposal.md b/workspaces/boost/openspec/changes/connector-config-hot-reload/proposal.md index c5d082c5c49..1faedd4b45e 100644 --- a/workspaces/boost/openspec/changes/connector-config-hot-reload/proposal.md +++ b/workspaces/boost/openspec/changes/connector-config-hot-reload/proposal.md @@ -14,7 +14,7 @@ The key distinction: Backstage's built-in `ConfigApi` loads config at startup wi ### Config Schemas -- Zod schema definitions for per-connector `boost.connectors.*` fields: `enabled`, `endpoint`, `schedule`, `batchSize`, `timeout` — all `configScope: db-overridable`. Deployment-time fields (`tls`, `credentials`, `namespace`) live under `ai-catalog.providers.*` and are not part of these schemas. +- Zod schema definitions for per-connector `boost.connectors.*` fields: `enabled`, `endpoint`, `schedule`, `batchSize`, `timeout` — all user-facing fields are `configScope: db-overridable`; `__schemaVersion` is `configScope: db-only` internal metadata. Deployment-time fields (`tls`, `credentials`, `namespace`) live under `ai-catalog.providers.*` and are not part of these schemas. - Runtime operational state (last sync timestamp, run status) lives in the health store (`boost_sync_attempts` table), not the config resolver. - Schema validation rejects invalid connector config values before write - Integration with `RuntimeConfigResolver`'s two-layer resolution diff --git a/workspaces/boost/openspec/changes/connector-config-hot-reload/specs/config-schemas/spec.md b/workspaces/boost/openspec/changes/connector-config-hot-reload/specs/config-schemas/spec.md index d0a3bf03f0b..a3f7110a281 100644 --- a/workspaces/boost/openspec/changes/connector-config-hot-reload/specs/config-schemas/spec.md +++ b/workspaces/boost/openspec/changes/connector-config-hot-reload/specs/config-schemas/spec.md @@ -13,20 +13,20 @@ Each connector has a Zod schema defining all configuration fields with `configSc #### Scenario: Jira connector config schema - **WHEN** Jira connector config schema is defined -- **THEN** schema includes `boost.connectors` fields only: `enabled` (boolean), `endpoint` (URL string), `schedule.intervalMs` (number), `schedule.cron` (string), `batchSize` (number), `timeout.connectionMs` (number) -- **AND** all fields are `configScope: db-overridable` (deployment-time fields like `tls.caFile`, `credentials.*`, and `namespace` live under `ai-catalog.providers..*` and are not part of this schema) +- **THEN** schema includes `boost.connectors` fields only: `enabled` (boolean), `endpoint` (URL string), `schedule.intervalMs` (number), `schedule.cron` (string), `batchSize` (number), `timeout.connectionMs` (number), `__schemaVersion` (number, internal metadata) +- **AND** all user-facing fields are `configScope: db-overridable`; `__schemaVersion` is `configScope: db-only` (deployment-time fields like `tls.caFile`, `credentials.*`, and `namespace` live under `ai-catalog.providers..*` and are not part of this schema) #### Scenario: GitHub connector config schema - **WHEN** GitHub connector config schema is defined -- **THEN** schema includes `boost.connectors` fields only: `enabled` (boolean), `endpoint` (URL string), `schedule.intervalMs` (number), `batchSize` (number) -- **AND** all fields are `configScope: db-overridable` (matching Jira pattern) +- **THEN** schema includes `boost.connectors` fields only: `enabled` (boolean), `endpoint` (URL string), `schedule.intervalMs` (number), `batchSize` (number), `__schemaVersion` (number, internal metadata) +- **AND** all user-facing fields are `configScope: db-overridable`; `__schemaVersion` is `configScope: db-only` (matching Jira pattern) #### Scenario: GitLab connector config schema - **WHEN** GitLab connector config schema is defined -- **THEN** schema includes `boost.connectors` fields only: `enabled` (boolean), `endpoint` (URL string), `schedule.intervalMs` (number), `batchSize` (number) -- **AND** all fields are `configScope: db-overridable` (matching Jira pattern) +- **THEN** schema includes `boost.connectors` fields only: `enabled` (boolean), `endpoint` (URL string), `schedule.intervalMs` (number), `batchSize` (number), `__schemaVersion` (number, internal metadata) +- **AND** all user-facing fields are `configScope: db-overridable`; `__schemaVersion` is `configScope: db-only` (matching Jira pattern) ### Requirement: RuntimeConfigResolver Integration @@ -88,9 +88,20 @@ Connector config schemas support versioning for backward compatibility. #### Scenario: Schema migration on version mismatch -- **WHEN** DB override has `schemaVersion: 1` and current schema is `schemaVersion: 2` -- **THEN** `RuntimeConfigResolver` applies migration logic to upgrade old config -- **AND** migrated config validates against current schema +- **WHEN** stored `boost.connectors..__schemaVersion` is `1` and `BOOST_CONNECTOR_SCHEMA_VERSION` is `2` +- **THEN** `RuntimeConfigResolver.migrateConnectorSchemas()` applies the migration registered under source version `1` +- **AND** migrated config validates against the current schema +- **AND** the stored `__schemaVersion` is stamped to `2` after the successful step + +#### Scenario: Future field rename or removal + +- **WHEN** a connector field is renamed, removed, or its value type changes +- **THEN** `BOOST_CONNECTOR_SCHEMA_VERSION` is incremented +- **AND** a migration function is registered on `ConnectorMigrationRegistry` keyed by the **source** version (key `1` upgrades v1 → v2) +- **AND** `RuntimeConfigResolver.migrateConnectorSchemas()` runs on plugin startup after `validateStoredValues()` +- **AND** a missing `__schemaVersion` is treated as v1 and written explicitly before intermediate migrations run +- **AND** each successful migration step stamps the next version so a later failure can resume +- **AND** migration functions must be idempotent (a function that throws after partial leaf writes will re-run) ### Requirement: Default Values diff --git a/workspaces/boost/openspec/changes/connector-config-hot-reload/tasks.md b/workspaces/boost/openspec/changes/connector-config-hot-reload/tasks.md index 4ebdac19bac..1d1f090084b 100644 --- a/workspaces/boost/openspec/changes/connector-config-hot-reload/tasks.md +++ b/workspaces/boost/openspec/changes/connector-config-hot-reload/tasks.md @@ -3,14 +3,14 @@ ## 1. Zod Schema Definitions (P0) — RHIDP-15340 - [ ] 1.1 Define Jira connector config Zod schema with `boost.connectors` fields only: `enabled` (boolean), `endpoint` (URL), `schedule.intervalMs` (number), `schedule.cron` (string), `batchSize` (number), `timeout.connectionMs` (number). Note: `tls.caFile`, `credentials.*`, and `namespace` are `ai-catalog.providers` fields — not part of the `boost.connectors` schema. -- [ ] 1.2 All `boost.connectors` fields are `configScope: db-overridable` (deployment-time fields like `credentials.*`, `tls.*`, and `namespace` live under `ai-catalog.providers..*`) +- [ ] 1.2 All user-facing `boost.connectors` fields are `configScope: db-overridable`; `__schemaVersion` is `configScope: db-only` internal metadata (deployment-time fields like `credentials.*`, `tls.*`, and `namespace` live under `ai-catalog.providers..*`) - [ ] 1.3 Define GitHub connector config Zod schema with connector-appropriate field subset (`enabled`, `endpoint`, `schedule.intervalMs`, `batchSize`) - [ ] 1.4 Define GitLab connector config Zod schema with connector-appropriate field subset (`enabled`, `endpoint`, `schedule.intervalMs`, `batchSize`) - [ ] 1.5 Add URL validation for `endpoint` field (must be valid https:// URL) - [ ] 1.6 Add positive number validation for `schedule.intervalMs`, `batchSize`, `timeout.connectionMs` - [ ] 1.7 Add cron expression validation for `schedule.cron` (via cron parser library) - [ ] 1.8 Define default values in schemas: `schedule.intervalMs: 300000` (5 min), `batchSize: 100`, `timeout.connectionMs: 30000` -- [ ] 1.9 Add schema versioning field: `schemaVersion: 1` in each schema +- [ ] 1.9 Add per-connector leaf `boost.connectors..__schemaVersion` (`configScope: db-only`, current value `BOOST_CONNECTOR_SCHEMA_VERSION`) - [ ] 1.10 Add unit tests for schema validation (valid configs pass, invalid configs rejected with correct error messages) ## 2. RuntimeConfigResolver Extension (P0) — RHIDP-15340 @@ -86,7 +86,7 @@ ## 8. Documentation (P2) - [ ] 8.1 Document `RuntimeConfigResolver` extension for connector config in architecture docs -- [ ] 8.2 Document `configScope` annotations and their meaning (`yaml-only`, `db-overridable`). Note: runtime operational state lives in the health store (`boost_sync_attempts` table), not the config resolver. +- [ ] 8.2 Document `configScope` annotations and their meaning (`yaml-only`, `db-overridable`, `db-only`). Note: runtime operational state lives in the health store (`boost_sync_attempts` table), not the config resolver. `__schemaVersion` is `db-only` internal metadata. - [ ] 8.3 Document connector config admin UI usage (how to toggle, change endpoint/schedule) - [ ] 8.4 Document propagation latency: 30s TTL + reconciliation interval - [ ] 8.5 Document credential rotation workflow and latency (≤60s kubelet + reconciliation interval) diff --git a/workspaces/boost/plugins/boost-backend/config.d.ts b/workspaces/boost/plugins/boost-backend/config.d.ts index 73e896d1e1c..d8f6ddb990d 100644 --- a/workspaces/boost/plugins/boost-backend/config.d.ts +++ b/workspaces/boost/plugins/boost-backend/config.d.ts @@ -211,6 +211,11 @@ export interface Config { */ connectionMs?: number; }; + /** + * Per-connector schema version (internal metadata). + * @configScope db-only + */ + __schemaVersion?: number; }; /** GitHub connector runtime configuration. */ github?: { @@ -237,6 +242,11 @@ export interface Config { * @configScope db-overridable */ batchSize?: number; + /** + * Per-connector schema version (internal metadata). + * @configScope db-only + */ + __schemaVersion?: number; }; /** GitLab connector runtime configuration. */ gitlab?: { @@ -263,6 +273,11 @@ export interface Config { * @configScope db-overridable */ batchSize?: number; + /** + * Per-connector schema version (internal metadata). + * @configScope db-only + */ + __schemaVersion?: number; }; /** * Open index signature preserving backward compatibility. diff --git a/workspaces/boost/plugins/boost-backend/report.api.md b/workspaces/boost/plugins/boost-backend/report.api.md index 8500bcd5bbb..c6161ce5f65 100644 --- a/workspaces/boost/plugins/boost-backend/report.api.md +++ b/workspaces/boost/plugins/boost-backend/report.api.md @@ -121,6 +121,9 @@ export interface BackendApprovalStoreOptions { // @public export const BOOST_CONFIG_SCHEMA_VERSION = 4; +// @public +export const BOOST_CONNECTOR_SCHEMA_VERSION = 1; + // @public export const boostAiProviderServiceFactory: ServiceFactory< AgenticProvider, @@ -221,6 +224,21 @@ export const boostConfigFields: { readonly configScope: ConfigScope; readonly description: string; }; + readonly 'boost.connectors.jira.__schemaVersion': { + readonly schema: z.ZodOptional; + readonly configScope: ConfigScope; + readonly description: string; + }; + readonly 'boost.connectors.github.__schemaVersion': { + readonly schema: z.ZodOptional; + readonly configScope: ConfigScope; + readonly description: string; + }; + readonly 'boost.connectors.gitlab.__schemaVersion': { + readonly schema: z.ZodOptional; + readonly configScope: ConfigScope; + readonly description: string; + }; readonly 'boost.connectors.jira.enabled': { readonly schema: z.ZodOptional; readonly configScope: ConfigScope; @@ -326,6 +344,9 @@ export interface ConfigFieldMeta { // @public export type ConfigScope = 'yaml-only' | 'db-overridable' | 'db-only'; +// @public +export const CONNECTOR_IDS: readonly ['jira', 'github', 'gitlab']; + // @public export interface ConnectorCandidate { connectorId: string; @@ -346,6 +367,18 @@ export interface ConnectorConfigReaderOptions { logger: LoggerService; } +// @public +export type ConnectorId = (typeof CONNECTOR_IDS)[number]; + +// @public +export type ConnectorMigrationFn = ( + connectorId: ConnectorId, + adminConfigService: AdminConfigService, +) => Promise; + +// @public +export type ConnectorMigrationRegistry = Map; + // @public export class ConversationAgentCache { constructor(options: ConversationAgentCacheOptions); @@ -616,6 +649,9 @@ export type ResourceLoader = (req: Request_2) => Promise< export class RuntimeConfigResolver { constructor(options: RuntimeConfigResolverOptions); invalidate(): Promise; + migrateConnectorSchemas( + migrations?: ConnectorMigrationRegistry, + ): Promise; resolve(key: BoostConfigKey): Promise; resolveAll(): Promise>; } diff --git a/workspaces/boost/plugins/boost-backend/src/config/AdminConfigService.test.ts b/workspaces/boost/plugins/boost-backend/src/config/AdminConfigService.test.ts index 15dd5ee7043..8bd05c69755 100644 --- a/workspaces/boost/plugins/boost-backend/src/config/AdminConfigService.test.ts +++ b/workspaces/boost/plugins/boost-backend/src/config/AdminConfigService.test.ts @@ -338,6 +338,21 @@ describe('AdminConfigService', () => { expect(removed).toEqual([]); }); + it('preserves __schemaVersion leaves (db-only metadata)', async () => { + // Write a __schemaVersion leaf via setOverride (which validates + // against the registered Zod schema and checks db-writability) + await service.setOverride('boost.connectors.jira.__schemaVersion', 1); + + const removed = await service.validateStoredValues(); + expect(removed).not.toContain('boost.connectors.jira.__schemaVersion'); + + // The value should still be readable + const value = await service.getOverride( + 'boost.connectors.jira.__schemaVersion', + ); + expect(value).toBe(1); + }); + it('removes values for unknown keys', async () => { // Insert directly into the mock rows mockKnex._rows.push({ diff --git a/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.migrateConnectorSchemas.test.ts b/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.migrateConnectorSchemas.test.ts new file mode 100644 index 00000000000..103bfc6d8d6 --- /dev/null +++ b/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.migrateConnectorSchemas.test.ts @@ -0,0 +1,227 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('./schemas', () => { + const actual = jest.requireActual('./schemas') as typeof import('./schemas'); + return { + ...actual, + BOOST_CONNECTOR_SCHEMA_VERSION: 2, + }; +}); + +import type { + CacheService, + LoggerService, + RootConfigService, +} from '@backstage/backend-plugin-api'; +import { + RuntimeConfigResolver, + type ConnectorMigrationRegistry, +} from './RuntimeConfigResolver'; +import { AdminConfigService } from './AdminConfigService'; +import { CONNECTOR_IDS, type ConnectorId } from './schemas'; + +function createMockLogger(): LoggerService { + return { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + child: jest.fn().mockReturnThis(), + }; +} + +function createMockCache(): CacheService { + return { + get: jest.fn(), + set: jest.fn(), + delete: jest.fn(), + withOptions: jest.fn().mockReturnThis(), + } as unknown as CacheService; +} + +function createMockConfig(): RootConfigService { + return { + getOptionalString: () => undefined, + getOptionalNumber: () => undefined, + getOptional: () => undefined, + getOptionalConfig: () => undefined, + } as unknown as RootConfigService; +} + +describe('migrateConnectorSchemas (BOOST_CONNECTOR_SCHEMA_VERSION=2)', () => { + let cache: CacheService; + let logger: LoggerService; + + beforeEach(() => { + cache = createMockCache(); + logger = createMockLogger(); + }); + + it('runs v1→v2 migration per connector and stamps 2', async () => { + const migrationFn = jest.fn().mockResolvedValue(undefined); + const migrations: ConnectorMigrationRegistry = new Map([[1, migrationFn]]); + + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(1), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config: createMockConfig(), + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(migrations); + + expect(migrationFn).toHaveBeenCalledTimes(CONNECTOR_IDS.length); + for (const connectorId of CONNECTOR_IDS) { + expect(migrationFn).toHaveBeenCalledWith(connectorId, adminConfigService); + expect(adminConfigService.setOverride).toHaveBeenCalledWith( + `boost.connectors.${connectorId}.__schemaVersion`, + 2, + ); + } + }); + + it('stamps 2 when no migration fn is registered (no-op step)', async () => { + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(1), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config: createMockConfig(), + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(); + + for (const connectorId of CONNECTOR_IDS) { + expect(adminConfigService.setOverride).toHaveBeenCalledWith( + `boost.connectors.${connectorId}.__schemaVersion`, + 2, + ); + } + }); + + it('writes v1 then runs v1→v2 when stored version is missing', async () => { + const migrationFn = jest.fn().mockResolvedValue(undefined); + const migrations: ConnectorMigrationRegistry = new Map([[1, migrationFn]]); + + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(undefined), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config: createMockConfig(), + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(migrations); + + for (const connectorId of CONNECTOR_IDS) { + const key = `boost.connectors.${connectorId}.__schemaVersion`; + expect(adminConfigService.setOverride).toHaveBeenCalledWith(key, 1); + expect(adminConfigService.setOverride).toHaveBeenCalledWith(key, 2); + } + expect(migrationFn).toHaveBeenCalledTimes(CONNECTOR_IDS.length); + }); + + it('does not stamp the failed connector to v2; remaining connectors still migrate', async () => { + const migrationFn = jest.fn(async (connectorId: ConnectorId) => { + if (connectorId === 'jira') { + throw new Error('jira migration failed'); + } + }); + const migrations: ConnectorMigrationRegistry = new Map([[1, migrationFn]]); + + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(1), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config: createMockConfig(), + adminConfigService, + logger, + }); + + await expect(resolver.migrateConnectorSchemas(migrations)).rejects.toThrow( + 'jira migration failed', + ); + + expect(adminConfigService.setOverride).not.toHaveBeenCalledWith( + 'boost.connectors.jira.__schemaVersion', + 2, + ); + expect(adminConfigService.setOverride).toHaveBeenCalledWith( + 'boost.connectors.github.__schemaVersion', + 2, + ); + expect(adminConfigService.setOverride).toHaveBeenCalledWith( + 'boost.connectors.gitlab.__schemaVersion', + 2, + ); + expect(cache.delete).toHaveBeenCalledWith('effective-config'); + }); + + it('rethrows the migration error when cache invalidation also fails', async () => { + const migrationFn = jest.fn(async (connectorId: ConnectorId) => { + if (connectorId === 'jira') { + throw new Error('jira migration failed'); + } + }); + const migrations: ConnectorMigrationRegistry = new Map([[1, migrationFn]]); + + cache.delete = jest + .fn() + .mockRejectedValue(new Error('cache delete failed')); + + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(1), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config: createMockConfig(), + adminConfigService, + logger, + }); + + await expect(resolver.migrateConnectorSchemas(migrations)).rejects.toThrow( + 'jira migration failed', + ); + expect(logger.error).toHaveBeenCalledWith( + 'Failed to invalidate config cache after connector schema migration', + expect.any(Error), + ); + }); +}); diff --git a/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.test.ts b/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.test.ts index 00d569f8ace..79b21174869 100644 --- a/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.test.ts +++ b/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.test.ts @@ -22,6 +22,7 @@ import type { import type { JsonValue } from '@backstage/types'; import { RuntimeConfigResolver } from './RuntimeConfigResolver'; import { AdminConfigService } from './AdminConfigService'; +import { CONNECTOR_IDS, BOOST_CONNECTOR_SCHEMA_VERSION } from './schemas'; function createMockLogger(): LoggerService { return { @@ -587,4 +588,151 @@ describe('RuntimeConfigResolver', () => { expect(allConfig.get('boost.connectors.gitlab.enabled')).toBe(true); }); }); + + describe('migrateConnectorSchemas', () => { + it('writes v1 when no __schemaVersion exists', async () => { + const config = createMockConfig({}); + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(undefined), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config, + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(); + + for (const connectorId of CONNECTOR_IDS) { + expect(adminConfigService.setOverride).toHaveBeenCalledWith( + `boost.connectors.${connectorId}.__schemaVersion`, + 1, + ); + } + }); + + it('treats missing version as v1 (logged)', async () => { + const config = createMockConfig({}); + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(undefined), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config, + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(); + + expect(logger.info).toHaveBeenCalledWith( + expect.stringContaining('treating as v1'), + ); + }); + + it('skips migration when stored version equals current', async () => { + const config = createMockConfig({}); + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest + .fn() + .mockResolvedValue(BOOST_CONNECTOR_SCHEMA_VERSION), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config, + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(); + + expect(adminConfigService.setOverride).not.toHaveBeenCalled(); + }); + + it('warns and skips when stored version is ahead of current', async () => { + const config = createMockConfig({}); + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(99), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config, + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(); + + expect(adminConfigService.setOverride).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('possible downgrade'), + ); + }); + + it('invalidates cache after migration completes', async () => { + const config = createMockConfig({}); + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockResolvedValue(undefined), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config, + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(); + + expect(cache.delete).toHaveBeenCalledWith('effective-config'); + }); + + it('handles each connector independently', async () => { + const config = createMockConfig({}); + + const adminConfigService = { + getAllOverrides: jest.fn().mockResolvedValue(new Map()), + getOverride: jest.fn().mockImplementation(async (key: string) => { + if (key === 'boost.connectors.jira.__schemaVersion') { + return BOOST_CONNECTOR_SCHEMA_VERSION; + } + if (key === 'boost.connectors.gitlab.__schemaVersion') { + return BOOST_CONNECTOR_SCHEMA_VERSION; + } + return undefined; + }), + setOverride: jest.fn().mockResolvedValue(undefined), + } as unknown as AdminConfigService; + + const resolver = new RuntimeConfigResolver({ + cache, + config, + adminConfigService, + logger, + }); + + await resolver.migrateConnectorSchemas(); + + expect(adminConfigService.setOverride).toHaveBeenCalledTimes(1); + expect(adminConfigService.setOverride).toHaveBeenCalledWith( + 'boost.connectors.github.__schemaVersion', + 1, + ); + }); + }); }); diff --git a/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.ts b/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.ts index 6d0d2d5b980..3fb83987363 100644 --- a/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.ts +++ b/workspaces/boost/plugins/boost-backend/src/config/RuntimeConfigResolver.ts @@ -23,10 +23,39 @@ import type { JsonValue } from '@backstage/types'; import { AdminConfigService } from './AdminConfigService'; import { boostConfigFields, + BOOST_CONNECTOR_SCHEMA_VERSION, + CONNECTOR_IDS, isSensitiveField, type BoostConfigKey, + type ConnectorId, } from './schemas'; +/** + * A migration function that transforms stored DB overrides for a + * connector from one schema version to the next. Receives the + * connector ID and the admin config service for reading/writing + * individual leaf values. Returns when the migration is complete. + * + * @public + */ +export type ConnectorMigrationFn = ( + connectorId: ConnectorId, + adminConfigService: AdminConfigService, +) => Promise; + +/** + * Registry of connector schema migrations keyed by the **source** + * version they upgrade from. For example, a migration registered + * under key `1` upgrades v1 → v2. + * + * Migrations are applied sequentially: v1 → v2 → v3 etc. Each + * migration must leave the data valid under the next version's + * schema. + * + * @public + */ +export type ConnectorMigrationRegistry = Map; + /** * Cache key for the merged effective config. * @@ -138,6 +167,132 @@ export class RuntimeConfigResolver { await this.invalidate(); } + /** + * Run connector schema migrations on startup. + * + * For each known connector, reads the stored `__schemaVersion` + * leaf from the DB. If missing, writes v1 explicitly (pre-versioning + * data is treated as v1) and then applies any registered migrations + * up to `BOOST_CONNECTOR_SCHEMA_VERSION`. The stored version is + * stamped after each successful step so a later failure can resume. + * + * Migration functions must be idempotent: a function that throws + * after partial leaf writes will re-run on the next startup. + * + * A failure for one connector is logged and does not skip the + * remaining connectors; the first error is rethrown after all + * connectors have been attempted. + * + * @param migrations - Optional registry of version-keyed migration + * functions. When omitted (or empty), only the version stamp is + * written/bumped — no data transforms are applied. + */ + async migrateConnectorSchemas( + migrations?: ConnectorMigrationRegistry, + ): Promise { + let firstError: unknown; + + for (const connectorId of CONNECTOR_IDS) { + try { + await this.migrateOneConnector(connectorId, migrations); + } catch (error) { + this.logger.error( + `Connector "${connectorId}" schema migration failed`, + error as Error, + ); + firstError ??= error; + } + } + + // Invalidate cache after migrations may have changed DB values. + // Preserve a connector migration error if invalidation also fails. + try { + await this.invalidate(); + } catch (invalidateError) { + this.logger.error( + 'Failed to invalidate config cache after connector schema migration', + invalidateError as Error, + ); + firstError ??= invalidateError; + } + + if (firstError) { + throw firstError; + } + } + + /** + * Migrate a single connector's stored schema version. + * + * @internal + */ + private async migrateOneConnector( + connectorId: ConnectorId, + migrations?: ConnectorMigrationRegistry, + ): Promise { + const versionKey = + `boost.connectors.${connectorId}.__schemaVersion` as BoostConfigKey; + + const stored = await this.adminConfigService.getOverride(versionKey); + let storedVersion = typeof stored === 'number' ? stored : undefined; + + if (storedVersion === undefined) { + this.logger.info( + `Connector "${connectorId}" has no stored schema version — ` + + `treating as v1, writing v1`, + ); + await this.adminConfigService.setOverride(versionKey, 1); + storedVersion = 1; + } + + if (storedVersion > BOOST_CONNECTOR_SCHEMA_VERSION) { + this.logger.warn( + `Connector "${connectorId}" schema v${storedVersion} is ahead of ` + + `current v${BOOST_CONNECTOR_SCHEMA_VERSION} (possible downgrade) — ` + + `skipping migration`, + ); + return; + } + + if (storedVersion === BOOST_CONNECTOR_SCHEMA_VERSION) { + this.logger.debug( + `Connector "${connectorId}" schema v${storedVersion} is current`, + ); + return; + } + + this.logger.info( + `Connector "${connectorId}" schema v${storedVersion} → ` + + `v${BOOST_CONNECTOR_SCHEMA_VERSION}: running migrations`, + ); + + for ( + let fromVersion = storedVersion; + fromVersion < BOOST_CONNECTOR_SCHEMA_VERSION; + fromVersion++ + ) { + const migrationFn = migrations?.get(fromVersion); + if (migrationFn) { + await migrationFn(connectorId, this.adminConfigService); + this.logger.info( + `Connector "${connectorId}": migrated v${fromVersion} → ` + + `v${fromVersion + 1}`, + ); + } else { + this.logger.debug( + `Connector "${connectorId}": no migration registered for ` + + `v${fromVersion} → v${fromVersion + 1} (no-op)`, + ); + } + await this.adminConfigService.setOverride(versionKey, fromVersion + 1); + } + + this.logger.info( + `Connector "${connectorId}" schema version bumped to ` + + `v${BOOST_CONNECTOR_SCHEMA_VERSION}`, + ); + } + /** * Get the merged effective config, using cache when available. * This is the single cache layer — no wrapper. diff --git a/workspaces/boost/plugins/boost-backend/src/config/index.ts b/workspaces/boost/plugins/boost-backend/src/config/index.ts index 6bb69654428..ffcf9137ee4 100644 --- a/workspaces/boost/plugins/boost-backend/src/config/index.ts +++ b/workspaces/boost/plugins/boost-backend/src/config/index.ts @@ -21,14 +21,19 @@ export { export { RuntimeConfigResolver, type RuntimeConfigResolverOptions, + type ConnectorMigrationFn, + type ConnectorMigrationRegistry, } from './RuntimeConfigResolver'; export { boostConfigFields, BOOST_CONFIG_SCHEMA_VERSION, + BOOST_CONNECTOR_SCHEMA_VERSION, + CONNECTOR_IDS, validateConfigValue, isDbWritable, isSensitiveField, type BoostConfigKey, + type ConnectorId, type ConfigScope, type ConfigFieldMeta, } from './schemas'; diff --git a/workspaces/boost/plugins/boost-backend/src/config/schemas.test.ts b/workspaces/boost/plugins/boost-backend/src/config/schemas.test.ts index 0332d2fbafd..db89870fdb5 100644 --- a/workspaces/boost/plugins/boost-backend/src/config/schemas.test.ts +++ b/workspaces/boost/plugins/boost-backend/src/config/schemas.test.ts @@ -18,6 +18,8 @@ import { ZodError } from 'zod'; import { boostConfigFields, BOOST_CONFIG_SCHEMA_VERSION, + BOOST_CONNECTOR_SCHEMA_VERSION, + CONNECTOR_IDS, validateConfigValue, isDbWritable, isSensitiveField, @@ -312,9 +314,11 @@ describe('connector config schemas', () => { ); }); - it('marks all connector fields as db-overridable', () => { + it('marks all non-metadata connector fields as db-overridable', () => { const connectorEntries = Object.entries(boostConfigFields).filter( - ([key]) => key.startsWith('boost.connectors.'), + ([key]) => + key.startsWith('boost.connectors.') && + !key.endsWith('.__schemaVersion'), ); expect(connectorEntries.length).toBeGreaterThan(0); connectorEntries.forEach(([, field]) => { @@ -542,4 +546,95 @@ describe('connector config schemas', () => { }); }); }); + + describe('__schemaVersion leaves', () => { + it('has BOOST_CONNECTOR_SCHEMA_VERSION set to 1', () => { + expect(BOOST_CONNECTOR_SCHEMA_VERSION).toBe(1); + }); + + it('exports CONNECTOR_IDS with jira, github, gitlab', () => { + expect(CONNECTOR_IDS).toEqual(['jira', 'github', 'gitlab']); + }); + + it.each(CONNECTOR_IDS)( + 'registers __schemaVersion leaf for %s connector', + connectorId => { + const key = `boost.connectors.${connectorId}.__schemaVersion`; + expect(Object.keys(boostConfigFields)).toContain(key); + }, + ); + + it.each(CONNECTOR_IDS)( + '__schemaVersion for %s has configScope db-only', + connectorId => { + const key = + `boost.connectors.${connectorId}.__schemaVersion` as keyof typeof boostConfigFields; + expect(boostConfigFields[key].configScope).toBe('db-only'); + }, + ); + + it.each(CONNECTOR_IDS)( + '__schemaVersion for %s is db-writable', + connectorId => { + const key = + `boost.connectors.${connectorId}.__schemaVersion` as keyof typeof boostConfigFields; + expect(isDbWritable(key)).toBe(true); + }, + ); + + it.each(CONNECTOR_IDS)( + '__schemaVersion for %s accepts positive integer', + connectorId => { + const key = + `boost.connectors.${connectorId}.__schemaVersion` as keyof typeof boostConfigFields; + expect(validateConfigValue(key, 1)).toBe(1); + expect(validateConfigValue(key, 2)).toBe(2); + }, + ); + + it.each(CONNECTOR_IDS)( + '__schemaVersion for %s accepts undefined', + connectorId => { + const key = + `boost.connectors.${connectorId}.__schemaVersion` as keyof typeof boostConfigFields; + expect(validateConfigValue(key, undefined)).toBeUndefined(); + }, + ); + + it.each(CONNECTOR_IDS)( + '__schemaVersion for %s rejects negative number', + connectorId => { + const key = + `boost.connectors.${connectorId}.__schemaVersion` as keyof typeof boostConfigFields; + expect(() => validateConfigValue(key, -1)).toThrow(ZodError); + }, + ); + + it.each(CONNECTOR_IDS)( + '__schemaVersion for %s rejects zero', + connectorId => { + const key = + `boost.connectors.${connectorId}.__schemaVersion` as keyof typeof boostConfigFields; + expect(() => validateConfigValue(key, 0)).toThrow(ZodError); + }, + ); + + it.each(CONNECTOR_IDS)( + '__schemaVersion for %s rejects non-integer', + connectorId => { + const key = + `boost.connectors.${connectorId}.__schemaVersion` as keyof typeof boostConfigFields; + expect(() => validateConfigValue(key, 1.5)).toThrow(ZodError); + }, + ); + + it.each(CONNECTOR_IDS)( + '__schemaVersion for %s is not marked as sensitive', + connectorId => { + const key = + `boost.connectors.${connectorId}.__schemaVersion` as keyof typeof boostConfigFields; + expect(isSensitiveField(key)).toBe(false); + }, + ); + }); }); diff --git a/workspaces/boost/plugins/boost-backend/src/config/schemas.ts b/workspaces/boost/plugins/boost-backend/src/config/schemas.ts index fd99b08b2a3..f93e104615c 100644 --- a/workspaces/boost/plugins/boost-backend/src/config/schemas.ts +++ b/workspaces/boost/plugins/boost-backend/src/config/schemas.ts @@ -177,10 +177,38 @@ export interface ConfigFieldMeta { * Current schema version. Stored alongside DB values to detect * schema evolution on startup. * + * Per-connector `__schemaVersion` leaves (`configScope: db-only`) are + * the versioning machinery itself and do not require bumping this + * constant (AGENTS.md "Adding new config fields" step 3). + * * @public */ export const BOOST_CONFIG_SCHEMA_VERSION = 4; +/** + * Current per-connector schema version. Stored as the + * `boost.connectors..__schemaVersion` leaf (configScope: db-only) + * and bumped when connector field semantics change (renames, removals, + * type changes). Missing values are treated as v1. + * + * @public + */ +export const BOOST_CONNECTOR_SCHEMA_VERSION = 1; + +/** + * Known connector identifiers that have registered config leaves. + * + * @public + */ +export const CONNECTOR_IDS = ['jira', 'github', 'gitlab'] as const; + +/** + * Union type of known connector identifiers. + * + * @public + */ +export type ConnectorId = (typeof CONNECTOR_IDS)[number]; + // --------------------------------------------------------------------------- // Connector field factories — shared patterns for per-connector leaves // --------------------------------------------------------------------------- @@ -243,6 +271,22 @@ function connectorBatchSize(label: string) { } as const; } +/** @internal */ +function connectorSchemaVersion(label: string) { + return { + schema: z + .number() + .int() + .positive() + .optional() + .describe('Connector config schema version (internal metadata)'), + configScope: 'db-only' as ConfigScope, + description: + `Per-connector schema version for ${label}. Written during migration, ` + + 'excluded from per-leaf Zod product validation. Missing → v1.', + } as const; +} + // --------------------------------------------------------------------------- // Individual field schemas with metadata // --------------------------------------------------------------------------- @@ -435,6 +479,11 @@ export const boostConfigFields = { 'Defaults to 100 when not set.', }, + // -- Connector schema version (db-only metadata) -- + 'boost.connectors.jira.__schemaVersion': connectorSchemaVersion('Jira'), + 'boost.connectors.github.__schemaVersion': connectorSchemaVersion('GitHub'), + 'boost.connectors.gitlab.__schemaVersion': connectorSchemaVersion('GitLab'), + // -- Connector config: Jira -- 'boost.connectors.jira.enabled': connectorEnabled('Jira', 'jira'), 'boost.connectors.jira.endpoint': connectorEndpoint( diff --git a/workspaces/boost/plugins/boost-backend/src/index.ts b/workspaces/boost/plugins/boost-backend/src/index.ts index 0fcff9d72a5..f21b166ab7a 100644 --- a/workspaces/boost/plugins/boost-backend/src/index.ts +++ b/workspaces/boost/plugins/boost-backend/src/index.ts @@ -40,12 +40,17 @@ export { RuntimeConfigResolver, boostConfigFields, BOOST_CONFIG_SCHEMA_VERSION, + BOOST_CONNECTOR_SCHEMA_VERSION, + CONNECTOR_IDS, validateConfigValue, isDbWritable, isSensitiveField, type AdminConfigServiceOptions, type RuntimeConfigResolverOptions, + type ConnectorMigrationFn, + type ConnectorMigrationRegistry, type BoostConfigKey, + type ConnectorId, type ConfigScope, type ConfigFieldMeta, } from './config'; diff --git a/workspaces/boost/plugins/boost-backend/src/plugin.ts b/workspaces/boost/plugins/boost-backend/src/plugin.ts index d6329423ec2..cfcf95ada40 100644 --- a/workspaces/boost/plugins/boost-backend/src/plugin.ts +++ b/workspaces/boost/plugins/boost-backend/src/plugin.ts @@ -181,6 +181,8 @@ export const boostPlugin = createBackendPlugin({ logger, }); + await runtimeConfigResolver.migrateConnectorSchemas(); + logger.info('Runtime configuration engine initialized'); // Initialize agent lifecycle store