From 22cc66227180e76348c63e9db04eff2746c792f6 Mon Sep 17 00:00:00 2001 From: Chris Chase Date: Wed, 18 Feb 2026 23:15:09 -0500 Subject: [PATCH] refactor: simplify env configuration workflow with improved error handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidate per-service .env files into single .env per environment, reorder deploy dependencies, and improve error handling across scripts. - Consolidate config/dev/.env.* files into single config/dev/.env - Reorder deploy: namespace → postgres → langfuse → mlflow → langflow → app - Fix undeploy to reverse deploy dependency order - Allow user-settable LANGFUSE_INIT_USER_PASSWORD in .env - Auto-inject Langfuse API keys from secrets-dev.yaml into Langflow - Validate eval-based secret generation output (fail on empty) - Add openssl fallback for admin password generation - Make envsubst a hard requirement (remove eval fallback) - Improve rollout status and RBAC error reporting - Fix stale comments referencing removed per-service config vars Co-Authored-By: Claude Opus 4.6 --- .gitignore | 12 +- CLAUDE.md | 14 +- Makefile | 44 +- README.md | 16 +- backend/CLAUDE.md | 2 +- config/dev/.env.backend.example | 66 --- config/dev/.env.example | 63 ++ config/dev/.env.frontend.example | 12 - config/dev/.env.langflow.example | 40 -- config/dev/.env.langfuse.example | 54 -- config/dev/.env.mlflow.example | 20 - config/dev/.env.oauth-proxy.example | 28 - config/dev/.env.postgres.example | 15 - config/dev/allowed-emails.txt.example | 14 +- config/dev/namespace-admins.txt.example | 22 +- config/local/.env.backend.example | 100 ---- config/local/.env.example | 66 +++ config/local/.env.frontend.example | 10 - config/local/.env.langflow.example | 36 -- config/local/.env.langfuse.example | 41 -- config/local/.env.mlflow.example | 14 - config/local/.env.oauth-proxy.example | 18 - config/local/.env.postgres.example | 14 - docs/AUTHENTICATION.md | 27 +- docs/DEPLOYMENT.md | 32 +- docs/DEVELOPMENT.md | 40 +- makefiles/deploy.mk | 12 +- scripts/deploy-app.sh | 4 +- scripts/deploy-langflow.sh | 77 ++- scripts/deploy-langfuse.sh | 48 +- scripts/deploy-mlflow.sh | 3 +- scripts/deploy.sh | 98 +++- scripts/dev-db.sh | 8 +- scripts/dev-langflow.sh | 14 +- scripts/dev-langfuse.sh | 12 +- scripts/dev-mlflow.sh | 12 +- scripts/dev-oauth.sh | 16 +- scripts/generate-config.sh | 751 +++++++++++++----------- scripts/undeploy.sh | 39 +- 39 files changed, 831 insertions(+), 1083 deletions(-) delete mode 100644 config/dev/.env.backend.example create mode 100644 config/dev/.env.example delete mode 100644 config/dev/.env.frontend.example delete mode 100644 config/dev/.env.langflow.example delete mode 100644 config/dev/.env.langfuse.example delete mode 100644 config/dev/.env.mlflow.example delete mode 100644 config/dev/.env.oauth-proxy.example delete mode 100644 config/dev/.env.postgres.example delete mode 100644 config/local/.env.backend.example create mode 100644 config/local/.env.example delete mode 100644 config/local/.env.frontend.example delete mode 100644 config/local/.env.langflow.example delete mode 100644 config/local/.env.langfuse.example delete mode 100644 config/local/.env.mlflow.example delete mode 100644 config/local/.env.oauth-proxy.example delete mode 100644 config/local/.env.postgres.example diff --git a/.gitignore b/.gitignore index a53ecf3..f32c2b0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,11 +13,13 @@ package-lock.json .env.production.local # Centralized config (keep .example files, ignore actual configs) +config/local/.env config/local/.env.* -!config/local/.env.*.example +!config/local/.env.example config/local/flow-sources.yaml +config/dev/.env config/dev/.env.* -!config/dev/.env.*.example +!config/dev/.env.example config/dev/flow-sources.yaml # IDE and Editors @@ -73,12 +75,6 @@ config/dev/namespace-admins.txt helm/*/secrets-*.yaml !helm/*/secrets-*.yaml.example -# Flow sources config (keep .example, ignore actual config) -config/flow-sources.yaml - -# LangFlow env vars (API keys for global variables — keep .example) -config/langflow.env - # Local development overrides .local/ diff --git a/CLAUDE.md b/CLAUDE.md index e958a0e..e884a62 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,7 +42,7 @@ make setup && make services-start && make db-seed && make dev - Deploy to prod: `make deploy-prod` **Troubleshooting:** -- API not working? Check `/api/v1/utils/health-check` → Verify `config/local/.env.*` files → Check CORS settings +- API not working? Check `/api/v1/utils/health-check` → Verify `config/local/.env` → Check CORS settings - Database issues? `make db-status` → `make db-logs` → `make db-shell` ## Project Structure @@ -63,8 +63,8 @@ make setup && make services-start && make db-seed && make dev │ ├── vite.config.ts # Vite configuration with /api proxy │ └── Dockerfile # Frontend container (nginx-based) ├── config/ # Centralized configuration (source of truth) -│ ├── local/ # Local development configs (.env.*.example) -│ └── dev/ # Cluster deployment configs (.env.*.example) +│ ├── local/ # Local development config (.env.example) +│ └── dev/ # Cluster deployment config (.env.example) ├── k8s/ # Kubernetes/OpenShift manifests │ ├── base/ # Base kustomize resources │ └── overlays/ # Environment-specific overlays (dev/prod) @@ -113,7 +113,7 @@ make setup && make services-start && make db-seed && make dev ### Local Development ```bash make setup # Install all dependencies -make env-setup # Copy config/local/.env.*.example to service dirs +make config-setup # Copy config/local/.env.example to config/local/.env make dev # Run both frontend and backend make dev-frontend # Run React dev server (port 8080) make dev-backend # Run FastAPI server (port 8000) @@ -302,7 +302,7 @@ This application uses **OAuth2 Proxy** for authentication. See [docs/AUTHENTICAT - ❌ Missing CORS configuration → Add origins to `backend/app/core/config.py` - ❌ Not using Pydantic models for validation → **ALWAYS** define request/response schemas -- ❌ Hardcoding URLs or sensitive values → Use environment variables (`config/local/.env.*` files for local dev) +- ❌ Hardcoding URLs or sensitive values → Use environment variables (`config/local/.env` for local dev) - ❌ Wrong HTTP status codes → 400 (bad input) vs 404 (not found) vs 409 (conflict) ### Frontend (React/PatternFly) @@ -321,13 +321,13 @@ This application uses **OAuth2 Proxy** for authentication. See [docs/AUTHENTICAT ### Git/Commits - ❌ Not following Conventional Commits → Use `feat:`, `fix:`, `refactor:`, etc. -- ❌ Committing `.env` files with secrets → Use `config/local/.env.*.example` templates +- ❌ Committing `.env` files with secrets → Use `config/local/.env.example` template ## Development Workflow ### Initial Setup 1. Install dependencies: `make setup` -2. Configure environment: `make env-setup` (copies from `config/local/`) +2. Configure environment: `make config-setup` (copies from `config/local/.env.example`) 3. Start all services: `make services-start` 4. Seed test data: `make db-seed` 5. Start development servers: `make dev` diff --git a/Makefile b/Makefile index 86bc3d5..dfaead3 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ include makefiles/helm.mk include makefiles/test.mk .PHONY: help setup setup-frontend setup-backend dev dev-frontend dev-backend dev-2 dev-frontend-2 dev-backend-2 -.PHONY: env-setup sync-version bump-version show-version health-backend health-frontend +.PHONY: config-setup config-reset env-setup sync-version bump-version show-version health-backend health-frontend .PHONY: clean clean-all fresh-start quick-start # Default target @@ -66,36 +66,14 @@ dev-frontend-2: ## Run second frontend instance (port 8081) dev-backend-2: ## Run second backend instance (port 8001) cd backend && uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8001 -# Environment Setup -env-setup: ## Copy config example files for local development - @echo "Setting up environment files from config/local/..." - @for example in config/local/.env.*.example; do \ - actual=$${example%.example}; \ - if [ ! -f "$$actual" ]; then \ - cp "$$example" "$$actual"; \ - echo " Created $$(basename $$actual)"; \ - fi; \ - done - @if [ ! -f config/local/flow-sources.yaml ]; then \ - cp config/local/flow-sources.yaml.example config/local/flow-sources.yaml; \ - echo " Created flow-sources.yaml"; \ - fi - @echo "" - @echo "Copying component configs..." - @if [ ! -f backend/.env ]; then \ - cp config/local/.env.backend backend/.env 2>/dev/null || true; \ - echo " Created backend/.env"; \ - fi - @if [ ! -f frontend/.env ]; then \ - cp config/local/.env.frontend frontend/.env 2>/dev/null || true; \ - echo " Created frontend/.env"; \ - fi - @echo "" - @echo "Config files are in config/local/. Edit them to configure:" - @echo " .env.backend - Database, OAuth, Langflow settings" - @echo " .env.postgres - Database credentials" - @echo " .env.langflow - LLM API keys for Langflow" - @echo " .env.langfuse - Langfuse service credentials" +# Config Setup (local by default, use config-setup-dev for cluster) +config-setup: ## Setup local config from examples + @./scripts/generate-config.sh local + +config-reset: ## Delete all local config (run config-setup after editing .env) + @./scripts/generate-config.sh reset local + +env-setup: config-setup ## Alias for config-setup # Version Management sync-version: ## Sync VERSION to pyproject.toml and package.json @@ -128,7 +106,7 @@ clean: ## Clean build artifacts and dependencies clean-all: clean ## Clean everything # Development Workflow -fresh-start: clean setup env-setup ## Clean setup for new development +fresh-start: clean setup config-setup ## Clean setup for new development @echo "Fresh development environment ready!" -quick-start: setup env-setup dev ## Quick start for development +quick-start: setup config-setup dev ## Quick start for development diff --git a/README.md b/README.md index f5efee7..708b2a1 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ A platform for hosting and orchestrating multiple LangFlow workflows, built on R git clone https://github.com/cfchase/multi-agent-platform cd multi-agent-platform make setup -make env-setup # Copy config from config/local/ +make config-setup # Copy config from config/local/ make services-start make db-seed # optional: load sample data make dev @@ -79,7 +79,7 @@ OAuth is enabled by default. Access app at . - **No OAuth credentials**: Uses mock OAuth server (any username/password works) - **With OAuth credentials**: Uses configured provider (Google, GitHub, Keycloak) -Configure OAuth in `config/local/.env.oauth-proxy` (local) or `config/dev/.env.oauth-proxy` (cluster): +Configure OAuth in `config/local/.env` (local) or `config/dev/.env` (cluster): ```bash OAUTH_CLIENT_ID=your-client-id @@ -98,16 +98,16 @@ For setup details, see [docs/AUTHENTICATION.md](docs/AUTHENTICATION.md). # 1. Login to your cluster oc login --server=https://your-cluster -# 2. Set up deployment config (creates config/dev/ files, auto-generates secrets) -make config-setup +# 2. Set up deployment config (creates config/dev/ files) +make config-setup-cluster -# 3. Configure OAuth credentials (required) -# Edit config/dev/.env.oauth-proxy with your Google OAuth Client ID and Secret -# Edit config/dev/.env.backend with the same OAuth credentials +# 3. Configure credentials (required) +# Edit config/dev/.env with OAuth, LLM keys, etc. # Edit config/dev/allowed-emails.txt with authorized email addresses # Edit config/dev/namespace-admins.txt with OpenShift usernames for admin access -# 4. Deploy everything +# 4. Generate deployment artifacts and deploy +make config-generate make deploy # 5. Verify deployment diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index ef6a043..662ff59 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -571,7 +571,7 @@ class Settings(BaseSettings): settings = Settings() ``` -**IMPORTANT**: Never commit `.env` files with secrets! Config source of truth is in `config/local/.env.backend.example`. Copy to `config/local/.env.backend` and then to `backend/.env` via `make env-setup`. Templates are in `config/local/.env.*.example`. +**IMPORTANT**: Never commit `.env` files with secrets! Config source of truth is `config/local/.env.example`. Run `make config-setup` to copy it to `config/local/.env` and `backend/.env`. ## Additional Resources diff --git a/config/dev/.env.backend.example b/config/dev/.env.backend.example deleted file mode 100644 index 47a468d..0000000 --- a/config/dev/.env.backend.example +++ /dev/null @@ -1,66 +0,0 @@ -# ======================================== -# Backend Configuration (Cluster/Dev) -# ======================================== -# Cluster-appropriate values for OpenShift dev environment. -# Copy to .env.backend and fill in your values. -# Run: make config-setup -# -# Legend: -# Uncommented = REQUIRED (set by user or auto-generated by generate-config.sh) -# Commented = OPTIONAL (uncomment and set if using that feature) - -# --- REQUIRED --- - -# Environment -# - "development": Requires real OAuth (configure .env.oauth-proxy) -ENVIRONMENT=development - -# Frontend URL (used for OAuth redirect URIs and CORS) -# FRONTEND_HOST auto-detects from request URL for OAuth redirects if not set. -# BACKEND_CORS_ORIGINS defaults to empty (same-origin only). Set both if -# frontend and backend are on different origins. -# FRONTEND_HOST=https://your-route-url.apps.your-cluster.com -# BACKEND_CORS_ORIGINS=https://your-route-url.apps.your-cluster.com - -# Security (auto-generated by generate-config.sh if left as placeholder) -SECRET_KEY=changethis_generate_a_secure_random_key - -# ======================================== -# Database Configuration -# ======================================== -# K8s service DNS for PostgreSQL -POSTGRES_SERVER=postgres -POSTGRES_PORT=5432 -POSTGRES_USER=app -# Auto-generated by generate-config.sh if left as 'changethis' -POSTGRES_PASSWORD=changethis -POSTGRES_DB=app - -# ======================================== -# Langflow Configuration -# ======================================== -# Internal K8s service DNS for Langflow -LANGFLOW_URL=http://langflow-service-backend:7860 -# LANGFLOW_DEFAULT_FLOW=My Chat Flow - -# ======================================== -# External Service Integrations (OAuth) -# ======================================== -# Token encryption key for storing OAuth tokens securely. -# REQUIRED if using Google Drive or Dataverse integrations. -# Auto-generated by generate-config.sh if left as placeholder. -TOKEN_ENCRYPTION_KEY=changethis_generate_a_fernet_key - -# --- OPTIONAL (uncomment to enable) --- - -# Google Drive OAuth — lets users connect their Google Drive to the app -# Uses a DIFFERENT redirect URI than the OAuth2 Proxy login. -# Setup: Add redirect URI to your Google OAuth client: -# https:///api/v1/integrations/oauth/callback/google_drive -# Can use the same client ID as .env.oauth-proxy if both redirect URIs are added. -# GOOGLE_CLIENT_ID=your-google-client-id.apps.googleusercontent.com -# GOOGLE_CLIENT_SECRET=your-google-client-secret - -# Dataverse OAuth — defaults to https://mcp.dataverse.redhat.com/auth in code -# Only override if using a different Dataverse instance. -# DATAVERSE_AUTH_URL=https://mcp.dataverse.redhat.com/auth diff --git a/config/dev/.env.example b/config/dev/.env.example new file mode 100644 index 0000000..9d9ed27 --- /dev/null +++ b/config/dev/.env.example @@ -0,0 +1,63 @@ +# Multi-Agent Platform - Cluster (Dev) Configuration +# Copy to .env and fill in your values. +# Run: make config-setup-cluster (then: make config-generate to create deployment artifacts) +# +# Auto-generated values (secrets, internal URLs) go directly into deployment +# artifacts (k8s overlays, Helm secrets). This file is NEVER mutated by scripts. + +# ======================================== +# OAuth (REQUIRED for cluster deployment) +# ======================================== +# Google OAuth Setup: +# 1. Go to https://console.cloud.google.com/apis/credentials +# 2. Create OAuth 2.0 Client ID (Web application) +# 3. Add redirect URI: https:///oauth2/callback +# 4. Copy Client ID and Client Secret below +# See docs/AUTHENTICATION.md for detailed walkthrough +OAUTH_CLIENT_ID=your-client-id +OAUTH_CLIENT_SECRET=your-client-secret +# OAUTH_ISSUER_URL= # Set for OIDC providers (Keycloak, Okta) + +# ======================================== +# LLM API Keys (set at least one) +# ======================================== +# OPENAI_API_KEY= +# ANTHROPIC_API_KEY= +# GEMINI_API_KEY= + +# ======================================== +# Google Drive Integration (optional) +# ======================================== +# Uses a DIFFERENT redirect URI than the OAuth2 Proxy login. +# Add redirect URI: https:///api/v1/integrations/oauth/callback/google_drive +# Can use the same client ID as OAuth above if both redirect URIs are added. +# GOOGLE_CLIENT_ID=your-google-client-id.apps.googleusercontent.com +# GOOGLE_CLIENT_SECRET=your-google-client-secret + +# ======================================== +# Langfuse Admin User (optional) +# ======================================== +# Set to real email/name for the Langfuse web UI admin login. +# Password is auto-generated if not set. Also used for Langflow superuser. +# LANGFUSE_INIT_USER_EMAIL=admin@your-domain.com +# LANGFUSE_INIT_USER_NAME=Admin +# LANGFUSE_INIT_USER_PASSWORD= + +# ======================================== +# Langflow Configuration (optional) +# ======================================== +# LANGFLOW_DEFAULT_FLOW=My Chat Flow + +# ======================================== +# Database Password (optional - auto-generated if not set) +# ======================================== +# Set this only if you need a specific password. +# If left empty or "changethis", generate-config.sh auto-generates one. +# POSTGRES_PASSWORD= + +# ======================================== +# Web Search APIs (optional) +# ======================================== +# TAVILY_API_KEY= +# GOOGLE_API_KEY= +# GOOGLE_CSE_ID= diff --git a/config/dev/.env.frontend.example b/config/dev/.env.frontend.example deleted file mode 100644 index ed765a4..0000000 --- a/config/dev/.env.frontend.example +++ /dev/null @@ -1,12 +0,0 @@ -# Frontend Configuration (Cluster/Dev) -# Only VITE_ prefixed variables are exposed to the browser. -# In production, nginx proxy handles API routing. -# -# Legend: -# Uncommented = REQUIRED -# Commented = OPTIONAL - -# --- REQUIRED --- - -# Empty so relative /api/ paths are used via nginx proxy -VITE_API_URL= diff --git a/config/dev/.env.langflow.example b/config/dev/.env.langflow.example deleted file mode 100644 index c50407e..0000000 --- a/config/dev/.env.langflow.example +++ /dev/null @@ -1,40 +0,0 @@ -# LangFlow Container Environment Variables (Cluster/Dev) -# These are forwarded to the LangFlow pod as environment variables -# so flows can reference them without hardcoding secrets. -# -# Legend: -# Uncommented = REQUIRED (but can be left empty if not using that provider) -# Commented = OPTIONAL - -# --- REQUIRED (set at least one LLM provider your flows use) --- - -OPENAI_API_KEY= -GEMINI_API_KEY= -ANTHROPIC_API_KEY= - -# --- OPTIONAL --- - -# Ollama (typically not used in cluster) -# OLLAMA_BASE_URL= - -# Web Search APIs (only if flows use web search components) -# TAVILY_API_KEY=your-tavily-key-here -# GOOGLE_API_KEY=your-google-api-key-here -# GOOGLE_CSE_ID=your-custom-search-engine-id - -# ======================================== -# Langfuse Tracing (auto-populated by generate-config.sh) -# ======================================== -# Placeholder values below are replaced with real keys during config setup. -# If not using Langfuse, these placeholders are harmless (invalid keys won't connect). -# Values are auto-synced from .env.langfuse by generate-config.sh. -LANGFUSE_SECRET_KEY=sk-your-secret-key -LANGFUSE_PUBLIC_KEY=pk-your-public-key -LANGFUSE_HOST=http://langfuse-web:3000 - -# ======================================== -# Langflow Runtime Configuration -# ======================================== -# Env vars exposed as Langflow global variables (accessible by flow components). -# Note: local dev (scripts/dev-langflow.sh) uses a different set without Langfuse vars. -LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=OPENAI_API_KEY,LANGFUSE_SECRET_KEY,LANGFUSE_PUBLIC_KEY,LANGFUSE_HOST diff --git a/config/dev/.env.langfuse.example b/config/dev/.env.langfuse.example deleted file mode 100644 index d090a98..0000000 --- a/config/dev/.env.langfuse.example +++ /dev/null @@ -1,54 +0,0 @@ -# Langfuse Configuration (Cluster/Dev) -# All secrets auto-generated by: make config-setup -# -# Legend: -# Uncommented = REQUIRED (auto-generated or must be set by user) -# Commented = OPTIONAL - -# --- REQUIRED (auto-generated by generate-config.sh) --- - -# ClickHouse -CLICKHOUSE_USER=default -CLICKHOUSE_PASSWORD=generate-random-value -# Redis -REDIS_PASSWORD=generate-random-value -# MinIO -MINIO_ROOT_USER=minio -MINIO_ROOT_PASSWORD=generate-random-value -# Langfuse Application -# Generate 64-char hex: python -c "import secrets; print(secrets.token_hex(32))" -ENCRYPTION_KEY=generate-random-hex-64-chars -NEXTAUTH_SECRET=generate-random-value -SALT=generate-random-value - -# --- REQUIRED (auto-calculated or must be set by user) --- - -# Langfuse URL (auto-calculated by deploy-langfuse.sh from OpenShift apps domain) -LANGFUSE_NEXTAUTH_URL=auto-calculated-by-deploy-script - -# Langfuse Init (project + API keys) -# Auto-synced to LANGFUSE_SECRET_KEY/PUBLIC_KEY in .env.langflow by generate-config.sh. -# Manual matching is not required when using the config generation script. -LANGFUSE_INIT_ORG_ID=multi-agent-platform -LANGFUSE_INIT_ORG_NAME="Multi-Agent Platform" -LANGFUSE_INIT_PROJECT_ID=default -LANGFUSE_INIT_PROJECT_NAME="Default Project" -LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-your-public-key -LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-your-secret-key -LANGFUSE_INIT_USER_EMAIL=admin@your-domain.com -LANGFUSE_INIT_USER_NAME=Admin -LANGFUSE_INIT_USER_PASSWORD=generate-secure-password - -# --- REQUIRED (auto-synced from .env.postgres by generate-config.sh) --- - -# Database (duplicated from postgres for self-contained config) -POSTGRES_USER=app -POSTGRES_PASSWORD=changethis -LANGFUSE_DB=langfuse -POSTGRES_PORT=5432 - -# --- OPTIONAL --- - -# Langfuse Ports (defaults work for most setups) -LANGFUSE_WEB_PORT=3000 -LANGFUSE_WORKER_PORT=3030 diff --git a/config/dev/.env.mlflow.example b/config/dev/.env.mlflow.example deleted file mode 100644 index 70b8fd3..0000000 --- a/config/dev/.env.mlflow.example +++ /dev/null @@ -1,20 +0,0 @@ -# MLflow Configuration (Cluster/Dev) -# -# Authentication: Handled by OpenShift OAuth proxy sidecar. -# No MLflow-specific auth configuration needed. -# See helm/mlflow/values-dev.yaml for OAuth proxy sidecar config. -# -# Legend: -# Uncommented = REQUIRED -# Commented = OPTIONAL - -# --- REQUIRED (auto-synced from .env.postgres by generate-config.sh) --- - -POSTGRES_USER=app -POSTGRES_PASSWORD=changethis -MLFLOW_DB=mlflow -POSTGRES_PORT=5432 - -# --- OPTIONAL --- - -# MLFLOW_PORT=5000 diff --git a/config/dev/.env.oauth-proxy.example b/config/dev/.env.oauth-proxy.example deleted file mode 100644 index 60ab315..0000000 --- a/config/dev/.env.oauth-proxy.example +++ /dev/null @@ -1,28 +0,0 @@ -# OAuth2 Proxy Configuration (Cluster/Dev) -# REQUIRED for cluster deployment with real OAuth. -# -# Legend: -# Uncommented = REQUIRED (must be set before deploying) -# Commented = OPTIONAL -# -# Google OAuth Setup: -# 1. Go to https://console.cloud.google.com/apis/credentials -# 2. Create OAuth 2.0 Client ID (Web application) -# 3. Add redirect URI: https:///oauth2/callback -# 4. Copy Client ID and Client Secret below -# -# See docs/AUTHENTICATION.md for detailed walkthrough - -# --- REQUIRED --- - -OAUTH_CLIENT_ID=your-client-id -OAUTH_CLIENT_SECRET=your-client-secret - -# Auto-generated by generate-config.sh dev if left as placeholder -# To manually generate: python3 -c "import secrets; print(secrets.token_urlsafe(32))" -OAUTH_COOKIE_SECRET=changethis_generate_a_secure_random_key - -# --- OPTIONAL --- - -# Set OIDC issuer URL to use Keycloak/Okta instead of Google -# OAUTH_ISSUER_URL=https://your-keycloak/realms/your-realm diff --git a/config/dev/.env.postgres.example b/config/dev/.env.postgres.example deleted file mode 100644 index e371e00..0000000 --- a/config/dev/.env.postgres.example +++ /dev/null @@ -1,15 +0,0 @@ -# PostgreSQL Configuration (Cluster/Dev) -# -# Legend: -# Uncommented = REQUIRED -# Commented = OPTIONAL - -# --- REQUIRED --- - -POSTGRES_USER=app -# Auto-generated by make config-setup if left as 'changethis' -POSTGRES_PASSWORD=changethis -POSTGRES_DB=app -POSTGRES_PORT=5432 -# Additional databases created on startup (space-separated) -ADDITIONAL_DBS="langflow langfuse mlflow" diff --git a/config/dev/allowed-emails.txt.example b/config/dev/allowed-emails.txt.example index bf83aba..4166f24 100644 --- a/config/dev/allowed-emails.txt.example +++ b/config/dev/allowed-emails.txt.example @@ -1,5 +1,15 @@ -# Email allowlist for main app (Google OAuth) +# ======================================== +# Email Allowlist for Main App (Google OAuth) +# ======================================== # One email per line. Only these users can log in to the platform. -# Copy to allowed-emails.txt and add your users. +# Lines starting with # are comments (ignored). +# Blank lines are ignored. +# +# Copy this file to allowed-emails.txt and replace with real emails. +# WARNING: Empty file or placeholder emails will prevent all logins. +# +# Example: +# user@company.com +# team-lead@company.com user1@example.com user2@example.com diff --git a/config/dev/namespace-admins.txt.example b/config/dev/namespace-admins.txt.example index 7b52c13..1ee8788 100644 --- a/config/dev/namespace-admins.txt.example +++ b/config/dev/namespace-admins.txt.example @@ -1,7 +1,19 @@ -# OpenShift users with admin access to the namespace. -# One username per line. These users get edit access to the namespace -# and can access Langflow/MLflow via OpenShift OAuth proxy. -# Copy to namespace-admins.txt and add your users. -# Username is the OpenShift username (e.g., from 'oc whoami'). +# ======================================== +# OpenShift Namespace Admins +# ======================================== +# One OpenShift username per line. +# These users get edit access to the namespace and can access +# Langflow and MLflow via OpenShift OAuth proxy. +# Lines starting with # are comments (ignored). +# Blank lines are ignored. +# +# Find your username with: oc whoami +# Copy this file to namespace-admins.txt and replace with real usernames. +# WARNING: Empty file will prevent all admin access to Langflow/MLflow. +# No wildcard patterns — list each user explicitly. +# +# Example: +# jdoe +# admin-user user1 user2 diff --git a/config/local/.env.backend.example b/config/local/.env.backend.example deleted file mode 100644 index 2f877f1..0000000 --- a/config/local/.env.backend.example +++ /dev/null @@ -1,100 +0,0 @@ -# ======================================== -# Backend Configuration -# ======================================== -# This is the source of truth for all server-side configuration. -# Copy to backend/.env and fill in your values. -# -# Legend: -# Uncommented = REQUIRED (must be set for the backend to function) -# Commented = OPTIONAL (uncomment and set if using that feature) - -# --- REQUIRED --- - -# API Configuration -API_V1_STR=/api/v1 -PROJECT_NAME="Multi-Agent Platform" -PORT=8000 - -# Environment -# - "development": Uses OAuth (mock or real) — default for local development -# - "local": Bypasses OAuth entirely with dev-user (quick UI-only work) -ENVIRONMENT=development - -# Frontend Configuration -# Where the frontend runs — used for OAuth redirect after callback. -# Use http://localhost:4180 when running with `make dev` (default, via OAuth proxy). -# Use http://localhost:8080 only with ENVIRONMENT=local (no OAuth proxy). -FRONTEND_HOST=http://localhost:4180 - -# CORS Configuration (comma-separated list) -BACKEND_CORS_ORIGINS=http://localhost:4180,http://localhost:8080,http://localhost:5173 - -# Security -# Generate with: python -c "import secrets; print(secrets.token_urlsafe(32))" -SECRET_KEY=changethis_generate_a_secure_random_key - -# ======================================== -# Database Configuration -# ======================================== -# For local development, use values that match scripts/dev-db.sh -POSTGRES_SERVER=localhost -POSTGRES_PORT=5432 -POSTGRES_USER=app -POSTGRES_PASSWORD=changethis -POSTGRES_DB=app - -# ======================================== -# Langflow Configuration -# ======================================== -# Chat feature uses Langflow for AI responses. -# -# Mock Mode (for testing without Langflow): -# LANGFLOW_URL=mock -# -# Self-hosted Langflow: -# LANGFLOW_URL=http://localhost:7860 -# LANGFLOW_DEFAULT_FLOW=My Chat Flow # Preferred (stable across imports) -# -# Langflow Cloud: -# LANGFLOW_URL=https://api.langflow.astra.datastax.com -# LANGFLOW_ID=your-project-id -# LANGFLOW_DEFAULT_FLOW=My Chat Flow -# LANGFLOW_API_KEY=your-api-key - -# Default: mock mode (simulated responses for testing) -LANGFLOW_URL=mock - -# For real Langflow, uncomment and configure: -# LANGFLOW_URL=http://localhost:7860 -# LANGFLOW_DEFAULT_FLOW=My Chat Flow # Preferred: stable across imports -# LANGFLOW_API_KEY= # Optional for self-hosted -# LANGFLOW_ID= # Required for Langflow Cloud - -# --- OPTIONAL (uncomment to enable) --- - -# ======================================== -# External Service Integrations (OAuth) -# ======================================== -# Users can connect external services to enable AI flows to access their data. -# Tokens are encrypted before storage in the database. - -# Token encryption key for storing OAuth tokens securely. -# REQUIRED if using Google Drive or Dataverse integrations below. -# Generate with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" -# TOKEN_ENCRYPTION_KEY=changethis_generate_a_fernet_key - -# Google Drive OAuth — lets users connect their Google Drive to the app -# Setup: -# 1. Create project at: https://console.cloud.google.com/ -# 2. Enable Google Drive API -# 3. Create OAuth 2.0 credentials (Web application) -# 4. Add redirect URI: http://localhost:8000/api/v1/integrations/oauth/callback/google_drive -# 5. Copy Client ID and Secret below -# GOOGLE_CLIENT_ID=your-google-client-id.apps.googleusercontent.com -# GOOGLE_CLIENT_SECRET=your-google-client-secret - -# Dataverse OAuth (Dynamic Client Registration) -# Defaults to https://mcp.dataverse.redhat.com/auth — only override if needed. -# Uses RFC 7591 dynamic client registration - no client credentials needed. -# Endpoints: {auth_url}/authorize, {auth_url}/token, {auth_url}/register -# DATAVERSE_AUTH_URL=https://mcp.dataverse.redhat.com/auth diff --git a/config/local/.env.example b/config/local/.env.example new file mode 100644 index 0000000..6a77b73 --- /dev/null +++ b/config/local/.env.example @@ -0,0 +1,66 @@ +# Multi-Agent Platform - Local Development Configuration +# +# Copy this file to .env and fill in your values. +# OAuth credentials are required. All other variables are optional. + +# ======================================== +# OAuth (REQUIRED) +# ======================================== +# Google OAuth Setup: +# 1. Go to https://console.cloud.google.com/apis/credentials +# 2. Create OAuth 2.0 Client ID (Web application) +# 3. Add redirect URI: http://localhost:4180/oauth2/callback +# 4. Copy Client ID and Client Secret below +OAUTH_CLIENT_ID= +OAUTH_CLIENT_SECRET= +# OAUTH_ISSUER_URL= # Set for OIDC providers (Keycloak, Okta) + +# ======================================== +# Google Drive Integration (optional) +# ======================================== +# Allows users to connect their Google Drive for document access. +# Add redirect URI: http://localhost:8000/api/v1/integrations/oauth/callback/google_drive +# Can use the same client ID as OAuth above if both redirect URIs are added. +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= + +# ======================================== +# LLM API Keys (optional - only needed for real AI features) +# ======================================== +# Not needed in mock mode. Set at least one when using real Langflow. +# OPENAI_API_KEY= +# ANTHROPIC_API_KEY= +# GEMINI_API_KEY= +# OLLAMA_BASE_URL=http://localhost:11434 + +# ======================================== +# Langflow Configuration (required) +# ======================================== +# Default: mock mode (simulated AI responses for testing) +# Set to http://localhost:7860 to use real Langflow (via make langflow-start). +# LANGFLOW_URL=mock +LANGFLOW_URL=http://localhost:7860 +# LANGFLOW_DEFAULT_FLOW=My Chat Flow + +# ======================================== +# Database Credentials (optional - defaults work for local dev) +# ======================================== +# Only change if you need non-default postgres credentials. +# POSTGRES_USER=app +# POSTGRES_PASSWORD=changethis +# POSTGRES_PORT=5432 + +# ======================================== +# Web Search APIs (optional - only if flows use web search) +# ======================================== +# DuckDuckGo requires no API key (preferred) +# TAVILY_API_KEY= +# GOOGLE_API_KEY= +# GOOGLE_CSE_ID= + +# ======================================== +# Advanced (rarely changed) +# ======================================== +# ENVIRONMENT=development +# FRONTEND_HOST=http://localhost:4180 +# BACKEND_CORS_ORIGINS=http://localhost:4180,http://localhost:8080,http://localhost:5173 diff --git a/config/local/.env.frontend.example b/config/local/.env.frontend.example deleted file mode 100644 index 1f6fa98..0000000 --- a/config/local/.env.frontend.example +++ /dev/null @@ -1,10 +0,0 @@ -# Frontend Configuration (Local) -# Only VITE_ prefixed variables are exposed to the browser. -# -# Legend: -# Uncommented = REQUIRED -# Commented = OPTIONAL - -# --- REQUIRED --- - -VITE_API_URL=http://localhost:8000 diff --git a/config/local/.env.langflow.example b/config/local/.env.langflow.example deleted file mode 100644 index 6b42f6b..0000000 --- a/config/local/.env.langflow.example +++ /dev/null @@ -1,36 +0,0 @@ -# LangFlow Container Environment Variables (Local) -# These are forwarded to the LangFlow container as global variables -# so flows can reference them without hardcoding secrets. -# -# Legend: -# Uncommented = REQUIRED (but can be left empty if not using that provider) -# Commented = OPTIONAL - -# --- REQUIRED (set at least one LLM provider your flows use) --- - -OPENAI_API_KEY= -GEMINI_API_KEY= -ANTHROPIC_API_KEY= - -# --- OPTIONAL --- - -# Ollama base URL (for local LLM inference) -# Default: http://host.containers.internal:11434 (container-to-host networking) -# OLLAMA_BASE_URL=http://host.containers.internal:11434 - -# Web Search APIs (only if flows use web search components) -# DuckDuckGo requires no API key (preferred) -# TAVILY_API_KEY=your-tavily-key-here -# GOOGLE_API_KEY=your-google-api-key-here -# GOOGLE_CSE_ID=your-custom-search-engine-id - -# ======================================== -# Langfuse Tracing -# ======================================== -# Automatically traces all LangFlow executions to Langfuse. -# Default values work with local dev Langfuse (make langfuse-start). -LANGFUSE_SECRET_KEY=sk-dev-secret-key -LANGFUSE_PUBLIC_KEY=pk-dev-public-key -# LANGFUSE_HOST is computed by dev-langflow.sh using container networking. -# Only override if using a non-local Langfuse instance. -# LANGFUSE_HOST=http://host.docker.internal:3000 diff --git a/config/local/.env.langfuse.example b/config/local/.env.langfuse.example deleted file mode 100644 index 82a114c..0000000 --- a/config/local/.env.langfuse.example +++ /dev/null @@ -1,41 +0,0 @@ -# Langfuse Development Configuration (Local) -# -# Legend: -# Uncommented = REQUIRED -# Commented = OPTIONAL - -# --- REQUIRED --- - -# ClickHouse -CLICKHOUSE_USER=default -CLICKHOUSE_PASSWORD=clickhouse -# Redis -REDIS_PASSWORD=redis -# MinIO -MINIO_ROOT_USER=minio -MINIO_ROOT_PASSWORD=miniosecret -# Langfuse Application -ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 -NEXTAUTH_SECRET=mysecret -SALT=saltysaltysaltysaltysaltysaltysalty -# Langfuse Init (dev user + project) -LANGFUSE_INIT_ORG_ID=multi-agent-platform -LANGFUSE_INIT_ORG_NAME="Multi-Agent Platform" -LANGFUSE_INIT_PROJECT_ID=default -LANGFUSE_INIT_PROJECT_NAME="Default Project" -LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-dev-public-key -LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-dev-secret-key -LANGFUSE_INIT_USER_EMAIL=dev@localhost.local -LANGFUSE_INIT_USER_NAME=Developer -LANGFUSE_INIT_USER_PASSWORD=devpassword123 -# Database (duplicated from postgres for self-contained config) -POSTGRES_USER=app -POSTGRES_PASSWORD=changethis -LANGFUSE_DB=langfuse -POSTGRES_PORT=5432 - -# --- OPTIONAL --- - -# Langfuse Ports (defaults work for most setups) -LANGFUSE_WEB_PORT=3000 -LANGFUSE_WORKER_PORT=3030 diff --git a/config/local/.env.mlflow.example b/config/local/.env.mlflow.example deleted file mode 100644 index f51fd97..0000000 --- a/config/local/.env.mlflow.example +++ /dev/null @@ -1,14 +0,0 @@ -# MLflow Development Configuration (Local) -# -# Legend: -# Uncommented = REQUIRED -# Commented = OPTIONAL - -# --- REQUIRED --- - -MLFLOW_PORT=5000 -# Database (duplicated from postgres for self-contained config) -POSTGRES_USER=app -POSTGRES_PASSWORD=changethis -MLFLOW_DB=mlflow -POSTGRES_PORT=5432 diff --git a/config/local/.env.oauth-proxy.example b/config/local/.env.oauth-proxy.example deleted file mode 100644 index e2fbac5..0000000 --- a/config/local/.env.oauth-proxy.example +++ /dev/null @@ -1,18 +0,0 @@ -# OAuth2 Proxy Configuration (Local) -# Required for ENVIRONMENT=development in backend. -# When no credentials are set, a mock OAuth server is used. -# -# Legend: -# Uncommented = REQUIRED (must be set for real OAuth) -# Commented = OPTIONAL - -# --- REQUIRED (for real OAuth — leave commented for mock server) --- - -# OAUTH_CLIENT_ID=your-client-id -# OAUTH_CLIENT_SECRET=your-client-secret -OAUTH_COOKIE_SECRET=changethis_generate_a_secure_random_key - -# --- OPTIONAL --- - -# Provider (leave empty for Google default, or set OIDC issuer URL) -# OAUTH_ISSUER_URL=https://your-keycloak/realms/your-realm diff --git a/config/local/.env.postgres.example b/config/local/.env.postgres.example deleted file mode 100644 index d48fb8f..0000000 --- a/config/local/.env.postgres.example +++ /dev/null @@ -1,14 +0,0 @@ -# PostgreSQL Development Configuration (Local) -# -# Legend: -# Uncommented = REQUIRED -# Commented = OPTIONAL - -# --- REQUIRED --- - -POSTGRES_USER=app -POSTGRES_PASSWORD=changethis -POSTGRES_DB=app -POSTGRES_PORT=5432 -# Additional databases created on startup (space-separated) -ADDITIONAL_DBS="langflow langfuse mlflow" diff --git a/docs/AUTHENTICATION.md b/docs/AUTHENTICATION.md index 2023c17..b696fee 100644 --- a/docs/AUTHENTICATION.md +++ b/docs/AUTHENTICATION.md @@ -100,7 +100,7 @@ OAuth running - access app at: http://localhost:4180 For quick UI development without OAuth overhead: ```bash -# Edit config/local/.env.backend (or backend/.env after make env-setup) +# Edit config/local/.env ENVIRONMENT=local FRONTEND_HOST=http://localhost:8080 @@ -119,7 +119,7 @@ In local mode: ## Real OAuth Providers -For testing with actual OAuth providers (Google, GitHub, Keycloak), configure credentials in `config/local/.env.oauth-proxy` (local) or `config/dev/.env.oauth-proxy` (cluster). +For testing with actual OAuth providers (Google, GitHub, Keycloak), configure credentials in `config/local/.env` (local) or `config/dev/.env` (cluster). ### Environment Variables @@ -202,16 +202,12 @@ Authorized redirect URIs: ```bash # Set up environment from config/local/ templates -make env-setup +make config-setup ``` -Add these values to `config/local/.env.backend`: +Add these values to `config/local/.env`: ```bash ENVIRONMENT=development -``` - -Add these values to `config/local/.env.oauth-proxy`: -```bash OAUTH_CLIENT_ID=your-client-id.apps.googleusercontent.com OAUTH_CLIENT_SECRET=your-client-secret # OAUTH_COOKIE_SECRET is auto-generated if not set @@ -255,21 +251,18 @@ Authorized redirect URIs: ```bash # Generate config files from templates -make config-setup +make config-setup-cluster # Or manually copy: -cp config/dev/.env.oauth-proxy.example config/dev/.env.oauth-proxy +cp config/dev/.env.example config/dev/.env # Edit with your credentials ``` -Contents of `config/dev/.env.oauth-proxy`: +Set OAuth values in `config/dev/.env`: ```bash -# Generate cookie secret: python -c "import secrets; print(secrets.token_urlsafe(32))" -cookie-secret=your-generated-cookie-secret - -# From Google Cloud Console -client-id=your-client-id.apps.googleusercontent.com -client-secret=your-client-secret +OAUTH_COOKIE_SECRET=your-generated-cookie-secret +GOOGLE_CLIENT_ID=your-client-id.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=your-client-secret ``` **Deploy:** diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index cd072fa..cd671c2 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -96,9 +96,9 @@ make push-prod # Uses TAG=prod ``` config/ ├── local/ # Local development configs -│ └── .env.*.example # Per-service config templates +│ └── .env.example # Consolidated config template ├── dev/ # Cluster deployment configs -│ └── .env.*.example # Per-service config templates +│ └── .env.example # Consolidated config template │ k8s/ ├── app/ # Deep Research app (Kustomize) @@ -134,7 +134,7 @@ helm/ └── values-dev.yaml │ scripts/ # Called via make targets (not directly) -├── generate-config.sh # make config-setup +├── generate-config.sh # make config-setup-cluster / config-generate ├── verify-deployment.sh # make verify-deploy ├── deploy.sh # make deploy ├── deploy-db.sh # make deploy-db @@ -213,31 +213,29 @@ The application uses a **consolidated pod deployment** with multiple containers: ### OAuth2 Proxy Secret Setup -**CRITICAL**: Before deploying, you must create the OAuth2 proxy secret file. +**CRITICAL**: Before deploying, you must configure OAuth credentials. -1. **Generate config files from templates:** +1. **Set up configuration:** ```bash - # For development (generates all config/dev/ files) - make config-setup + # Generate config from template + make config-setup-cluster - # Or manually copy the OAuth proxy config: - cp config/dev/.env.oauth-proxy.example config/dev/.env.oauth-proxy + # Or manually copy the consolidated config: + cp config/dev/.env.example config/dev/.env ``` -2. **Generate a cookie secret:** +2. **Edit `config/dev/.env` with your OAuth provider credentials:** ```bash - openssl rand -base64 32 | tr -- '+/' '-_' + OAUTH_CLIENT_ID=your-oauth-client-id + OAUTH_CLIENT_SECRET=your-oauth-client-secret ``` -3. **Edit the config file with your OAuth provider credentials:** +3. **Generate deployment artifacts** (secrets are auto-generated): ```bash - # config/dev/.env.oauth-proxy - client-id=your-oauth-client-id - client-secret=your-oauth-client-secret - cookie-secret= + make config-generate ``` -4. **The config file is gitignored** - never commit OAuth secrets! +4. **The `.env` file is gitignored** - never commit OAuth secrets! See [AUTHENTICATION.md](AUTHENTICATION.md) for OAuth provider configuration details. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 0c62fe2..a26df79 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -20,9 +20,9 @@ cd multi-agent-platform # 2. Install all dependencies make setup -# 3. Configure environment (copies from config/local/) -make env-setup -# Edit config/local/.env.backend if needed (default: ENVIRONMENT=development with OAuth) +# 3. Configure environment (copies config/local/.env.example to config/local/.env) +make config-setup +# Edit config/local/.env if needed (default: ENVIRONMENT=development with OAuth) # 4. Start PostgreSQL database make db-start @@ -111,37 +111,34 @@ git status ### Environment Files -Configuration templates live in `config/local/` (source of truth): +Configuration is consolidated into a single file per environment: ``` -config/local/.env.backend.example # Backend configuration template -config/local/.env.frontend.example # Frontend configuration template -config/local/.env.langflow.example # Langflow configuration template -config/local/.env.langfuse.example # Langfuse configuration template -config/local/.env.mlflow.example # MLflow configuration template -config/local/.env.oauth-proxy.example # OAuth proxy configuration template -config/local/.env.postgres.example # PostgreSQL configuration template +config/local/.env.example # Local development configuration template (all services) +config/dev/.env.example # Cluster deployment configuration template (all services) ``` -Copy from examples: +Copy from example: ```bash -make env-setup # Copies config/local/.env.*.example to service directories +make config-setup # Copies config/local/.env.example to config/local/.env ``` +Edit `config/local/.env` to configure LLM API keys, OAuth credentials, database settings, Langflow options, and other service variables. + ### Authentication Modes **OAuth Mode (default):** ```bash -# In config/local/.env.backend (or backend/.env after env-setup) +# In config/local/.env ENVIRONMENT=development ``` - Uses OAuth proxy on port 4180 (mock OAuth if no credentials configured) - Access the app at `http://localhost:4180` -- For real OAuth, configure `config/local/.env.oauth-proxy` — see [AUTHENTICATION.md](AUTHENTICATION.md) +- For real OAuth, configure OAuth credentials in `config/local/.env` -- see [AUTHENTICATION.md](AUTHENTICATION.md) **No-Auth Mode (quick UI-only work):** ```bash -# In config/local/.env.backend (or backend/.env after env-setup) +# In config/local/.env ENVIRONMENT=local ``` - No OAuth required, uses a default "dev-user" for all requests @@ -149,7 +146,7 @@ ENVIRONMENT=local ### Key Variables -**Backend (`config/local/.env.backend`):** +All variables are in `config/local/.env`: ```bash ENVIRONMENT=development # development (OAuth, default) or local (no auth) POSTGRES_SERVER=localhost @@ -159,11 +156,6 @@ POSTGRES_DB=app POSTGRES_PORT=5432 ``` -**Frontend (`config/local/.env.frontend`):** -```bash -VITE_API_URL=/api # Proxy handled by Vite -``` - ## Database Management ### Common Commands @@ -223,7 +215,7 @@ make mlflow-stop - Configure flows for the Chat feature via environment variables: ```bash -# In config/local/.env.backend for mock mode (no Langflow required): +# In config/local/.env for mock mode (no Langflow required): LANGFLOW_URL=mock # For local Langflow: @@ -352,7 +344,7 @@ curl http://localhost:8000/api/v1/utils/health-check # (visible in terminal running `make dev-backend`) # Verify .env configuration -cat config/local/.env.backend +cat config/local/.env ``` ### Frontend Issues diff --git a/makefiles/deploy.mk b/makefiles/deploy.mk index af26571..16f13a0 100644 --- a/makefiles/deploy.mk +++ b/makefiles/deploy.mk @@ -5,7 +5,7 @@ .PHONY: deploy-db deploy-app deploy-langflow deploy-mlflow deploy-langfuse .PHONY: generate-k8s-secrets generate-k8s-secrets-dev generate-k8s-secrets-prod .PHONY: generate-langfuse-secrets-dev generate-admin-secret-dev get-admin-credentials -.PHONY: config-setup config-reset +.PHONY: config-setup-cluster config-reset-cluster config-generate # Generate K8s secrets from backend/.env (if they don't exist) generate-k8s-secrets-dev: @@ -68,13 +68,15 @@ get-admin-credentials: ## Show admin credentials and URLs for LangFlow/Langfuse/ generate-langfuse-secrets-dev: @./scripts/generate-langfuse-secrets.sh -# Config setup and reset -config-setup: ## Generate cluster config from examples (skips existing files) +# Cluster config setup, reset, and artifact generation +config-setup-cluster: ## Setup cluster config from examples (skips existing files) @./scripts/generate-config.sh dev -config-reset: ## Delete and regenerate all cluster config (use when moving to a new cluster) +config-reset-cluster: ## Delete all cluster config (run config-setup-cluster after editing .env) @./scripts/generate-config.sh reset dev - @./scripts/generate-config.sh dev + +config-generate: ## Generate k8s/helm deployment artifacts from config/dev/.env + @./scripts/generate-config.sh all # Individual component deployment targets deploy-db: ## Deploy PostgreSQL database only diff --git a/scripts/deploy-app.sh b/scripts/deploy-app.sh index 5f2829d..ff89819 100755 --- a/scripts/deploy-app.sh +++ b/scripts/deploy-app.sh @@ -52,10 +52,10 @@ if [[ ! -f "$OAUTH_SECRET_FILE" ]]; then fi # Check if backend config source exists -BACKEND_CONFIG_SOURCE="$PROJECT_ROOT/config/dev/.env.backend" +BACKEND_CONFIG_SOURCE="$PROJECT_ROOT/config/dev/.env" if [[ ! -f "$BACKEND_CONFIG_SOURCE" ]]; then echo "Error: Backend config not found at $BACKEND_CONFIG_SOURCE" - echo "Please copy config/dev/.env.backend.example to config/dev/.env.backend and configure it." + echo "Please copy config/dev/.env.example to config/dev/.env and configure it." exit 1 fi diff --git a/scripts/deploy-langflow.sh b/scripts/deploy-langflow.sh index de845f3..029fe15 100755 --- a/scripts/deploy-langflow.sh +++ b/scripts/deploy-langflow.sh @@ -61,35 +61,69 @@ if [[ ! -f "$VALUES_FILE" ]]; then exit 1 fi -# Create langflow-credentials secret from config/dev/.env.langflow +# Create langflow-credentials secret from consolidated .env # Contains LLM API keys and tracing config — loaded via envFrom in the post-renderer patch -LANGFLOW_ENV_FILE="$PROJECT_ROOT/config/$ENVIRONMENT/.env.langflow" +# Only extract Langflow-relevant keys (LLM API keys, search APIs) +LANGFLOW_ENV_FILE="$PROJECT_ROOT/config/$ENVIRONMENT/.env" +LANGFLOW_KEYS="OPENAI_API_KEY|GEMINI_API_KEY|ANTHROPIC_API_KEY|OLLAMA_BASE_URL|TAVILY_API_KEY|GOOGLE_API_KEY|GOOGLE_CSE_ID|LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT" + +LITERAL_ARGS=() if [[ -f "$LANGFLOW_ENV_FILE" ]]; then - # Filter to non-empty, non-comment key=value lines - LANGFLOW_CREDS=$(grep -v '^\s*#' "$LANGFLOW_ENV_FILE" | grep -v '^\s*$' | grep '=.' || true) + # Filter to Langflow-relevant, non-empty, non-comment key=value lines + LANGFLOW_CREDS=$(grep -v '^\s*#' "$LANGFLOW_ENV_FILE" | grep -v '^\s*$' | grep -E "^(${LANGFLOW_KEYS})=." || true) if [[ -n "$LANGFLOW_CREDS" ]]; then - # Build --from-literal args for each non-empty key - LITERAL_ARGS=() while IFS= read -r line; do key="${line%%=*}" value="${line#*=}" [[ -z "$key" || -z "$value" ]] && continue LITERAL_ARGS+=("--from-literal=${key}=${value}") done <<< "$LANGFLOW_CREDS" + fi +else + echo "Warning: $LANGFLOW_ENV_FILE not found" +fi - if [[ ${#LITERAL_ARGS[@]} -gt 0 ]]; then - if oc get secret langflow-credentials -n "$NAMESPACE" &>/dev/null; then - oc delete secret langflow-credentials -n "$NAMESPACE" - fi - oc create secret generic langflow-credentials \ - "${LITERAL_ARGS[@]}" -n "$NAMESPACE" - echo "Created langflow-credentials secret (${#LITERAL_ARGS[@]} keys)" - fi +# Inject Langfuse tracing keys from generated secrets-dev.yaml +# These are auto-generated API keys, not user-settable, so read from the artifact +LANGFUSE_SECRETS="$PROJECT_ROOT/helm/langfuse/secrets-${ENVIRONMENT}.yaml" +if [[ -f "$LANGFUSE_SECRETS" ]]; then + # Extract additionalEnv value by name from YAML (e.g., "- name: KEY\n value: VAL") + _yaml_env_val() { awk "/name: ${1}\$/{getline; gsub(/.*value: *\"?/,\"\"); gsub(/\"? *$/,\"\"); print; exit}" "$2" 2>/dev/null; } + LF_PK=$(_yaml_env_val "LANGFUSE_INIT_PROJECT_PUBLIC_KEY" "$LANGFUSE_SECRETS") + LF_SK=$(_yaml_env_val "LANGFUSE_INIT_PROJECT_SECRET_KEY" "$LANGFUSE_SECRETS") + if [[ -n "$LF_PK" ]]; then + LITERAL_ARGS+=("--from-literal=LANGFUSE_PUBLIC_KEY=${LF_PK}") + fi + if [[ -n "$LF_SK" ]]; then + LITERAL_ARGS+=("--from-literal=LANGFUSE_SECRET_KEY=${LF_SK}") + fi + if [[ -z "$LF_PK" || -z "$LF_SK" ]]; then + echo "Warning: Could not extract Langfuse API keys from $LANGFUSE_SECRETS" + echo " LANGFUSE_PUBLIC_KEY: ${LF_PK:-(empty)}" + echo " LANGFUSE_SECRET_KEY: ${LF_SK:-(empty)}" + echo " Langflow tracing may not work. Verify secrets-${ENVIRONMENT}.yaml format." else - echo "No non-empty values in $LANGFLOW_ENV_FILE — skipping langflow-credentials secret" + echo "Added Langfuse tracing keys from secrets-${ENVIRONMENT}.yaml" fi + # Langfuse internal service URL (Helm release "langfuse", service "langfuse-web" on port 3000) + LITERAL_ARGS+=("--from-literal=LANGFUSE_HOST=http://langfuse-web:3000") else - echo "Warning: $LANGFLOW_ENV_FILE not found — langflow-credentials secret not created" + echo "Warning: $LANGFUSE_SECRETS not found — Langfuse tracing not configured" +fi + +if [[ ${#LITERAL_ARGS[@]} -gt 0 ]]; then + if oc get secret langflow-credentials -n "$NAMESPACE" &>/dev/null; then + if ! oc delete secret langflow-credentials -n "$NAMESPACE"; then + echo "Error: Failed to delete existing langflow-credentials secret." + echo " Check RBAC permissions: oc auth can-i delete secrets -n $NAMESPACE" + exit 1 + fi + fi + oc create secret generic langflow-credentials \ + "${LITERAL_ARGS[@]}" -n "$NAMESPACE" + echo "Created langflow-credentials secret (${#LITERAL_ARGS[@]} keys)" +else + echo "No credentials to configure — skipping langflow-credentials secret" fi # Add Helm repo @@ -149,7 +183,13 @@ oc apply -f "$PROJECT_ROOT/k8s/langflow/base/route.yaml" -n "$NAMESPACE" # Wait for LangFlow to be ready (Helm chart creates a StatefulSet, not Deployment) echo "Waiting for LangFlow to be ready (this may take a few minutes)..." -oc rollout status statefulset/${RELEASE_NAME}-service -n "$NAMESPACE" --timeout=300s || true +if ! oc rollout status statefulset/${RELEASE_NAME}-service -n "$NAMESPACE" --timeout=300s; then + echo "" + echo "Warning: LangFlow did not become ready within 300s" + echo " Check pod status: oc get pods -n $NAMESPACE -l app=langflow-service" + echo " Check logs: oc logs -n $NAMESPACE -l app=langflow-service --tail=50" + echo " Continuing with remaining deployments..." +fi # Get route URL ROUTE_URL=$(oc get route langflow -n "$NAMESPACE" -o jsonpath='{.spec.host}' 2>/dev/null || echo "") @@ -167,5 +207,4 @@ echo "Internal backend access via langflow-service-backend:7860 bypasses OAuth." echo "" echo "Check status: oc get pods -n $NAMESPACE -l app=langflow-service" echo "" -echo "NOTE: Backend reads Langflow URL from backend-config secret." -echo " Ensure config/dev/.env.backend has LANGFLOW_URL set correctly." +echo "Backend connects to Langflow via internal service URL (langflow-service-backend:7860, auto-configured)." diff --git a/scripts/deploy-langfuse.sh b/scripts/deploy-langfuse.sh index b89a926..670ddf7 100755 --- a/scripts/deploy-langfuse.sh +++ b/scripts/deploy-langfuse.sh @@ -56,45 +56,34 @@ if ! oc get secret admin-credentials -n "$NAMESPACE" &> /dev/null; then fi # Auto-calculate LANGFUSE_NEXTAUTH_URL from OpenShift apps domain +# Exported as env var for generate-config.sh envsubst pipeline (never written to user config files) APPS_DOMAIN_ERR=$(mktemp) APPS_DOMAIN=$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}' 2>"$APPS_DOMAIN_ERR" || echo "") if [[ -n "$APPS_DOMAIN" ]]; then - LANGFUSE_NEXTAUTH_URL="https://langfuse-${NAMESPACE}.${APPS_DOMAIN}" - LANGFUSE_ENV_FILE="$PROJECT_ROOT/config/${ENVIRONMENT}/.env.langfuse" - if [[ -f "$LANGFUSE_ENV_FILE" ]]; then - if grep -q "^LANGFUSE_NEXTAUTH_URL=" "$LANGFUSE_ENV_FILE"; then - sed -i.bak "s|^LANGFUSE_NEXTAUTH_URL=.*|LANGFUSE_NEXTAUTH_URL=${LANGFUSE_NEXTAUTH_URL}|" "$LANGFUSE_ENV_FILE" - rm -f "${LANGFUSE_ENV_FILE}.bak" - else - echo "LANGFUSE_NEXTAUTH_URL=${LANGFUSE_NEXTAUTH_URL}" >> "$LANGFUSE_ENV_FILE" - fi - # Verify the write succeeded - actual_url=$(grep "^LANGFUSE_NEXTAUTH_URL=" "$LANGFUSE_ENV_FILE" | cut -d= -f2-) - if [ "$actual_url" != "$LANGFUSE_NEXTAUTH_URL" ]; then - echo "Error: Failed to update LANGFUSE_NEXTAUTH_URL in $LANGFUSE_ENV_FILE" - rm -f "$APPS_DOMAIN_ERR" - exit 1 - fi - echo "Set LANGFUSE_NEXTAUTH_URL=${LANGFUSE_NEXTAUTH_URL}" - fi + export LANGFUSE_NEXTAUTH_URL="https://langfuse-${NAMESPACE}.${APPS_DOMAIN}" + echo "Auto-calculated LANGFUSE_NEXTAUTH_URL=${LANGFUSE_NEXTAUTH_URL}" else echo "Warning: Could not detect OpenShift apps domain." if [[ -s "$APPS_DOMAIN_ERR" ]]; then echo " Reason: $(cat "$APPS_DOMAIN_ERR")" fi - # Fail if LANGFUSE_NEXTAUTH_URL is still a placeholder - if grep -q "^LANGFUSE_NEXTAUTH_URL=auto-calculated-by-deploy-script" "$PROJECT_ROOT/config/${ENVIRONMENT}/.env.langfuse" 2>/dev/null; then - echo "Error: LANGFUSE_NEXTAUTH_URL is still a placeholder and cannot be auto-calculated." - echo "Please set it manually in config/${ENVIRONMENT}/.env.langfuse" + # Check if LANGFUSE_NEXTAUTH_URL is set from consolidated .env + LANGFUSE_ENV_FILE="$PROJECT_ROOT/config/${ENVIRONMENT}/.env" + if [[ -f "$LANGFUSE_ENV_FILE" ]]; then + set -a; source "$LANGFUSE_ENV_FILE"; set +a + fi + if [[ -z "$LANGFUSE_NEXTAUTH_URL" || "$LANGFUSE_NEXTAUTH_URL" == "auto-calculated-by-deploy-script" ]]; then + echo "Error: LANGFUSE_NEXTAUTH_URL cannot be auto-calculated and is not set in config/${ENVIRONMENT}/.env" + echo "Please set it manually in config/${ENVIRONMENT}/.env" echo "Example: LANGFUSE_NEXTAUTH_URL=https://langfuse-${NAMESPACE}.apps.your-cluster.example.com" rm -f "$APPS_DOMAIN_ERR" exit 1 fi - echo "Using existing LANGFUSE_NEXTAUTH_URL value." + echo "Using LANGFUSE_NEXTAUTH_URL from config/${ENVIRONMENT}/.env" fi rm -f "$APPS_DOMAIN_ERR" -# Always regenerate secrets from source of truth (config/dev/.env.langfuse) +# Always regenerate secrets from source of truth (config/dev/.env) SECRETS_FILE="$PROJECT_ROOT/helm/langfuse/secrets-${ENVIRONMENT}.yaml" echo "Generating Langfuse secrets..." "$SCRIPT_DIR/generate-config.sh" helm-langfuse --force @@ -140,7 +129,13 @@ oc create route edge langfuse --service="${RELEASE_NAME}-web" --port=3000 -n "$N # Wait for Langfuse to be ready echo "Waiting for Langfuse to be ready..." -oc rollout status deployment/${RELEASE_NAME}-web -n "$NAMESPACE" --timeout=300s || true +if ! oc rollout status deployment/${RELEASE_NAME}-web -n "$NAMESPACE" --timeout=300s; then + echo "" + echo "Warning: Langfuse did not become ready within 300s" + echo " Check pod status: oc get pods -n $NAMESPACE -l app.kubernetes.io/instance=langfuse" + echo " Check logs: oc logs -n $NAMESPACE -l app.kubernetes.io/name=langfuse-web --tail=50" + echo " Continuing with remaining deployments..." +fi # Get route URL ROUTE_URL=$(oc get route langfuse -n "$NAMESPACE" -o jsonpath='{.spec.host}' 2>/dev/null || echo "") @@ -158,5 +153,4 @@ echo " make get-admin-credentials" echo "" echo "Check status: oc get pods -n $NAMESPACE -l app.kubernetes.io/instance=langfuse" echo "" -echo "NOTE: Backend reads Langfuse URL from backend-config secret." -echo " Ensure config/dev/.env.backend has LANGFUSE_HOST set correctly." +echo "Backend connects to Langfuse via internal service URL (langfuse-web:3000, auto-configured)." diff --git a/scripts/deploy-mlflow.sh b/scripts/deploy-mlflow.sh index 943839c..f339f60 100755 --- a/scripts/deploy-mlflow.sh +++ b/scripts/deploy-mlflow.sh @@ -146,5 +146,4 @@ echo "" echo "Check status: oc get pods -n $NAMESPACE -l app.kubernetes.io/instance=mlflow" echo " (Should show 2/2 containers ready: mlflow + oauth-proxy)" echo "" -echo "NOTE: Backend reads MLflow URI from backend-config secret." -echo " Ensure config/dev/.env.backend has MLFLOW_TRACKING_URI set correctly." +echo "Backend connects to MLflow via internal service URL (mlflow:5000, auto-configured)." diff --git a/scripts/deploy.sh b/scripts/deploy.sh index ad70816..9219329 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -28,18 +28,18 @@ check_deploy_prerequisites() { check_openshift_login # Config files - local required_configs=(".env.backend" ".env.oauth-proxy" ".env.postgres") + local required_configs=(".env") for cfg in "${required_configs[@]}"; do if [ ! -f "$config_dir/$cfg" ]; then log_error "Missing config: $config_dir/$cfg" - log_error "Run: ./scripts/generate-config.sh $environment" + log_error "Run: cp $config_dir/.env.example $config_dir/.env" exit 1 fi done # Check OAuth credentials are not placeholder - if grep -q "OAUTH_CLIENT_ID=your-client-id" "$config_dir/.env.oauth-proxy" 2>/dev/null; then - log_error "OAuth credentials not configured in $config_dir/.env.oauth-proxy" + if grep -q "OAUTH_CLIENT_ID=your-client-id" "$config_dir/.env" 2>/dev/null; then + log_error "OAuth credentials not configured in $config_dir/.env" echo "" echo " To set up Google OAuth:" echo " 1. Go to https://console.cloud.google.com/apis/credentials" @@ -59,8 +59,7 @@ check_deploy_prerequisites() { echo " Authorized redirect URI: https:///oauth2/callback" fi echo "" - echo " 3. Copy the Client ID and Secret into $config_dir/.env.oauth-proxy" - echo " and $config_dir/.env.backend" + echo " 3. Copy the Client ID and Secret into $config_dir/.env" echo "" echo " See docs/AUTHENTICATION.md for detailed walkthrough" exit 1 @@ -110,23 +109,51 @@ echo "Environment: $ENVIRONMENT" echo "Namespace: $NAMESPACE" echo "" -# Generate all k8s secrets from config/dev/ before any component deploys -echo "Generating secrets from config/${ENVIRONMENT}/..." -"$SCRIPT_DIR/generate-config.sh" k8s --force +# Deploy in dependency order: +# 1. Namespace + secrets/credentials/OAuth (prerequisites) +# 2. PostgreSQL (database — needed by all services) +# 3. Langfuse (tracing — generates API keys needed by Langflow) +# 4. MLflow (experiment tracking — independent) +# 5. Langflow (workflow engine — depends on Langfuse keys) +# 6. App (frontend/backend — depends on all service credentials) + +# Step 1: Ensure namespace exists, generate secrets, set up cluster prerequisites +echo "Ensuring namespace $NAMESPACE exists..." +oc create namespace "$NAMESPACE" --dry-run=client -o yaml | oc apply -f - echo "" -# Deploy components in order -# Note: App deploys LAST so it picks up all credential secrets from AI tools - -echo "Step 1/5: Deploying PostgreSQL..." -"$SCRIPT_DIR/deploy-db.sh" "$ENVIRONMENT" "$NAMESPACE" +echo "Generating k8s secrets from config/${ENVIRONMENT}/..." +"$SCRIPT_DIR/generate-config.sh" k8s --force echo "" -# Generate admin credentials (required by LangFlow and Langfuse) +# Admin credentials (shared by LangFlow and Langfuse) +# Uses LANGFUSE_INIT_USER_PASSWORD from user's .env or auto-generates echo "Generating admin credentials if needed..." if ! oc get secret admin-credentials -n "$NAMESPACE" &> /dev/null; then - ADMIN_EMAIL="admin@localhost.local" - ADMIN_PASS=$(python3 -c "import secrets; print(secrets.token_urlsafe(16))") + CONFIG_ENV="$PROJECT_ROOT/config/$ENVIRONMENT/.env" + + # Resolve admin password: user .env > auto-generate + ADMIN_PASS="" + if [ -f "$CONFIG_ENV" ]; then + ADMIN_PASS=$(grep -E "^LANGFUSE_INIT_USER_PASSWORD=" "$CONFIG_ENV" 2>/dev/null | cut -d= -f2- | sed 's/^["'"'"']//;s/["'"'"']$//') + fi + if [ -z "$ADMIN_PASS" ]; then + echo "No LANGFUSE_INIT_USER_PASSWORD found in .env — auto-generating admin password" + ADMIN_PASS=$(python3 -c "import secrets; print(secrets.token_urlsafe(16))" 2>/dev/null || openssl rand -base64 16 2>/dev/null || "") + if [ -z "$ADMIN_PASS" ]; then + echo "Error: Cannot auto-generate admin password (need python3 or openssl)" + echo " Set LANGFUSE_INIT_USER_PASSWORD in config/$ENVIRONMENT/.env instead" + exit 1 + fi + fi + + # Resolve admin email from user .env or default + ADMIN_EMAIL="" + if [ -f "$CONFIG_ENV" ]; then + ADMIN_EMAIL=$(grep -E "^LANGFUSE_INIT_USER_EMAIL=" "$CONFIG_ENV" 2>/dev/null | cut -d= -f2- | sed 's/^["'"'"']//;s/["'"'"']$//') + fi + ADMIN_EMAIL="${ADMIN_EMAIL:-admin@localhost.local}" + oc create secret generic admin-credentials \ --from-literal=email="$ADMIN_EMAIL" \ --from-literal=password="$ADMIN_PASS" \ @@ -139,19 +166,16 @@ else fi echo "" -# Setup namespace admin group from config (for Langflow/MLflow OAuth access) +# Namespace admin group (for Langflow/MLflow OAuth access) ADMINS_FILE="$PROJECT_ROOT/config/$ENVIRONMENT/namespace-admins.txt" GROUP_NAME="${NAMESPACE}-admins" if [ -f "$ADMINS_FILE" ]; then echo "Setting up namespace admin group: $GROUP_NAME" - # Create group if it doesn't exist if ! oc get group "$GROUP_NAME" &> /dev/null 2>&1; then oc adm groups new "$GROUP_NAME" echo "Created group: $GROUP_NAME" fi - # Add users from config file while IFS= read -r user || [ -n "$user" ]; do - # Skip comments and empty lines [[ "$user" =~ ^#.*$ || -z "$user" ]] && continue user=$(echo "$user" | tr -d '[:space:]') if ! oc get group "$GROUP_NAME" -o jsonpath='{.users[*]}' 2>/dev/null | grep -qw "$user"; then @@ -160,31 +184,43 @@ if [ -f "$ADMINS_FILE" ]; then echo " Already in group: $user" fi done < "$ADMINS_FILE" - # Grant edit role on namespace - oc adm policy add-role-to-group edit "$GROUP_NAME" -n "$NAMESPACE" 2>/dev/null - echo "Granted edit role to $GROUP_NAME in $NAMESPACE" + if ! oc adm policy add-role-to-group edit "$GROUP_NAME" -n "$NAMESPACE" 2>&1; then + echo "Warning: Failed to grant edit role to $GROUP_NAME in $NAMESPACE" + echo " You may need cluster-admin permissions to assign roles" + else + echo "Granted edit role to $GROUP_NAME in $NAMESPACE" + fi else echo "No namespace-admins.txt found at $ADMINS_FILE - skipping group setup" fi echo "" -# Setup shared OAuth resources BEFORE individual service deploys +# Shared OAuth resources (ServiceAccount, session secret) ensure_supporting_services_oauth "$NAMESPACE" echo "" -echo "Step 2/5: Deploying LangFlow..." -"$SCRIPT_DIR/deploy-langflow.sh" "$ENVIRONMENT" "$NAMESPACE" +# Step 2: PostgreSQL (all services depend on this) +echo "Step 2/6: Deploying PostgreSQL..." +"$SCRIPT_DIR/deploy-db.sh" "$ENVIRONMENT" "$NAMESPACE" echo "" -echo "Step 3/5: Deploying MLFlow..." +# Step 3: Langfuse (generates secrets-dev.yaml with API keys needed by Langflow) +echo "Step 3/6: Deploying Langfuse..." +"$SCRIPT_DIR/deploy-langfuse.sh" "$ENVIRONMENT" "$NAMESPACE" +echo "" + +# Step 4: MLflow (independent) +echo "Step 4/6: Deploying MLFlow..." "$SCRIPT_DIR/deploy-mlflow.sh" "$ENVIRONMENT" "$NAMESPACE" echo "" -echo "Step 4/5: Deploying Langfuse..." -"$SCRIPT_DIR/deploy-langfuse.sh" "$ENVIRONMENT" "$NAMESPACE" +# Step 5: Langflow (reads Langfuse API keys from secrets-dev.yaml) +echo "Step 5/6: Deploying LangFlow..." +"$SCRIPT_DIR/deploy-langflow.sh" "$ENVIRONMENT" "$NAMESPACE" echo "" -echo "Step 5/5: Deploying Multi-Agent Platform App..." +# Step 6: App (depends on all service credentials) +echo "Step 6/6: Deploying Multi-Agent Platform App..." "$SCRIPT_DIR/deploy-app.sh" "$ENVIRONMENT" "$NAMESPACE" echo "" diff --git a/scripts/dev-db.sh b/scripts/dev-db.sh index 4040722..ed0feab 100755 --- a/scripts/dev-db.sh +++ b/scripts/dev-db.sh @@ -16,10 +16,10 @@ init_container_tool || exit 1 # Configuration PROJECT_ROOT="${SCRIPT_DIR}/.." -# Load config from centralized config directory -POSTGRES_CONFIG="$PROJECT_ROOT/config/local/.env.postgres" -if [ -f "$POSTGRES_CONFIG" ]; then - set -a; source "$POSTGRES_CONFIG"; set +a +# Load consolidated config +CONFIG_FILE="$PROJECT_ROOT/config/local/.env" +if [ -f "$CONFIG_FILE" ]; then + set -a; source "$CONFIG_FILE"; set +a fi POSTGRES_VERSION="${POSTGRES_VERSION:-15-alpine}" diff --git a/scripts/dev-langflow.sh b/scripts/dev-langflow.sh index 0109074..b0a490b 100755 --- a/scripts/dev-langflow.sh +++ b/scripts/dev-langflow.sh @@ -20,10 +20,10 @@ LANGFLOW_PORT="${LANGFLOW_PORT:-7860}" PROJECT_ROOT="${SCRIPT_DIR}/.." DATA_DIR="${PROJECT_ROOT}/.local/langflow" -# Load PostgreSQL config from centralized config directory -POSTGRES_CONFIG="$PROJECT_ROOT/config/local/.env.postgres" -if [ -f "$POSTGRES_CONFIG" ]; then - set -a; source "$POSTGRES_CONFIG"; set +a +# Load consolidated config +CONFIG_FILE="$PROJECT_ROOT/config/local/.env" +if [ -f "$CONFIG_FILE" ]; then + set -a; source "$CONFIG_FILE"; set +a fi # Database connection (connects to shared PostgreSQL) @@ -46,11 +46,7 @@ get_db_host() { DB_HOST=$(get_db_host) DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}" -# Load LangFlow-specific env vars (API keys for global variables, etc.) -LANGFLOW_CONFIG="$PROJECT_ROOT/config/local/.env.langflow" -if [ -f "$LANGFLOW_CONFIG" ]; then - set -a; source "$LANGFLOW_CONFIG"; set +a -fi +# LangFlow-specific env vars (API keys, etc.) are loaded from consolidated config above case "$1" in start) diff --git a/scripts/dev-langfuse.sh b/scripts/dev-langfuse.sh index c051ec2..4e2648e 100755 --- a/scripts/dev-langfuse.sh +++ b/scripts/dev-langfuse.sh @@ -17,14 +17,10 @@ init_container_tool || exit 1 # Configuration PROJECT_ROOT="${SCRIPT_DIR}/.." -# Load config from centralized config directory -LANGFUSE_CONFIG="$PROJECT_ROOT/config/local/.env.langfuse" -if [ -f "$LANGFUSE_CONFIG" ]; then - set -a; source "$LANGFUSE_CONFIG"; set +a -fi -POSTGRES_CONFIG="$PROJECT_ROOT/config/local/.env.postgres" -if [ -f "$POSTGRES_CONFIG" ]; then - set -a; source "$POSTGRES_CONFIG"; set +a +# Load consolidated config +CONFIG_FILE="$PROJECT_ROOT/config/local/.env" +if [ -f "$CONFIG_FILE" ]; then + set -a; source "$CONFIG_FILE"; set +a fi LANGFUSE_VERSION="${LANGFUSE_VERSION:-3}" diff --git a/scripts/dev-mlflow.sh b/scripts/dev-mlflow.sh index 1729543..89bfcf5 100755 --- a/scripts/dev-mlflow.sh +++ b/scripts/dev-mlflow.sh @@ -20,14 +20,10 @@ MLFLOW_PORT="${MLFLOW_PORT:-5000}" PROJECT_ROOT="${SCRIPT_DIR}/.." DATA_DIR="${PROJECT_ROOT}/.local/mlflow" -# Load config from centralized config directory -MLFLOW_CONFIG="$PROJECT_ROOT/config/local/.env.mlflow" -if [ -f "$MLFLOW_CONFIG" ]; then - set -a; source "$MLFLOW_CONFIG"; set +a -fi -POSTGRES_CONFIG="$PROJECT_ROOT/config/local/.env.postgres" -if [ -f "$POSTGRES_CONFIG" ]; then - set -a; source "$POSTGRES_CONFIG"; set +a +# Load consolidated config +CONFIG_FILE="$PROJECT_ROOT/config/local/.env" +if [ -f "$CONFIG_FILE" ]; then + set -a; source "$CONFIG_FILE"; set +a fi # Database connection (connects to shared PostgreSQL) diff --git a/scripts/dev-oauth.sh b/scripts/dev-oauth.sh index e4caa15..61a4baf 100755 --- a/scripts/dev-oauth.sh +++ b/scripts/dev-oauth.sh @@ -4,7 +4,7 @@ # Runs oauth2-proxy locally for testing OAuth authentication flow # Supports Google OAuth (default) or OIDC providers (Keycloak, Okta, etc.) # -# Requires OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET in config/local/.env.oauth-proxy +# Requires OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET in config/local/.env set -e @@ -20,10 +20,10 @@ OAUTH_PROXY_CONTAINER="app-oauth-proxy-dev" OAUTH_PORT="${OAUTH_PORT:-4180}" OAUTH_PROXY_IMAGE="quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" -# Load OAuth config from centralized config directory -OAUTH_CONFIG="${SCRIPT_DIR}/../config/local/.env.oauth-proxy" -if [ -f "$OAUTH_CONFIG" ]; then - set -a; source "$OAUTH_CONFIG"; set +a +# Load consolidated config +CONFIG_FILE="${SCRIPT_DIR}/../config/local/.env" +if [ -f "$CONFIG_FILE" ]; then + set -a; source "$CONFIG_FILE"; set +a fi # Ensure cookie secret exists @@ -56,7 +56,7 @@ is_oauth_configured() { start_oauth() { if ! is_oauth_configured; then log_warn "OAuth not configured - skipping OAuth2 Proxy" - log_info "To enable OAuth, set OAUTH_CLIENT_ID/SECRET in config/local/.env.oauth-proxy" + log_info "To enable OAuth, set OAUTH_CLIENT_ID/SECRET in config/local/.env" log_info "Using ENVIRONMENT=local with dev-user for local development" return 0 fi @@ -138,13 +138,13 @@ show_help() { echo "Usage: $0 {start|stop|status|logs|help}" echo "" echo "Commands:" - echo " start Start OAuth2 Proxy (requires credentials in config/local/.env.oauth-proxy)" + echo " start Start OAuth2 Proxy (requires credentials in config/local/.env)" echo " stop Stop OAuth2 Proxy" echo " status Check OAuth2 Proxy status" echo " logs Follow OAuth2 Proxy logs" echo " help Show this help message" echo "" - echo "Required environment variables in config/local/.env.oauth-proxy:" + echo "Required environment variables in config/local/.env:" echo " OAUTH_CLIENT_ID OAuth Client ID" echo " OAUTH_CLIENT_SECRET OAuth Client Secret" echo "" diff --git a/scripts/generate-config.sh b/scripts/generate-config.sh index bd71e43..5e612ae 100755 --- a/scripts/generate-config.sh +++ b/scripts/generate-config.sh @@ -4,7 +4,7 @@ # Reads from config/ as source of truth, generates target-specific formats # # Usage: ./scripts/generate-config.sh [command] -# Commands: local, k8s, helm-langfuse, all, help +# Commands: local, dev, k8s, helm-langfuse, all, reset, help set -e @@ -43,6 +43,127 @@ copy_if_missing() { fi } +# Copy source to dest, warning if dest exists and differs +warn_on_diff() { + local src="$1" + local dest="$2" + local label="${3:-$dest}" + if [ ! -f "$dest" ]; then + cp "$src" "$dest" + log_info "Created: $label" + elif diff -q "$src" "$dest" >/dev/null 2>&1; then + log_info "Up to date: $label" + else + log_warn "File differs from source: $label" + diff --color "$src" "$dest" 2>/dev/null || diff "$src" "$dest" || true + log_warn "To update: cp $src $dest" + fi +} + +# Validate access control files for cluster deployment +validate_access_control_files() { + local config_dir="$1" + local warnings=0 + + local emails_file="$config_dir/allowed-emails.txt" + if [ ! -f "$emails_file" ]; then + log_warn "Missing: $emails_file (no users will be able to log in)" + warnings=$((warnings + 1)) + else + local email_count + email_count=$(grep -cv '^\s*#\|^\s*$' "$emails_file" 2>/dev/null || echo 0) + if [ "$email_count" -eq 0 ]; then + log_warn "Empty: $emails_file (no users will be able to log in)" + warnings=$((warnings + 1)) + elif grep -q "example.com" "$emails_file"; then + log_warn "Placeholder emails in: $emails_file — replace with real addresses before deploying" + warnings=$((warnings + 1)) + fi + fi + + local admins_file="$config_dir/namespace-admins.txt" + if [ ! -f "$admins_file" ]; then + log_warn "Missing: $admins_file (no admin access to Langflow/MLflow)" + warnings=$((warnings + 1)) + else + local admin_count + admin_count=$(grep -cv '^\s*#\|^\s*$' "$admins_file" 2>/dev/null || echo 0) + if [ "$admin_count" -eq 0 ]; then + log_warn "Empty: $admins_file (no admin access to Langflow/MLflow)" + warnings=$((warnings + 1)) + elif grep -q "^user[12]$" "$admins_file"; then + log_warn "Placeholder usernames in: $admins_file — replace with real OpenShift usernames before deploying" + warnings=$((warnings + 1)) + fi + fi + + return $warnings +} + +# Get secret value: user-provided > existing artifact > generate new +# Implements idempotent secret generation — re-runs preserve existing values +get_or_generate_secret() { + local user_val="$1" # Value from user's .env (may be empty/placeholder) + local artifact_file="$2" # Path to existing artifact + local artifact_key="$3" # Grep key in artifact (e.g., "POSTGRES_PASSWORD" or "cookie-secret") + local placeholder="$4" # Placeholder string to detect + local gen_cmd="$5" # Command to generate new value + + # 1. User explicitly set a non-placeholder value + if [ -n "$user_val" ] && [ "$user_val" != "$placeholder" ]; then + echo "$user_val" + return + fi + + # 2. Reuse from existing artifact (idempotent) + if [ -f "$artifact_file" ]; then + local existing + existing=$(grep "^${artifact_key}=" "$artifact_file" 2>/dev/null | head -1 | cut -d= -f2-) + if [ -n "$existing" ] && [ "$existing" != "$placeholder" ]; then + echo "$existing" + return + fi + fi + + # 3. Generate new value + local generated + generated=$(eval "$gen_cmd" 2>/dev/null) || true + if [ -z "$generated" ]; then + log_error "Failed to generate value for $artifact_key" + return 1 + fi + echo "$generated" +} + +# Extract a value from YAML artifact by searching for "name: KEY" then grabbing next "value:" line +# Used for idempotent Langfuse secret reuse from secrets-dev.yaml +get_yaml_env_value() { + local yaml_file="$1" + local env_name="$2" + if [ ! -f "$yaml_file" ]; then + echo "" + return + fi + # Find line with "name: ENV_NAME" and get the value from the next line + local value + value=$(awk "/name: ${env_name}\$/{getline; gsub(/.*value: *\"?/,\"\"); gsub(/\"? *$/,\"\"); print; exit}" "$yaml_file" 2>/dev/null || echo "") + echo "$value" +} + +# Extract simple YAML value by key path (e.g., "password:" under a specific section) +# Returns first match — use for unique keys only +get_yaml_simple_value() { + local yaml_file="$1" + local key="$2" # e.g., "password:" — matches first occurrence + if [ ! -f "$yaml_file" ]; then + echo "" + return + fi + local value + value=$(grep "${key}" "$yaml_file" 2>/dev/null | head -1 | sed 's/.*: *"\{0,1\}\([^"]*\)"\{0,1\} *$/\1/') + echo "$value" +} + # envsubst with explicit variable list, or fallback if envsubst not available envsubst_or_fallback() { local template="$1" @@ -52,11 +173,8 @@ envsubst_or_fallback() { if command -v envsubst &> /dev/null; then envsubst "$var_list" < "$template" > "$output" else - log_warn "envsubst not found, using shell variable expansion fallback" - # Read template and substitute variables using eval - while IFS= read -r line; do - eval "echo \"$line\"" - done < "$template" > "$output" + log_error "envsubst is required but not found. Install gettext-base (apt) or gettext (brew)." + return 1 fi } @@ -68,356 +186,200 @@ envsubst_or_fallback() { cmd_local() { log_info "Setting up local development config..." - # Copy all .env.*.example files to .env.* if they don't exist - for example_file in "$CONFIG_LOCAL"/.env.*.example; do - if [ -f "$example_file" ]; then - local target="${example_file%.example}" - copy_if_missing "$example_file" "$target" - fi - done + # Copy consolidated .env.example to .env if it doesn't exist + copy_if_missing "$CONFIG_LOCAL/.env.example" "$CONFIG_LOCAL/.env" # Copy flow-sources.yaml.example if present if [ -f "$CONFIG_LOCAL/flow-sources.yaml.example" ]; then copy_if_missing "$CONFIG_LOCAL/flow-sources.yaml.example" "$CONFIG_LOCAL/flow-sources.yaml" fi - # Copy backend and frontend .env files if not present - if [ -f "$CONFIG_LOCAL/.env.backend" ]; then - copy_if_missing "$CONFIG_LOCAL/.env.backend" "$PROJECT_ROOT/backend/.env" + # Always sync config/local/.env to backend/.env (source of truth is config/local/.env) + if [ -f "$CONFIG_LOCAL/.env" ]; then + cp "$CONFIG_LOCAL/.env" "$PROJECT_ROOT/backend/.env" + log_info "Synced: backend/.env ← config/local/.env" fi - if [ -f "$CONFIG_LOCAL/.env.frontend" ]; then - copy_if_missing "$CONFIG_LOCAL/.env.frontend" "$PROJECT_ROOT/frontend/.env" + + # Create minimal frontend/.env if it doesn't exist + if [ -f "$PROJECT_ROOT/frontend/.env" ]; then + log_info "Already exists: frontend/.env" + else + echo "VITE_API_URL=http://localhost:8000" > "$PROJECT_ROOT/frontend/.env" + log_info "Created: frontend/.env (VITE_API_URL=http://localhost:8000)" fi + echo "" log_info "Local config setup complete" + echo " config/local/.env - Set LLM keys and any overrides" + echo " backend/.env - Copied from config/local/.env" + echo " frontend/.env - VITE_API_URL (defaults to localhost:8000)" } -# Setup cluster dev config files from examples, auto-generating secrets +# Setup cluster dev config files from examples (no secret generation, no mutation) cmd_dev() { log_info "Setting up cluster dev config from examples..." - local created=0 - - # Copy all .env.*.example files to .env.* if they don't exist - for example_file in "$CONFIG_DEV"/.env.*.example; do - if [ -f "$example_file" ]; then - local target="${example_file%.example}" - if [ -f "$target" ]; then - log_info "Already exists: $target" - else - cp "$example_file" "$target" - created=$((created + 1)) - log_info "Created: $target" - fi - fi - done - - # Copy non-.env example files (text configs) - for example_file in "$CONFIG_DEV"/*.example; do - if [ -f "$example_file" ]; then - # Skip .env.* files (already handled above) - case "$(basename "$example_file")" in - .env.*) continue ;; - esac - local target="${example_file%.example}" - if [ -f "$target" ]; then - log_info "Already exists: $target" - else - cp "$example_file" "$target" - created=$((created + 1)) - log_info "Created: $target" - fi - fi - done - - if [ "$created" -eq 0 ]; then - log_info "All config files already exist" - fi - - # Auto-generate secrets (replaces placeholders — safe to re-run) - log_info "Checking for placeholder secrets..." - - # POSTGRES_PASSWORD in .env.postgres, synced to .env.backend, .env.langfuse, .env.mlflow - if [ -f "$CONFIG_DEV/.env.postgres" ]; then - local pg_pass="" - if grep -q "POSTGRES_PASSWORD=changethis" "$CONFIG_DEV/.env.postgres"; then - pg_pass=$(python3 -c "import secrets; print(secrets.token_urlsafe(16))" 2>/dev/null || openssl rand -base64 16) - sed -i.bak "s|POSTGRES_PASSWORD=changethis|POSTGRES_PASSWORD=${pg_pass}|" "$CONFIG_DEV/.env.postgres" - rm -f "$CONFIG_DEV/.env.postgres.bak" - log_info "Generated POSTGRES_PASSWORD" + # Copy consolidated .env.example to .env if it doesn't exist + if [ ! -f "$CONFIG_DEV/.env" ]; then + if [ -f "$CONFIG_DEV/.env.example" ]; then + cp "$CONFIG_DEV/.env.example" "$CONFIG_DEV/.env" + log_info "Created: config/dev/.env" else - # Read existing password for syncing to other files - pg_pass=$(grep "^POSTGRES_PASSWORD=" "$CONFIG_DEV/.env.postgres" | cut -d= -f2) - fi - - # Sync POSTGRES_PASSWORD to all files that duplicate it - if [ -n "$pg_pass" ]; then - for sync_file in "$CONFIG_DEV/.env.backend" "$CONFIG_DEV/.env.langfuse" "$CONFIG_DEV/.env.mlflow"; do - if [ -f "$sync_file" ] && grep -q "POSTGRES_PASSWORD=changethis" "$sync_file"; then - sed -i.bak "s|POSTGRES_PASSWORD=changethis|POSTGRES_PASSWORD=${pg_pass}|" "$sync_file" - rm -f "${sync_file}.bak" - fi - done + log_error "config/dev/.env.example not found — cannot initialize config" + exit 1 fi + else + log_info "Already exists: config/dev/.env" fi - # SECRET_KEY in .env.backend - if [ -f "$CONFIG_DEV/.env.backend" ]; then - local secret_key - secret_key=$(python3 -c "import secrets; print(secrets.token_urlsafe(32))" 2>/dev/null || openssl rand -base64 32) - sed -i.bak "s|SECRET_KEY=changethis_generate_a_secure_random_key|SECRET_KEY=${secret_key}|" "$CONFIG_DEV/.env.backend" - rm -f "$CONFIG_DEV/.env.backend.bak" - log_info "Generated SECRET_KEY" + # Copy access control and flow-sources example files if missing + copy_if_missing "$CONFIG_DEV/allowed-emails.txt.example" "$CONFIG_DEV/allowed-emails.txt" + copy_if_missing "$CONFIG_DEV/namespace-admins.txt.example" "$CONFIG_DEV/namespace-admins.txt" + if [ -f "$CONFIG_DEV/flow-sources.yaml.example" ]; then + copy_if_missing "$CONFIG_DEV/flow-sources.yaml.example" "$CONFIG_DEV/flow-sources.yaml" fi - # TOKEN_ENCRYPTION_KEY in .env.backend - if [ -f "$CONFIG_DEV/.env.backend" ]; then - local token_key - token_key=$(python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" 2>/dev/null || "") - if [ -n "$token_key" ]; then - sed -i.bak "s|TOKEN_ENCRYPTION_KEY=changethis_generate_a_fernet_key|TOKEN_ENCRYPTION_KEY=${token_key}|" "$CONFIG_DEV/.env.backend" - rm -f "$CONFIG_DEV/.env.backend.bak" - log_info "Generated TOKEN_ENCRYPTION_KEY" - else - log_warn "Could not auto-generate TOKEN_ENCRYPTION_KEY (cryptography package not available)" - fi - fi + # Validate access control files + validate_access_control_files "$CONFIG_DEV" || true - # Langfuse secrets in .env.langfuse - if [ -f "$CONFIG_DEV/.env.langfuse" ]; then - # ENCRYPTION_KEY - 64-char hex - local enc_key - enc_key=$(python3 -c "import secrets; print(secrets.token_hex(32))" 2>/dev/null || openssl rand -hex 32) - sed -i.bak "s|ENCRYPTION_KEY=generate-random-hex-64-chars|ENCRYPTION_KEY=${enc_key}|" "$CONFIG_DEV/.env.langfuse" - rm -f "$CONFIG_DEV/.env.langfuse.bak" - - # NEXTAUTH_SECRET, SALT, passwords - for placeholder_var in NEXTAUTH_SECRET SALT CLICKHOUSE_PASSWORD REDIS_PASSWORD MINIO_ROOT_PASSWORD LANGFUSE_INIT_USER_PASSWORD; do - local gen_val - gen_val=$(python3 -c "import secrets; print(secrets.token_urlsafe(24))" 2>/dev/null || openssl rand -base64 24) - sed -i.bak "s|${placeholder_var}=generate-random-value|${placeholder_var}=${gen_val}|" "$CONFIG_DEV/.env.langfuse" - rm -f "$CONFIG_DEV/.env.langfuse.bak" - sed -i.bak "s|${placeholder_var}=generate-secure-password|${placeholder_var}=${gen_val}|" "$CONFIG_DEV/.env.langfuse" - rm -f "$CONFIG_DEV/.env.langfuse.bak" - done + log_info "Dev config setup complete" + echo "" + echo "Configure these files before deploying:" + echo " config/dev/.env - Set OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET, LLM keys" + echo " config/dev/allowed-emails.txt - Email addresses for main app access" + echo " config/dev/namespace-admins.txt - OpenShift usernames for Langflow/MLflow access" + echo "" + echo "Then run: make config-generate (generates deployment artifacts with auto-generated secrets)" +} - log_info "Generated Langfuse secrets" +# Generate Kubernetes secret .env files from config/dev/.env (single source) +cmd_k8s() { + local force="${1:-}" - # Generate Langfuse API keys (pk-/sk- prefixed) if still placeholders - if grep -q "LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-your-public-key" "$CONFIG_DEV/.env.langfuse"; then - local pk_suffix - pk_suffix=$(python3 -c "import secrets; print(secrets.token_urlsafe(24))" 2>/dev/null || openssl rand -base64 24 | tr '+/' '-_' | tr -d '=') - sed -i.bak "s|LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-your-public-key|LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-${pk_suffix}|" "$CONFIG_DEV/.env.langfuse" - rm -f "$CONFIG_DEV/.env.langfuse.bak" - log_info "Generated LANGFUSE_INIT_PROJECT_PUBLIC_KEY" - fi - if grep -q "LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-your-secret-key" "$CONFIG_DEV/.env.langfuse"; then - local sk_suffix - sk_suffix=$(python3 -c "import secrets; print(secrets.token_urlsafe(24))" 2>/dev/null || openssl rand -base64 24 | tr '+/' '-_' | tr -d '=') - sed -i.bak "s|LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-your-secret-key|LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-${sk_suffix}|" "$CONFIG_DEV/.env.langfuse" - rm -f "$CONFIG_DEV/.env.langfuse.bak" - log_info "Generated LANGFUSE_INIT_PROJECT_SECRET_KEY" - fi + log_info "Generating Kubernetes secret files from config/dev/.env..." + + if [ ! -f "$CONFIG_DEV/.env" ]; then + log_error "Config not found: config/dev/.env — run 'generate-config.sh dev' first" + exit 1 fi - # Sync Langfuse API keys from .env.langfuse into .env.langflow - if [ -f "$CONFIG_DEV/.env.langfuse" ] && [ -f "$CONFIG_DEV/.env.langflow" ]; then - local lf_pk lf_sk - lf_pk=$(grep "^LANGFUSE_INIT_PROJECT_PUBLIC_KEY=" "$CONFIG_DEV/.env.langfuse" | cut -d= -f2-) - lf_sk=$(grep "^LANGFUSE_INIT_PROJECT_SECRET_KEY=" "$CONFIG_DEV/.env.langfuse" | cut -d= -f2-) - - local synced=0 - if [ -n "$lf_pk" ] && [ "$lf_pk" != "pk-your-public-key" ]; then - sed -i.bak "s|^LANGFUSE_PUBLIC_KEY=.*|LANGFUSE_PUBLIC_KEY=${lf_pk}|" "$CONFIG_DEV/.env.langflow" - rm -f "$CONFIG_DEV/.env.langflow.bak" - synced=1 - fi - if [ -n "$lf_sk" ] && [ "$lf_sk" != "sk-your-secret-key" ]; then - sed -i.bak "s|^LANGFUSE_SECRET_KEY=.*|LANGFUSE_SECRET_KEY=${lf_sk}|" "$CONFIG_DEV/.env.langflow" - rm -f "$CONFIG_DEV/.env.langflow.bak" - synced=1 - fi + load_env "$CONFIG_DEV/.env" - # Ensure LANGFUSE_HOST is uncommented and set - if grep -q "^# LANGFUSE_HOST=" "$CONFIG_DEV/.env.langflow"; then - sed -i.bak "s|^# LANGFUSE_HOST=.*|LANGFUSE_HOST=http://langfuse-web:3000|" "$CONFIG_DEV/.env.langflow" - rm -f "$CONFIG_DEV/.env.langflow.bak" - synced=1 - fi + # --- Artifact file paths --- + local oauth_target="$PROJECT_ROOT/k8s/app/overlays/dev/oauth-proxy-secret.env" + local postgres_target="$PROJECT_ROOT/k8s/postgres/overlays/dev/postgres-secret.env" + local langflow_target="$PROJECT_ROOT/k8s/langflow/overlays/dev/langflow-secret.env" + local backend_target="$PROJECT_ROOT/k8s/app/overlays/dev/backend-config.env" + local emails_target="$PROJECT_ROOT/k8s/app/overlays/dev/allowed-emails.txt" - if [ "$synced" -eq 1 ]; then - log_info "Synced Langfuse keys to .env.langflow" - else - log_warn "Langfuse keys not yet generated - skipping sync to .env.langflow" - fi - fi + # --- Compute secrets (idempotent: user-provided > existing artifact > generate) --- - # OAUTH_COOKIE_SECRET in .env.oauth-proxy - if [ -f "$CONFIG_DEV/.env.oauth-proxy" ]; then - local cookie_secret - cookie_secret=$(python3 -c "import secrets; print(secrets.token_urlsafe(32))" 2>/dev/null || openssl rand -base64 32) - sed -i.bak "s|OAUTH_COOKIE_SECRET=changethis_generate_a_secure_random_key|OAUTH_COOKIE_SECRET=${cookie_secret}|" "$CONFIG_DEV/.env.oauth-proxy" - rm -f "$CONFIG_DEV/.env.oauth-proxy.bak" - log_info "Generated OAUTH_COOKIE_SECRET" - fi + local pg_password + pg_password=$(get_or_generate_secret \ + "${POSTGRES_PASSWORD:-}" "$postgres_target" "POSTGRES_PASSWORD" "changethis" \ + "python3 -c \"import secrets; print(secrets.token_urlsafe(16))\" 2>/dev/null || openssl rand -base64 16") - log_info "Dev config setup complete" - echo "" - echo "Files created in config/dev/. You still need to set:" - echo " .env.oauth-proxy - OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET (Google OAuth)" - echo " .env.backend - GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET (Google Drive integration, optional)" - echo " .env.langflow - OPENAI_API_KEY (or other LLM keys)" - echo " allowed-emails.txt - Email addresses for main app access" - echo " namespace-admins.txt - OpenShift usernames for Langflow/MLflow access" -} - -# Generate Kubernetes secret .env files from config/dev/ -cmd_k8s() { - local force="${1:-}" + local pg_user="${POSTGRES_USER:-app}" + local pg_db="${POSTGRES_DB:-app}" + local pg_port="${POSTGRES_PORT:-5432}" - log_info "Generating Kubernetes secret files from config/dev/..." + local secret_key + secret_key=$(get_or_generate_secret \ + "${SECRET_KEY:-}" "$backend_target" "SECRET_KEY" "" \ + "python3 -c \"import secrets; print(secrets.token_urlsafe(32))\" 2>/dev/null || openssl rand -base64 32") - # --- Pre-generate passwords that must be consistent across sections --- - local _generated_postgres_password="" - if [ -f "$CONFIG_DEV/.env.postgres" ]; then - load_env "$CONFIG_DEV/.env.postgres" - if [ -z "$POSTGRES_PASSWORD" ] || [ "$POSTGRES_PASSWORD" = "changethis" ]; then - _generated_postgres_password=$(python3 -c "import secrets; print(secrets.token_urlsafe(16))" 2>/dev/null || openssl rand -base64 16) - log_info "Auto-generated POSTGRES_PASSWORD" - fi + local token_enc_key + token_enc_key=$(get_or_generate_secret \ + "${TOKEN_ENCRYPTION_KEY:-}" "$backend_target" "TOKEN_ENCRYPTION_KEY" "" \ + "python3 -c \"from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())\" 2>/dev/null || echo ''") + if [ -z "$token_enc_key" ]; then + log_warn "Could not auto-generate TOKEN_ENCRYPTION_KEY (cryptography package not available)" fi + local cookie_secret + cookie_secret=$(get_or_generate_secret \ + "${OAUTH_COOKIE_SECRET:-}" "$oauth_target" "cookie-secret" "" \ + "python3 -c \"import secrets; print(secrets.token_urlsafe(32))\" 2>/dev/null || openssl rand -base64 32") + # --- OAuth Proxy Secret --- - local oauth_target="$PROJECT_ROOT/k8s/app/overlays/dev/oauth-proxy-secret.env" if [ -f "$oauth_target" ] && [ "$force" != "--force" ]; then log_info "Already exists: $oauth_target (use --force to overwrite)" else - if [ ! -f "$CONFIG_DEV/.env.oauth-proxy" ]; then - log_warn "Config not found: $CONFIG_DEV/.env.oauth-proxy - skipping OAuth secret" - else - load_env "$CONFIG_DEV/.env.oauth-proxy" - - # Generate cookie secret if still placeholder - if [ -z "$OAUTH_COOKIE_SECRET" ] || [ "$OAUTH_COOKIE_SECRET" = "changethis_generate_a_secure_random_key" ]; then - OAUTH_COOKIE_SECRET=$(python3 -c "import secrets; print(secrets.token_urlsafe(32))" 2>/dev/null || openssl rand -base64 32) - fi - - mkdir -p "$(dirname "$oauth_target")" - cat > "$oauth_target" < "$oauth_target" < "$langflow_target" < "$postgres_target" < "$postgres_target" < "$langflow_target" </dev/null || openssl rand -base64 32) - log_info "Auto-generated SECRET_KEY" - fi - - # Auto-generate TOKEN_ENCRYPTION_KEY if empty, unset, or placeholder - if [ -z "$TOKEN_ENCRYPTION_KEY" ] || [ "$TOKEN_ENCRYPTION_KEY" = "changethis_generate_a_fernet_key" ]; then - TOKEN_ENCRYPTION_KEY=$(python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" 2>/dev/null || echo "") - if [ -z "$TOKEN_ENCRYPTION_KEY" ]; then - log_warn "Could not auto-generate TOKEN_ENCRYPTION_KEY (cryptography package not available)" - else - log_info "Auto-generated TOKEN_ENCRYPTION_KEY" - fi - fi - - mkdir -p "$(dirname "$backend_target")" - { - echo "# Backend Configuration Secret" - echo "# Generated from config/dev/.env.backend by generate-config.sh" - echo "# DO NOT commit this file to git!" - echo "" - echo "ENVIRONMENT=${ENVIRONMENT}" - echo "SECRET_KEY=${SECRET_KEY}" - [ -n "$FRONTEND_HOST" ] && echo "FRONTEND_HOST=${FRONTEND_HOST}" - [ -n "$BACKEND_CORS_ORIGINS" ] && echo "BACKEND_CORS_ORIGINS=${BACKEND_CORS_ORIGINS}" - [ -n "$TOKEN_ENCRYPTION_KEY" ] && echo "TOKEN_ENCRYPTION_KEY=${TOKEN_ENCRYPTION_KEY}" - [ -n "$LANGFLOW_URL" ] && echo "LANGFLOW_URL=${LANGFLOW_URL}" - [ -n "$LANGFLOW_DEFAULT_FLOW" ] && echo "LANGFLOW_DEFAULT_FLOW=${LANGFLOW_DEFAULT_FLOW}" - [ -n "$GOOGLE_CLIENT_ID" ] && echo "GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID}" - [ -n "$GOOGLE_CLIENT_SECRET" ] && echo "GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET}" - [ -n "$DATAVERSE_AUTH_URL" ] && echo "DATAVERSE_AUTH_URL=${DATAVERSE_AUTH_URL}" - } > "$backend_target" - log_info "Created: $backend_target" - fi + local environment="${ENVIRONMENT:-development}" + local langflow_url="${LANGFLOW_URL:-http://langflow-service-backend:7860}" + + mkdir -p "$(dirname "$backend_target")" + { + echo "# Backend Configuration Secret" + echo "# Generated from config/dev/.env by generate-config.sh" + echo "# DO NOT commit this file to git!" + echo "" + echo "ENVIRONMENT=${environment}" + echo "SECRET_KEY=${secret_key}" + [ -n "$token_enc_key" ] && echo "TOKEN_ENCRYPTION_KEY=${token_enc_key}" + echo "LANGFLOW_URL=${langflow_url}" + [ -n "${LANGFLOW_DEFAULT_FLOW:-}" ] && echo "LANGFLOW_DEFAULT_FLOW=${LANGFLOW_DEFAULT_FLOW}" + [ -n "${FRONTEND_HOST:-}" ] && echo "FRONTEND_HOST=${FRONTEND_HOST}" + [ -n "${BACKEND_CORS_ORIGINS:-}" ] && echo "BACKEND_CORS_ORIGINS=${BACKEND_CORS_ORIGINS}" + [ -n "${GOOGLE_CLIENT_ID:-}" ] && echo "GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID}" + [ -n "${GOOGLE_CLIENT_SECRET:-}" ] && echo "GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET}" + [ -n "${DATAVERSE_AUTH_URL:-}" ] && echo "DATAVERSE_AUTH_URL=${DATAVERSE_AUTH_URL}" + } > "$backend_target" + log_info "Created: $backend_target" fi # --- Email Allowlist --- - local emails_target="$PROJECT_ROOT/k8s/app/overlays/dev/allowed-emails.txt" if [ -f "$emails_target" ] && [ "$force" != "--force" ]; then log_info "Already exists: $emails_target (use --force to overwrite)" else @@ -425,7 +387,7 @@ EOF log_warn "Config not found: $CONFIG_DEV/allowed-emails.txt - skipping email allowlist" else # Copy, stripping comment lines - grep -v '^#' "$CONFIG_DEV/allowed-emails.txt" | grep -v '^$' > "$emails_target" + grep -v '^#' "$CONFIG_DEV/allowed-emails.txt" | grep -v '^$' > "$emails_target" || true log_info "Created: $emails_target" fi fi @@ -433,11 +395,11 @@ EOF log_info "K8s secret generation complete" } -# Generate Langfuse Helm secrets from config/dev/ +# Generate Langfuse Helm secrets from config/dev/.env (single source) cmd_helm_langfuse() { local force="${1:-}" - log_info "Generating Langfuse Helm secrets from config/dev/..." + log_info "Generating Langfuse Helm secrets from config/dev/.env..." local template="$PROJECT_ROOT/helm/langfuse/secrets-dev.yaml.template" local output="$PROJECT_ROOT/helm/langfuse/secrets-dev.yaml" @@ -452,12 +414,90 @@ cmd_helm_langfuse() { return 1 fi - if [ ! -f "$CONFIG_DEV/.env.langfuse" ]; then - log_error "Config not found: $CONFIG_DEV/.env.langfuse" - return 1 + # Load user-settable values from single .env + if [ -f "$CONFIG_DEV/.env" ]; then + load_env "$CONFIG_DEV/.env" fi - load_env "$CONFIG_DEV/.env.langfuse" + # Helper: generate or reuse a secret + # Priority: user .env value > existing YAML artifact value > generate new + _langfuse_secret() { + local user_val="$1" + local yaml_key="$2" # YAML env name or simple key for extraction + local placeholder="$3" + local gen_cmd="$4" + local yaml_type="${5:-env}" # "env" for additionalEnv, "simple" for direct YAML value + + # 1. User-provided non-placeholder value + if [ -n "$user_val" ] && [ "$user_val" != "$placeholder" ]; then + echo "$user_val" + return + fi + + # 2. Reuse from existing artifact + if [ -f "$output" ]; then + local existing="" + if [ "$yaml_type" = "env" ]; then + existing=$(get_yaml_env_value "$output" "$yaml_key") + else + existing=$(get_yaml_simple_value "$output" "$yaml_key") + fi + if [ -n "$existing" ] && [ "$existing" != "$placeholder" ] && [ "$existing" != '${'"$yaml_key"'}' ]; then + echo "$existing" + return + fi + fi + + # 3. Generate new + local generated + generated=$(eval "$gen_cmd" 2>/dev/null) || true + if [ -z "$generated" ]; then + log_error "Failed to generate Langfuse secret for $yaml_key" + return 1 + fi + echo "$generated" + } + + # Generate or reuse Langfuse internal secrets (idempotent) + local gen_secret="python3 -c \"import secrets; print(secrets.token_urlsafe(24))\" 2>/dev/null || openssl rand -base64 24" + local gen_hex="python3 -c \"import secrets; print(secrets.token_hex(32))\" 2>/dev/null || openssl rand -hex 32" + + export REDIS_PASSWORD + REDIS_PASSWORD=$(_langfuse_secret "${REDIS_PASSWORD:-}" "redis" "" "$gen_secret" "simple") + + export CLICKHOUSE_PASSWORD + CLICKHOUSE_PASSWORD=$(_langfuse_secret "${CLICKHOUSE_PASSWORD:-}" "clickhouse" "" "$gen_secret" "simple") + + export ENCRYPTION_KEY + ENCRYPTION_KEY=$(_langfuse_secret "${ENCRYPTION_KEY:-}" "encryptionKey" "" "$gen_hex" "simple") + + export SALT + SALT=$(_langfuse_secret "${SALT:-}" "salt" "" "$gen_secret" "simple") + + export NEXTAUTH_SECRET + NEXTAUTH_SECRET=$(_langfuse_secret "${NEXTAUTH_SECRET:-}" "secret" "" "$gen_secret" "simple") + + export LANGFUSE_INIT_USER_PASSWORD + LANGFUSE_INIT_USER_PASSWORD=$(_langfuse_secret "${LANGFUSE_INIT_USER_PASSWORD:-}" "LANGFUSE_INIT_USER_PASSWORD" "" "$gen_secret" "env") + + # Langfuse API keys (pk-/sk- prefixed) + local gen_pk="echo pk-\$(python3 -c \"import secrets; print(secrets.token_urlsafe(24))\" 2>/dev/null || openssl rand -base64 24 | tr '+/' '-_' | tr -d '=')" + local gen_sk="echo sk-\$(python3 -c \"import secrets; print(secrets.token_urlsafe(24))\" 2>/dev/null || openssl rand -base64 24 | tr '+/' '-_' | tr -d '=')" + + export LANGFUSE_INIT_PROJECT_PUBLIC_KEY + LANGFUSE_INIT_PROJECT_PUBLIC_KEY=$(_langfuse_secret "${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}" "LANGFUSE_INIT_PROJECT_PUBLIC_KEY" "pk-your-public-key" "$gen_pk" "env") + + export LANGFUSE_INIT_PROJECT_SECRET_KEY + LANGFUSE_INIT_PROJECT_SECRET_KEY=$(_langfuse_secret "${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}" "LANGFUSE_INIT_PROJECT_SECRET_KEY" "sk-your-secret-key" "$gen_sk" "env") + + # Defaults for non-user variables + export LANGFUSE_NEXTAUTH_URL="${LANGFUSE_NEXTAUTH_URL:-auto-calculated-by-deploy-script}" + export LANGFUSE_INIT_USER_EMAIL="${LANGFUSE_INIT_USER_EMAIL:-admin@your-domain.com}" + export LANGFUSE_INIT_USER_NAME="${LANGFUSE_INIT_USER_NAME:-Admin}" + export LANGFUSE_INIT_ORG_ID="${LANGFUSE_INIT_ORG_ID:-multi-agent-platform}" + export LANGFUSE_INIT_ORG_NAME="${LANGFUSE_INIT_ORG_NAME:-Multi-Agent Platform}" + export LANGFUSE_INIT_PROJECT_ID="${LANGFUSE_INIT_PROJECT_ID:-default}" + export LANGFUSE_INIT_PROJECT_NAME="${LANGFUSE_INIT_PROJECT_NAME:-Default Project}" # Use selective envsubst to avoid replacing stray $ in YAML local var_list='$REDIS_PASSWORD $CLICKHOUSE_PASSWORD $SALT $ENCRYPTION_KEY $LANGFUSE_NEXTAUTH_URL $NEXTAUTH_SECRET $LANGFUSE_INIT_PROJECT_PUBLIC_KEY $LANGFUSE_INIT_PROJECT_SECRET_KEY $LANGFUSE_INIT_USER_EMAIL $LANGFUSE_INIT_USER_NAME $LANGFUSE_INIT_USER_PASSWORD $LANGFUSE_INIT_ORG_ID $LANGFUSE_INIT_ORG_NAME $LANGFUSE_INIT_PROJECT_ID $LANGFUSE_INIT_PROJECT_NAME' @@ -467,54 +507,61 @@ cmd_helm_langfuse() { log_info "Created: $output" } -# Delete all generated config files so they can be regenerated fresh -# Usage: cmd_reset (currently only "dev" supported) +# Delete copied/generated config files so they can be regenerated fresh +# Source .env files are preserved (user's config); created from .env.example if missing. +# Usage: cmd_reset ("local", "dev", or "all"; default: "all") cmd_reset() { - local environment="${1:-dev}" + local environment="${1:-all}" - if [[ "$environment" != "dev" ]]; then - log_error "Reset only supports 'dev' environment currently" + if [[ "$environment" != "local" && "$environment" != "dev" && "$environment" != "all" ]]; then + log_error "Unknown environment: $environment (use 'local', 'dev', or omit for both)" exit 1 fi - log_info "Removing generated config files for $environment..." - - # config/dev/ — remove non-example files - for f in "$CONFIG_DEV"/.env.*; do - [ -f "$f" ] || continue - case "$f" in *.example) continue ;; esac - rm -f "$f" - log_info "Removed: $f" - done - for f in allowed-emails.txt namespace-admins.txt flow-sources.yaml; do - if [ -f "$CONFIG_DEV/$f" ]; then - rm -f "$CONFIG_DEV/$f" - log_info "Removed: $CONFIG_DEV/$f" - fi - done - - # k8s overlay generated secrets - local k8s_generated=( - "$PROJECT_ROOT/k8s/app/overlays/dev/oauth-proxy-secret.env" - "$PROJECT_ROOT/k8s/app/overlays/dev/backend-config.env" - "$PROJECT_ROOT/k8s/app/overlays/dev/allowed-emails.txt" - "$PROJECT_ROOT/k8s/langflow/overlays/dev/langflow-secret.env" - "$PROJECT_ROOT/k8s/postgres/overlays/dev/postgres-secret.env" - ) - for f in "${k8s_generated[@]}"; do - if [ -f "$f" ]; then - rm -f "$f" - log_info "Removed: $f" + if [[ "$environment" == "local" || "$environment" == "all" ]]; then + log_info "Removing copied config files for local..." + + # Remove service directory copies (generated from config/local/.env) + for f in "$PROJECT_ROOT/backend/.env" "$PROJECT_ROOT/frontend/.env"; do + if [ -f "$f" ]; then + rm -f "$f" + log_info "Removed: $(basename $(dirname $f))/$(basename $f)" + fi + done + + # Ensure source .env exists (create from example if missing) + copy_if_missing "$CONFIG_LOCAL/.env.example" "$CONFIG_LOCAL/.env" + fi + + if [[ "$environment" == "dev" || "$environment" == "all" ]]; then + log_info "Removing generated config files for dev..." + + # Remove generated deployment artifacts + local k8s_generated=( + "$PROJECT_ROOT/k8s/app/overlays/dev/oauth-proxy-secret.env" + "$PROJECT_ROOT/k8s/app/overlays/dev/backend-config.env" + "$PROJECT_ROOT/k8s/app/overlays/dev/allowed-emails.txt" + "$PROJECT_ROOT/k8s/langflow/overlays/dev/langflow-secret.env" + "$PROJECT_ROOT/k8s/postgres/overlays/dev/postgres-secret.env" + ) + for f in "${k8s_generated[@]}"; do + if [ -f "$f" ]; then + rm -f "$f" + log_info "Removed: $f" + fi + done + + # Helm generated secrets + if [ -f "$PROJECT_ROOT/helm/langfuse/secrets-dev.yaml" ]; then + rm -f "$PROJECT_ROOT/helm/langfuse/secrets-dev.yaml" + log_info "Removed: helm/langfuse/secrets-dev.yaml" fi - done - # Helm generated secrets - if [ -f "$PROJECT_ROOT/helm/langfuse/secrets-dev.yaml" ]; then - rm -f "$PROJECT_ROOT/helm/langfuse/secrets-dev.yaml" - log_info "Removed: helm/langfuse/secrets-dev.yaml" + # Ensure source .env exists (create from example if missing) + copy_if_missing "$CONFIG_DEV/.env.example" "$CONFIG_DEV/.env" fi - log_info "Reset complete. Run './scripts/generate-config.sh dev' to regenerate." + log_info "Reset complete. Run 'make config-setup' or 'make config-setup-cluster' to sync." } # Run all generation commands @@ -530,28 +577,30 @@ cmd_help() { echo "Usage: $0 [command] [options]" echo "" echo "Unified config generation script." - echo "Reads from config/ as source of truth, generates target-specific formats." + echo "Reads single .env per environment, generates deployment artifacts." + echo "User config files are NEVER modified by this script." echo "" echo "Commands:" - echo " local Setup local development config (copy .example files)" - echo " dev Setup cluster dev config (copy .example files, auto-generate secrets)" - echo " k8s Generate Kubernetes secret .env files from config/dev/" - echo " helm-langfuse Generate Langfuse Helm secrets YAML from config/dev/" - echo " all Run k8s + helm-langfuse" - echo " reset [env] Delete all generated config for environment (default: dev)" - echo " Use when moving to a new cluster to start fresh" + echo " local Setup local dev config (copy .env.example, create backend/.env)" + echo " dev Setup cluster dev config (copy .env.example + access control files)" + echo " k8s Generate K8s secret files from config/dev/.env" + echo " helm-langfuse Generate Langfuse Helm secrets from config/dev/.env" + echo " all Run k8s + helm-langfuse (generates all deployment artifacts)" + echo " reset [env] Delete generated config (default: all; options: local, dev, all)" + echo " Use to start fresh with config from examples" echo " help Show this help message" echo "" echo "Options:" echo " --force Overwrite existing generated files" echo "" echo "Source directories:" - echo " config/local/ Local development config (.env.*.example templates)" - echo " config/dev/ Cluster/dev deployment config" + echo " config/local/.env Local development config" + echo " config/dev/.env Cluster/dev deployment config" echo "" - echo "Generated targets:" + echo "Generated targets (from config/dev/.env):" echo " k8s/app/overlays/dev/oauth-proxy-secret.env" echo " k8s/app/overlays/dev/backend-config.env" + echo " k8s/app/overlays/dev/allowed-emails.txt" echo " k8s/langflow/overlays/dev/langflow-secret.env" echo " k8s/postgres/overlays/dev/postgres-secret.env" echo " helm/langfuse/secrets-dev.yaml" @@ -578,7 +627,7 @@ case "${1:-help}" in cmd_all "${2:-}" ;; reset) - cmd_reset "${2:-dev}" + cmd_reset "${2:-all}" ;; help|--help|-h) cmd_help diff --git a/scripts/undeploy.sh b/scripts/undeploy.sh index c616f55..bf2fdea 100755 --- a/scripts/undeploy.sh +++ b/scripts/undeploy.sh @@ -44,22 +44,15 @@ if ! oc whoami &> /dev/null; then exit 1 fi -# Undeploy in reverse order -# Services are deployed via helm template | oc apply, so delete by label -echo "Step 1/5: Removing Langfuse..." -oc delete route langfuse -n "$NAMESPACE" 2>/dev/null || true -oc delete statefulset -l app.kubernetes.io/instance=langfuse -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true -oc delete all,configmap,secret,serviceaccount,role,rolebinding -l app.kubernetes.io/instance=langfuse -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true -echo "" - -echo "Step 2/5: Removing MLFlow..." -oc delete route mlflow -n "$NAMESPACE" 2>/dev/null || true -oc delete service mlflow-external -n "$NAMESPACE" 2>/dev/null || true -oc delete all,configmap,secret,serviceaccount -l app.kubernetes.io/instance=mlflow -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true -oc delete -k "$PROJECT_ROOT/k8s/mlflow/overlays/$ENVIRONMENT" -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +# Undeploy in reverse dependency order (opposite of deploy.sh): +# Deploy: namespace → postgres → langfuse → mlflow → langflow → app +# Undeploy: app → langflow → mlflow → langfuse → shared resources → postgres +echo "Step 1/5: Removing Multi-Agent Platform App..." +oc delete -k "$PROJECT_ROOT/k8s/app/overlays/$ENVIRONMENT" -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +oc delete secret backend-config -n "$NAMESPACE" 2>/dev/null || true echo "" -echo "Step 3/5: Removing LangFlow..." +echo "Step 2/5: Removing LangFlow..." oc delete route langflow -n "$NAMESPACE" 2>/dev/null || true oc delete service langflow-external -n "$NAMESPACE" 2>/dev/null || true oc delete statefulset -l release=langflow -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true @@ -67,19 +60,27 @@ oc delete all,configmap,secret,serviceaccount -l release=langflow -n "$NAMESPACE oc delete -k "$PROJECT_ROOT/k8s/langflow/overlays/$ENVIRONMENT" -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true echo "" -echo "Step 4/5: Removing Multi-Agent Platform App..." -oc delete -k "$PROJECT_ROOT/k8s/app/overlays/$ENVIRONMENT" -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true -oc delete secret backend-config -n "$NAMESPACE" 2>/dev/null || true +echo "Step 3/5: Removing MLFlow..." +oc delete route mlflow -n "$NAMESPACE" 2>/dev/null || true +oc delete service mlflow-external -n "$NAMESPACE" 2>/dev/null || true +oc delete all,configmap,secret,serviceaccount -l app.kubernetes.io/instance=mlflow -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +oc delete -k "$PROJECT_ROOT/k8s/mlflow/overlays/$ENVIRONMENT" -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +echo "" + +echo "Step 4/5: Removing Langfuse..." +oc delete route langfuse -n "$NAMESPACE" 2>/dev/null || true +oc delete statefulset -l app.kubernetes.io/instance=langfuse -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true +oc delete all,configmap,secret,serviceaccount,role,rolebinding -l app.kubernetes.io/instance=langfuse -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true echo "" -echo "Removing shared OAuth resources..." +echo "Removing shared OAuth resources and admin credentials..." oc delete sa supporting-services-proxy -n "$NAMESPACE" 2>/dev/null || true oc delete secret supporting-services-proxy-session -n "$NAMESPACE" 2>/dev/null || true +oc delete secret admin-credentials -n "$NAMESPACE" 2>/dev/null || true echo "" echo "Step 5/5: Removing PostgreSQL..." oc delete -k "$PROJECT_ROOT/k8s/postgres/overlays/$ENVIRONMENT" -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true -oc delete secret admin-credentials -n "$NAMESPACE" 2>/dev/null || true echo "" echo "==================================="