diff --git a/.github/COPILOT_SETUP_GUIDE.md b/.github/COPILOT_SETUP_GUIDE.md index 26f023a2e..79e1ae5f8 100644 --- a/.github/COPILOT_SETUP_GUIDE.md +++ b/.github/COPILOT_SETUP_GUIDE.md @@ -382,7 +382,7 @@ Aria includes **20+ specialized prompts** in `.github/prompts/` that guide Copil | Prompt | Purpose | |--------|---------| -| `agi.prompt.md` | Chain-of-thought AGI reasoning | +| `agi.prompt.md` | Autonomous long-running work with internal reasoning (chain-of-thought is hidden) | | `debug.prompt.md` | Systematic debugging protocol | | `reason.prompt.md` | Structured analysis & planning | | `review.prompt.md` | Security, performance, correctness review | @@ -394,7 +394,7 @@ Aria includes **20+ specialized prompts** in `.github/prompts/` that guide Copil **Explicit reference**: ``` -Use the agi.prompt for chain-of-thought reasoning about this architecture +Use the agi.prompt for autonomous long-running work requiring internal reasoning Use the debug.prompt to systematically diagnose this error ``` diff --git a/.github/agents/agi-reasoning.agent.md b/.github/agents/agi-reasoning.agent.md index 3219e3001..9c31a382c 100644 --- a/.github/agents/agi-reasoning.agent.md +++ b/.github/agents/agi-reasoning.agent.md @@ -1,6 +1,6 @@ --- name: agi-reasoning -description: "AGI reasoning and autonomous decision-making agent. Specializes in chain-of-thought reasoning, task decomposition, self-reflection, and multi-step planning using the AGI provider system.\n\nTrigger phrases include:\n- 'reason through this problem'\n- 'break this down step by step'\n- 'think through this autonomously'\n- 'use AGI reasoning'\n- 'chain of thought'\n- 'self-reflection'\n- 'autonomous planning'\n\nExamples:\n- User says 'reason through this architecture decision' → invoke for structured multi-step analysis\n- User asks 'break down this complex feature into tasks' → invoke for task decomposition\n- User says 'autonomously plan and implement this feature' → invoke for planning + execution with self-correction\n\nThis agent leverages the AGI provider's reasoning chains, task decomposition, and self-reflection capabilities." +description: "AGI reasoning and autonomous decision-making agent. Uses internal chain-of-thought reasoning (not exposed in output) for task decomposition, self-reflection, and multi-step planning — only the final answer is delivered to the user. For visible step-by-step reasoning, use reason.prompt instead.\n\nTrigger phrases include:\n- 'reason through this problem'\n- 'break this down step by step'\n- 'think through this autonomously'\n- 'use AGI reasoning'\n- 'autonomous planning'\n- 'self-reflection'\n\nExamples:\n- User says 'reason through this architecture decision' → invoke for structured multi-step analysis\n- User asks 'break down this complex feature into tasks' → invoke for task decomposition\n- User says 'autonomously plan and implement this feature' → invoke for planning + execution with self-correction\n\nThis agent leverages the AGI provider's reasoning chains, task decomposition, and self-reflection capabilities. Chain-of-thought steps are completed internally; only the final answer is delivered." tools: - edit - search diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8e394efff..045bd10e6 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,411 +1,411 @@ - - -# Aria — Copilot Quick Guide - -*Last updated: November 29, 2025* - -Short & actionable summary for AI agents editing Aria — an interactive AI character platform with autonomous learning, quantum ML integration, and multi-provider chat backends. - -## Architecture - -- **Interactive AI Character Platform** with 3D animated avatar, natural language movement commands, and real-time object interaction -- **Three isolated projects + Functions integration layer:** - - `ai-projects/quantum-ml/` — MCP server, web dashboard, quantum ML pipelines (separate venv) - - `ai-projects/chat-cli/` — chat CLI with multi-provider support (separate venv) - - `AI/microsoft_phi-silica-3.6_v1/` — Phi-3.5 LoRA fine-tuning (separate venv) - - `function_app.py` — Azure Functions integration exposing all APIs -- **Integration points:** - - `function_app.py` dynamically imports from ai-projects/chat-cli/src and ai-projects/quantum-ml/src (adds to sys.path) - - Shared infra in `shared/`: re-exports chat providers, DB engines, telemetry, Cosmos client -- **Web Interfaces:** - - `apps/aria/` — Interactive Aria character interface with CSS animations, eye tracking, gestures - - `apps/chat/` — Streaming chat UI with SSE support -- **API endpoints** (via `function_app.py`): - - `/api/chat` — streaming chat SSE - - `/api/chat-web` — web UI HTML - - `/api/tts` — Azure Speech TTS (falls back to local if enabled) - - `/api/quantum/*` — quantum job submission/monitoring - - `/api/ai/status` — health check showing active provider, env vars, DB pool, Cosmos status - - `/api/quantum-llm/status` — quantum-powered LLM backend info (backend, qubits, fallback, provider) - - `/api/quantum-llm/chat` — non-streaming quantum-augmented LLM completion - - `/api/quantum-llm/stream` — SSE streaming quantum-augmented LLM (same format as `/api/chat/stream`) -- **Aria Web API endpoints** (via `apps/aria/server.py` on port 8080): - - `GET /api/aria/state` — current stage state (position, objects, expressions) - - `POST /api/aria/command` — process natural language commands - - `POST /api/aria/execute` — auto-execute action sequences (plan or execute mode) - - `POST /api/aria/object` — manage objects (add, update, remove) - - `POST /api/aria/world` — LLM-powered themed world generation - -## Key Features - -**Interactive Character System:** -- 3D CSS-animated character with smooth transitions and physics-based movement -- Natural language command processing ("move left", "wave at me", "dance", "jump", "pickup ball") -- **Auto-Execute System**: LLM-powered action parser converts natural language to structured action sequences - - 8 core actions: move, say, pickup, drop, throw, gesture, look, wait - - Plan mode (preview actions) and execute mode (run sequences) - - Dual-mode parsing: LLM-powered + rule-based fallback -- Object interaction system (add, pickup, drop, throw with trajectory physics) -- **World Generation**: LLM-powered themed environment creation -- Eye tracking and attention system (follows mouse cursor) -- Emotion/gesture system (wave, dance, jump, idle animations) -- Real-time speech synthesis via Azure TTS or local fallback -- Server-synchronized state management (character position, objects, expressions) - -**Autonomous Learning:** -- Self-discovering dataset collection from multiple sources -- Adaptive epoch selection based on performance history -- Automatic model promotion when accuracy thresholds met -- Performance degradation detection and alerting -- Continuous 30-minute training cycles with graceful error recovery - -**Multi-Provider Chat:** -- Azure OpenAI, OpenAI, LMStudio, local models -- LoRA adapter support for fine-tuned models -- Automatic provider fallback chain -- SSE-based streaming responses - -## Quick Commands (from repo root) - -```bash -# === AUTONOMOUS SYSTEMS (Self-Managing) === -# Start autonomous training (continuous 30-min cycles) -nohup python scripts/autonomous_training_orchestrator.py > data_out/autonomous_training.log 2>&1 & - -# Trigger immediate cycle (skip 30-min wait) -pkill -USR1 -f autonomous_training - -# Full repo automation (Aria + training + quantum + monitoring) -python scripts/repo_automation.py --start -python scripts/repo_automation.py --status -./scripts/start_repo_automation.sh full # Bash wrapper with menu -./scripts/start_repo_automation.sh stop # Stop all components - -# Aria character automation (server + continuous training) -python scripts/aria_automation.py --mode full -python scripts/aria_automation.py --status - -# === ARIA CHARACTER WEB UI === -cd apps/aria && python server.py # Start Aria web interface (port 8080) -# Access at: http://localhost:8080 -# Auto-Execute UI: http://localhost:8080/auto-execute.html -# Commands: "move left", "wave", "dance", "jump", "pickup ball", "throw" -# Complex: "Walk to the table and pick up the apple", "Say hello and wave" - -# === AZURE FUNCTIONS & APIs === -func host start # Start Functions host (serves all APIs) -curl http://localhost:7071/api/ai/status | jq # Health check - -# === TESTING & VALIDATION === -python scripts/test_runner.py --unit # Fast unit tests -python scripts/test_runner.py --all # All tests -python scripts/test_runner.py --unit --coverage # Unit tests + coverage -python scripts/test_runner.py --list-suites # Show available suites -python scripts/test_runner.py --unit --watch # Re-run on file changes -python ai-projects/chat-cli/src/chat_cli.py --provider local --once "Hello" # Smoke test -python scripts/fast_validate.py # Quick validation (datasets, scripts, venvs, configs, providers, deps) -python scripts/cleanup_artifacts.py # Preview old artifact cleanup (dry-run) -python scripts/cleanup_artifacts.py --apply # Actually delete old artifacts - -# === ORCHESTRATORS (Manual Execution) === -python scripts/autotrain.py --dry-run # Validate training config (12 jobs) -python scripts/quantum_autorun.py --dry-run # Validate quantum config -python scripts/evaluation_autorun.py --dry-run # Validate evaluation config - -# === TRAINING PIPELINES === -python scripts/automated_training_pipeline.py --quick # Quick LoRA (TinyLlama) -python scripts/train_and_promote.py --quick --auto-promote # Train + auto-deploy - -# === MCP & TOOLS === -python ai-projects/quantum-ml/quantum_mcp_server.py # Start quantum MCP server - -# === MONITORING & DIAGNOSTICS === -curl http://localhost:7071/api/ai/status | jq # Comprehensive health check -python scripts/status_dashboard.py # Unified orchestrator status -python scripts/status_dashboard.py --watch # Auto-refresh every 10s -python scripts/resource_monitor.py --snapshot # CPU/memory/disk/GPU snapshot -python scripts/system_health_check.py # Full system health report -python scripts/training_analytics.py # Performance trends & insights -tail -f data_out/autonomous_training.log # Live autonomous training logs -watch -n 5 'cat data_out/autonomous_training_status.json | python -m json.tool' # Live status -``` - -## Critical Patterns - -**Autonomous/self-managing systems:** -- `scripts/autonomous_training_orchestrator.py` — Continuous learning with 30-min cycles (infinite by default) - - Self-discovers datasets (scans `datasets/quantum`, `datasets/chat`, `datasets/massive_quantum`) - - Self-optimizes: Adaptive epochs `[25, 50, 100, 200]` based on performance history - - Self-heals: Graceful error handling, continues on failure, logs to `data_out/autonomous_training.log` - - State: `data_out/autonomous_training_status.json` (cycles_completed, best_accuracy, dataset_inventory) - - Config: `config/autonomous_training.yaml` (cycle_interval_minutes, epochs_progression, min_datasets) - - Trigger: Time-based (30min) OR signal-based (`pkill -USR1 -f autonomous_training`) -- `scripts/repo_automation.py` — Full repo automation (all components: Aria + training + quantum + datasets) -- `scripts/aria_automation.py` — Aria-specific automation (server on port 8080 + continuous training) -- `scripts/master_orchestrator.py` — Coordinates all sub-orchestrators with schedules/dependencies - - Config: `config/master_orchestrator.yaml` (cron schedules, priorities, retry logic, timeouts) - -**Data conventions:** -- `datasets/` is **read-only** — never modify existing datasets -- All outputs go to `data_out//` with `status.json` as source of truth -- Chat datasets: `[{"messages": [{"role": "user|assistant", "content": "..."}]}]` -- LoRA adapters need both `adapter_config.json` + `adapter_model.safetensors` - -**Provider detection chain** (in `shared/chat_providers.py`): -1. Explicit choice (--provider flag) -2. LMStudio (if `LMSTUDIO_BASE_URL` configured) -3. Azure OpenAI (needs all 4: `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT`, `AZURE_OPENAI_API_VERSION`) -4. OpenAI (needs `OPENAI_API_KEY`) -5. LoRA (explicit --provider lora with adapter path) -6. Local fallback (zero-dependency echo) - -**Config precedence:** -`YAML base` < `CLI flags` < `per-job YAML` < `env vars` - -**YAML orchestrators:** -- All in `scripts/` with matching root YAMLs (e.g., `autotrain.yaml`, `quantum_autorun.yaml`) -- Write `data_out//status.json` with machine-readable job status -- Support `--dry-run` to validate before execution - -**Autonomous training implementation patterns:** -```python -# State machine: discovery → collection → training → analysis → optimization → deployment -async def run_single_cycle(cycle_number): - await discover_datasets() # Scan datasets/, catalog by category - await download_new_datasets() # Download if below min_datasets threshold - epochs = await select_optimal_epochs() # Adaptive: increase if accuracy < 0.70 or plateau - results = await train_cycle(epochs) # Distributed training with multiprocessing - await analyze_performance(results) # Track metrics, detect degradation - await optimization_cycle() # Hyperparameter tuning (if enabled) - await deployment_cycle() # Auto-deploy if accuracy > 0.90 (if enabled) -``` - -**Process management:** -- Autonomous systems run via `nohup` in background, logs to `data_out/*.log` -- Check status: `ps aux | grep -E "(autonomous|aria)" | grep -v grep` -- Manual trigger: `pkill -USR1 -f autonomous_training` forces immediate cycle -- Graceful shutdown: `pkill -TERM -f autonomous_training` - -**Performance monitoring & observability:** -- **Health endpoint**: `GET /api/ai/status` — Comprehensive system diagnostics - - Active provider detection (azure|openai|local|lora) - - Environment variable presence (Azure OpenAI, OpenAI, Cosmos, SQL) - - ML library availability (torch, transformers, peft) — in-process & venv - - SQL pool metrics with saturation alerts (warns at ≥80%) - - Cosmos DB health check (lazy connection) - - Quantum environment status (qiskit, pennylane, Azure Quantum backends) - - LoRA adapter readiness (adapter_config.json, tokenizer) -- **Status files**: All orchestrators write `data_out//status.json` - - Schema: `{total_jobs, succeeded, failed, running, last_updated, avg_duration}` - - Autonomous training: `{cycles_completed, best_accuracy, performance_history[], dataset_inventory}` -- **Monitoring scripts**: - - `scripts/status_dashboard.py` — Unified view of all orchestrators (supports --watch, --export) - - `scripts/resource_monitor.py` — CPU/memory/disk/GPU with threshold alerts - - `scripts/system_health_check.py` — Comprehensive health report (venvs, Azure Functions, datasets) - - `scripts/training_analytics.py` — Performance trends, improvement rates, plateau detection -- **Performance degradation alerts**: Auto-detect >5% accuracy drops between cycles -- **Metrics tracked**: mean_accuracy, median_accuracy, max_accuracy, successful_count, exceptional_models -- **Notification config**: `config/notification_config.yaml` (email/SMTP/local alerts) - -## Where to Edit - -| Change | File(s) | -|--------|---------| -| Add/modify API endpoint | `function_app.py` | -| Quantum LLM pipeline | `ai-projects/quantum-ml/src/quantum_llm/pipeline.py` | -| Quantum token sampler | `ai-projects/quantum-ml/src/quantum_llm/quantum_sampler.py` | -| Quantum embedding transformer | `ai-projects/quantum-ml/src/quantum_llm/quantum_embeddings.py` | -| Quantum provider router | `ai-projects/quantum-ml/src/quantum_llm/quantum_router.py` | -| Quantum LLM config | `ai-projects/quantum-ml/src/quantum_llm/config.py` | -| Chat provider logic | `ai-projects/chat-cli/src/chat_providers.py` (re-exported by `shared/chat_providers.py`) | -| Training orchestration | `scripts/autotrain.py` + root `autotrain.yaml` | -| Autonomous training behavior | `scripts/autonomous_training_orchestrator.py` + `config/autonomous_training.yaml` | -| Master orchestrator (schedules/coordination) | `scripts/master_orchestrator.py` + `config/master_orchestrator.yaml` | -| Aria automation | `scripts/aria_automation.py` (server + training + health monitoring) | -| Full repo automation | `scripts/repo_automation.py` (all components + backups + notifications) | -| Quantum jobs | `scripts/quantum_autorun.py` + root `quantum_autorun.yaml` | -| MCP server tools | `ai-projects/quantum-ml/quantum_mcp_server.py` | -| Shared DB/telemetry | `shared/sql_engine.py`, `shared/telemetry.py`, `shared/cosmos_client.py` | -| Aria character interface | `apps/aria/index.html`, `apps/aria/aria_controller.js`, `apps/aria/server.py` | -| Aria movement/gestures | `apps/aria/aria_controller.js` (command parsing & animation triggers) | -| Semantic memory/embeddings | `shared/chat_memory.py` (generate_embedding, fetch_similar, store) | -| Token management | `ai-projects/chat-cli/src/token_utils.py` (counting, pruning) | -| LLM tool generation | `ai-projects/llm-maker/src/tool_maker.py`, `tool_validator.py` | -| Website generation | `ai-projects/llm-maker/src/website_maker.py` | -| Cooking AI recipes | `ai-projects/cooking-ai/src/agents/recipe_agent.py` | -| Subscriptions/monetization | `shared/subscription_manager.py` + `setup_monetization.py` | -| DB logging (fault-tolerant) | `shared/db_logging.py` (SP wrappers) | -| Vision/expression AI | `scripts/vision_inference.py` (TinyConvNet) | -| Batch model evaluation | `scripts/batch_evaluator.py` + `config/evaluation/` | -| Monitoring dashboard | `apps/dashboard/` (hub, analytics, GPU monitor) | -| AGI reasoning | `ai-projects/chat-cli/src/agi_provider.py` | -| Test runner | `scripts/test_runner.py` (centralized suite orchestrator) | -| Request validation | `shared/request_validator.py` (JSON schema validation) | -| Artifact cleanup | `scripts/cleanup_artifacts.py` (data_out/ retention) | -| Model evaluation | `scripts/evaluate_model.py` (delegates or fallback metrics) | -| Fast validation | `scripts/fast_validate.py` (configs, providers, deps) | - -## Safety Rules - -- Always `--dry-run` orchestrators before GPU/QPU execution -- Quantum: simulate locally first, then use `azure_ionq_simulator`, only then real QPU -- Real QPU jobs require `azure_confirm_cost: true` in YAML + cost estimate review -- Never hardcode secrets — use `local.settings.json` (dev) or Azure App Settings (prod) -- Monitor DB pool via `/api/ai/status` (warns at ≥80% saturation) - -## Testing & Validation - -- **Unit tests:** `pytest tests/ -m "not slow and not azure"` or `python scripts/test_runner.py --unit` -- **Integration tests:** `python scripts/test_runner.py --integration` -- **All tests:** `python scripts/test_runner.py --all` -- **VS Code Test Explorer:** Use 🧪 icon for interactive test running -- **Markers:** `@pytest.mark.slow`, `@pytest.mark.azure`, `@pytest.mark.integration` -- **Chat dataset validation:** `python scripts/validate_datasets.py --category chat` - -## Optional Services - -**SQL persistence** (optional): -- Enable via `QAI_DB_CONN` env var (SQLite, PostgreSQL, Azure SQL) -- Pool size: `QAI_SQL_POOL_SIZE` (default: 10) -- Health: Check `/api/ai/status` for pool saturation (warns ≥80%) - -**Cosmos DB** (optional, feature-flagged): -- Enable: `QAI_ENABLE_COSMOS=true` -- Config: `COSMOS_ENDPOINT`, `COSMOS_KEY`, `COSMOS_DATABASE`, `COSMOS_CONTAINER` -- Partition key: `/session_id`, enable TTL for cost savings - -**Telemetry** (optional): -- Application Insights via `APPLICATIONINSIGHTS_CONNECTION_STRING` -- Non-blocking, gracefully degrades if unavailable - -## Modular Instructions - -This repo uses component-specific instruction files in `.github/instructions/`: -- `functions.instructions.md` — Azure Functions API endpoints -- `shared-python.instructions.md` — Shared infrastructure patterns -- `quantum-ai*.instructions.md` — Quantum ML workflows -- `talk-to-ai*.instructions.md` — Chat CLI patterns -- `lora*.instructions.md` — LoRA fine-tuning patterns -- `chat-web.instructions.md` — Frontend SSE integration -- `agi-provider.instructions.md` — AGI reasoning system -- `aria-character.instructions.md` — Interactive character system -- `aria-web.instructions.md` — Aria web server module -- `autonomous-training.instructions.md` — Autonomous training orchestration -- `training-scripts.instructions.md` — Training script patterns -- `orchestrator-configs.instructions.md` — YAML orchestrator configs -- `dashboard.instructions.md` — Monitoring dashboard -- `tests.instructions.md` — Testing infrastructure -- `llm-maker.instructions.md` — Safe tool/website generation -- `cooking-ai.instructions.md` — Cooking AI recipe agent -- `chat-providers.instructions.md` — Multi-provider chat system -- `chat-memory.instructions.md` — Semantic memory & embeddings -- `subscription.instructions.md` — Subscription/monetization -- `db-logging.instructions.md` — Fault-tolerant DB logging -- `telemetry.instructions.md` — OpenTelemetry setup -- `token-utils.instructions.md` — Token counting & context pruning -- `evaluation.instructions.md` — Batch evaluation & analytics -- `vision-inference.instructions.md` — Vision AI & CNN models - -These are automatically applied by VS Code based on file paths. Check attachment indicators to see which rules are active. - -## Custom Coding Agents - -Available agents in `.github/agents/`: - -| Agent | Purpose | -|-------|---------| -| `ai.agent.md` | Primary autonomous agent — task decomposition, multi-step execution | -| `my-agent.agent.md` | QAI specialist — quantum-AI/ML development | -| `agi-reasoning.agent.md` | Chain-of-thought reasoning, self-reflection (CoT is internal, final answer only) | -| `visible-reasoning.agent.md` | Visible step-by-step reasoning, shows CoT trace to users | -| `aria-character.agent.md` | Interactive character commands, animations | -| `autonomous-trainer.agent.md` | LoRA training lifecycle, model promotion | -| `full-stack-debugger.agent.md` | Cross-stack issue diagnosis | -| `automated-code-fixer.agent.md` | Autonomous code improvements | -| `ai-architect.agent.md` | AI pipeline design, provider integration | -| `llm-maker.agent.md` | Safe tool/website generation | -| `chat-provider.agent.md` | Multi-provider chat, streaming, memory | -| `platform-ops.agent.md` | Subscriptions, monitoring, deployment | -| `vision-ai.agent.md` | Expression/emotion classification | -| `data-pipeline.agent.md` | Batch evaluation, dataset management | - -**Mode equivalents now live in `.github/agents/`**: -- `AI_model_training.agent.md` — End-to-end LoRA training, evaluation, and model promotion -- `Aria_character_development.agent.md` — Interactive character commands, actions, world generation -- `Quantum_ML_development.agent.md` — Quantum circuits, simulation, Azure Quantum pipelines -- `Full_stack_debugging.agent.md` — Cross-stack diagnostic protocol -- `AI_chat_development.agent.md` — Multi-provider chat, streaming, memory, self-learning -- `Azure_function_codegen_and_deployment.agent.md` — Enterprise Azure Functions workflow with IaC -- `Azure_Static_Web_App.agent.md` — Static web app deployment patterns - -**Prompts** (`.github/prompts/`): -- `agi.prompt.md` — Autonomous AGI reasoning with multi-step analysis and self-correction (chain-of-thought is internal, not exposed in output) -- `reason.prompt.md` — Visible step-by-step reasoning that exposes chain-of-thought, confidence scores, and self-reflection to the user (uses `visible-reasoning` agent) -- `debug.prompt.md` — Systematic diagnostic protocol -- `review.prompt.md` — Code review (correctness, security, performance) -- `aria-command.prompt.md` — Natural language → Aria actions -- `train.prompt.md` — Training execution with safety -- `quantum.prompt.md` — Cost-aware quantum workflows -- `chat.prompt.md` — Multi-provider chat with memory -- `generate-tool.prompt.md` — Safe Python tool generation -- `generate-website.prompt.md` — Complete website generation -- `evaluate.prompt.md` — Model evaluation & benchmarking -- `deploy.prompt.md` — Model/service deployment -- `optimize.prompt.md` — Performance analysis & optimization - -**Usage**: Agents are invoked automatically based on context or explicitly selected in GitHub Copilot interfaces. - -## Coding Agent Best Practices - -**For AI Coding Agents working in this repository:** - -1. **Always Check Context** - - Read `.github/copilot-instructions.md` (this file) first - - Check for relevant `.github/instructions/*.instructions.md` files based on file paths - - Reference `.github/agents/my-agent.agent.md` for QAI-specific patterns - -2. **Safety-First Approach** - - `--dry-run` all orchestrators before GPU/QPU execution - - Never modify files in `datasets/` (read-only) - - Check `/api/ai/status` before making provider-dependent changes - - Verify test suite passes: `python scripts/test_runner.py --unit` - -3. **Follow Established Patterns** - - Provider detection chain: Azure OpenAI → OpenAI → LMStudio → Local - - Config precedence: `YAML base` < `CLI flags` < `per-job YAML` < `env vars` - - Status files: Always write to `data_out//status.json` - - Autonomous systems: Use signal-based triggers (`pkill -USR1`) for immediate execution - -4. **Testing & Validation** - - Run unit tests before committing: `python scripts/test_runner.py --unit` - - Use `scripts/fast_validate.py` for quick cross-component validation - - Check health endpoint: `curl http://localhost:7071/api/ai/status | jq` - - Monitor logs: `tail -f data_out/autonomous_training.log` - -5. **Documentation Updates** - - Update this file when adding major features or changing workflows - - Keep component-specific instructions in `.github/instructions/` in sync - - Update PR checklist if adding new safety requirements - - Document new orchestrators in "Where to Edit" table - -6. **Cost & Resource Awareness** - - Quantum: Simulate locally first, then Azure simulator, only then real QPU - - Monitor DB pool saturation via `/api/ai/status` (warns at ≥80%) - - Check GPU/CPU usage: `python scripts/resource_monitor.py --snapshot` - - Review training analytics: `python scripts/training_analytics.py` - -## PR Checklist for AI Agents & Reviewers - -Before submitting or approving PRs, verify: - -- [ ] **Dry-run orchestrators**: If modifying YAML configs or orchestrators, run `--dry-run` to validate changes before committing -- [ ] **Provider detection intact**: Changes to `shared/chat_providers.py` or `function_app.py` don't break detection chain (test with `/api/ai/status`) -- [ ] **Dataset immutability**: No modifications to `datasets/` — all outputs written to `data_out/` -- [ ] **Status.json compliance**: Orchestrator changes maintain status JSON writes to `data_out//status.json` -- [ ] **Test suite passes**: Run `python scripts/test_runner.py --unit` (or `--all` for integration tests) -- [ ] **No hardcoded secrets**: All API keys/connection strings use env vars or `local.settings.json` -- [ ] **Quantum cost gates**: QPU jobs include `azure_confirm_cost: true` in YAML configs -- [ ] **LoRA adapter validity**: If modifying training scripts, verify output includes both `adapter_config.json` + `adapter_model.safetensors` -- [ ] **Documentation sync**: Update relevant READMEs/instruction files if changing core workflows or adding features - -Full/verbose guidance and advanced examples are preserved at `.github/copilot-instructions.full.md`. Ask me to expand any area or add examples for a specific change. + + +# Aria — Copilot Quick Guide + +*Last updated: November 29, 2025* + +Short & actionable summary for AI agents editing Aria — an interactive AI character platform with autonomous learning, quantum ML integration, and multi-provider chat backends. + +## Architecture + +- **Interactive AI Character Platform** with 3D animated avatar, natural language movement commands, and real-time object interaction +- **Three isolated projects + Functions integration layer:** + - `ai-projects/quantum-ml/` — MCP server, web dashboard, quantum ML pipelines (separate venv) + - `ai-projects/chat-cli/` — chat CLI with multi-provider support (separate venv) + - `AI/microsoft_phi-silica-3.6_v1/` — Phi-3.5 LoRA fine-tuning (separate venv) + - `function_app.py` — Azure Functions integration exposing all APIs +- **Integration points:** + - `function_app.py` dynamically imports from ai-projects/chat-cli/src and ai-projects/quantum-ml/src (adds to sys.path) + - Shared infra in `shared/`: re-exports chat providers, DB engines, telemetry, Cosmos client +- **Web Interfaces:** + - `apps/aria/` — Interactive Aria character interface with CSS animations, eye tracking, gestures + - `apps/chat/` — Streaming chat UI with SSE support +- **API endpoints** (via `function_app.py`): + - `/api/chat` — streaming chat SSE + - `/api/chat-web` — web UI HTML + - `/api/tts` — Azure Speech TTS (falls back to local if enabled) + - `/api/quantum/*` — quantum job submission/monitoring + - `/api/ai/status` — health check showing active provider, env vars, DB pool, Cosmos status + - `/api/quantum-llm/status` — quantum-powered LLM backend info (backend, qubits, fallback, provider) + - `/api/quantum-llm/chat` — non-streaming quantum-augmented LLM completion + - `/api/quantum-llm/stream` — SSE streaming quantum-augmented LLM (same format as `/api/chat/stream`) +- **Aria Web API endpoints** (via `apps/aria/server.py` on port 8080): + - `GET /api/aria/state` — current stage state (position, objects, expressions) + - `POST /api/aria/command` — process natural language commands + - `POST /api/aria/execute` — auto-execute action sequences (plan or execute mode) + - `POST /api/aria/object` — manage objects (add, update, remove) + - `POST /api/aria/world` — LLM-powered themed world generation + +## Key Features + +**Interactive Character System:** +- 3D CSS-animated character with smooth transitions and physics-based movement +- Natural language command processing ("move left", "wave at me", "dance", "jump", "pickup ball") +- **Auto-Execute System**: LLM-powered action parser converts natural language to structured action sequences + - 8 core actions: move, say, pickup, drop, throw, gesture, look, wait + - Plan mode (preview actions) and execute mode (run sequences) + - Dual-mode parsing: LLM-powered + rule-based fallback +- Object interaction system (add, pickup, drop, throw with trajectory physics) +- **World Generation**: LLM-powered themed environment creation +- Eye tracking and attention system (follows mouse cursor) +- Emotion/gesture system (wave, dance, jump, idle animations) +- Real-time speech synthesis via Azure TTS or local fallback +- Server-synchronized state management (character position, objects, expressions) + +**Autonomous Learning:** +- Self-discovering dataset collection from multiple sources +- Adaptive epoch selection based on performance history +- Automatic model promotion when accuracy thresholds met +- Performance degradation detection and alerting +- Continuous 30-minute training cycles with graceful error recovery + +**Multi-Provider Chat:** +- Azure OpenAI, OpenAI, LMStudio, local models +- LoRA adapter support for fine-tuned models +- Automatic provider fallback chain +- SSE-based streaming responses + +## Quick Commands (from repo root) + +```bash +# === AUTONOMOUS SYSTEMS (Self-Managing) === +# Start autonomous training (continuous 30-min cycles) +nohup python scripts/autonomous_training_orchestrator.py > data_out/autonomous_training.log 2>&1 & + +# Trigger immediate cycle (skip 30-min wait) +pkill -USR1 -f autonomous_training + +# Full repo automation (Aria + training + quantum + monitoring) +python scripts/repo_automation.py --start +python scripts/repo_automation.py --status +./scripts/start_repo_automation.sh full # Bash wrapper with menu +./scripts/start_repo_automation.sh stop # Stop all components + +# Aria character automation (server + continuous training) +python scripts/aria_automation.py --mode full +python scripts/aria_automation.py --status + +# === ARIA CHARACTER WEB UI === +cd apps/aria && python server.py # Start Aria web interface (port 8080) +# Access at: http://localhost:8080 +# Auto-Execute UI: http://localhost:8080/auto-execute.html +# Commands: "move left", "wave", "dance", "jump", "pickup ball", "throw" +# Complex: "Walk to the table and pick up the apple", "Say hello and wave" + +# === AZURE FUNCTIONS & APIs === +func host start # Start Functions host (serves all APIs) +curl http://localhost:7071/api/ai/status | jq # Health check + +# === TESTING & VALIDATION === +python scripts/test_runner.py --unit # Fast unit tests +python scripts/test_runner.py --all # All tests +python scripts/test_runner.py --unit --coverage # Unit tests + coverage +python scripts/test_runner.py --list-suites # Show available suites +python scripts/test_runner.py --unit --watch # Re-run on file changes +python ai-projects/chat-cli/src/chat_cli.py --provider local --once "Hello" # Smoke test +python scripts/fast_validate.py # Quick validation (datasets, scripts, venvs, configs, providers, deps) +python scripts/cleanup_artifacts.py # Preview old artifact cleanup (dry-run) +python scripts/cleanup_artifacts.py --apply # Actually delete old artifacts + +# === ORCHESTRATORS (Manual Execution) === +python scripts/autotrain.py --dry-run # Validate training config (12 jobs) +python scripts/quantum_autorun.py --dry-run # Validate quantum config +python scripts/evaluation_autorun.py --dry-run # Validate evaluation config + +# === TRAINING PIPELINES === +python scripts/automated_training_pipeline.py --quick # Quick LoRA (TinyLlama) +python scripts/train_and_promote.py --quick --auto-promote # Train + auto-deploy + +# === MCP & TOOLS === +python ai-projects/quantum-ml/quantum_mcp_server.py # Start quantum MCP server + +# === MONITORING & DIAGNOSTICS === +curl http://localhost:7071/api/ai/status | jq # Comprehensive health check +python scripts/status_dashboard.py # Unified orchestrator status +python scripts/status_dashboard.py --watch # Auto-refresh every 10s +python scripts/resource_monitor.py --snapshot # CPU/memory/disk/GPU snapshot +python scripts/system_health_check.py # Full system health report +python scripts/training_analytics.py # Performance trends & insights +tail -f data_out/autonomous_training.log # Live autonomous training logs +watch -n 5 'cat data_out/autonomous_training_status.json | python -m json.tool' # Live status +``` + +## Critical Patterns + +**Autonomous/self-managing systems:** +- `scripts/autonomous_training_orchestrator.py` — Continuous learning with 30-min cycles (infinite by default) + - Self-discovers datasets (scans `datasets/quantum`, `datasets/chat`, `datasets/massive_quantum`) + - Self-optimizes: Adaptive epochs `[25, 50, 100, 200]` based on performance history + - Self-heals: Graceful error handling, continues on failure, logs to `data_out/autonomous_training.log` + - State: `data_out/autonomous_training_status.json` (cycles_completed, best_accuracy, dataset_inventory) + - Config: `config/autonomous_training.yaml` (cycle_interval_minutes, epochs_progression, min_datasets) + - Trigger: Time-based (30min) OR signal-based (`pkill -USR1 -f autonomous_training`) +- `scripts/repo_automation.py` — Full repo automation (all components: Aria + training + quantum + datasets) +- `scripts/aria_automation.py` — Aria-specific automation (server on port 8080 + continuous training) +- `scripts/master_orchestrator.py` — Coordinates all sub-orchestrators with schedules/dependencies + - Config: `config/master_orchestrator.yaml` (cron schedules, priorities, retry logic, timeouts) + +**Data conventions:** +- `datasets/` is **read-only** — never modify existing datasets +- All outputs go to `data_out//` with `status.json` as source of truth +- Chat datasets: `[{"messages": [{"role": "user|assistant", "content": "..."}]}]` +- LoRA adapters need both `adapter_config.json` + `adapter_model.safetensors` + +**Provider detection chain** (in `shared/chat_providers.py`): +1. Explicit choice (--provider flag) +2. LMStudio (if `LMSTUDIO_BASE_URL` configured) +3. Azure OpenAI (needs all 4: `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT`, `AZURE_OPENAI_API_VERSION`) +4. OpenAI (needs `OPENAI_API_KEY`) +5. LoRA (explicit --provider lora with adapter path) +6. Local fallback (zero-dependency echo) + +**Config precedence:** +`YAML base` < `CLI flags` < `per-job YAML` < `env vars` + +**YAML orchestrators:** +- All in `scripts/` with matching root YAMLs (e.g., `autotrain.yaml`, `quantum_autorun.yaml`) +- Write `data_out//status.json` with machine-readable job status +- Support `--dry-run` to validate before execution + +**Autonomous training implementation patterns:** +```python +# State machine: discovery → collection → training → analysis → optimization → deployment +async def run_single_cycle(cycle_number): + await discover_datasets() # Scan datasets/, catalog by category + await download_new_datasets() # Download if below min_datasets threshold + epochs = await select_optimal_epochs() # Adaptive: increase if accuracy < 0.70 or plateau + results = await train_cycle(epochs) # Distributed training with multiprocessing + await analyze_performance(results) # Track metrics, detect degradation + await optimization_cycle() # Hyperparameter tuning (if enabled) + await deployment_cycle() # Auto-deploy if accuracy > 0.90 (if enabled) +``` + +**Process management:** +- Autonomous systems run via `nohup` in background, logs to `data_out/*.log` +- Check status: `ps aux | grep -E "(autonomous|aria)" | grep -v grep` +- Manual trigger: `pkill -USR1 -f autonomous_training` forces immediate cycle +- Graceful shutdown: `pkill -TERM -f autonomous_training` + +**Performance monitoring & observability:** +- **Health endpoint**: `GET /api/ai/status` — Comprehensive system diagnostics + - Active provider detection (azure|openai|local|lora) + - Environment variable presence (Azure OpenAI, OpenAI, Cosmos, SQL) + - ML library availability (torch, transformers, peft) — in-process & venv + - SQL pool metrics with saturation alerts (warns at ≥80%) + - Cosmos DB health check (lazy connection) + - Quantum environment status (qiskit, pennylane, Azure Quantum backends) + - LoRA adapter readiness (adapter_config.json, tokenizer) +- **Status files**: All orchestrators write `data_out//status.json` + - Schema: `{total_jobs, succeeded, failed, running, last_updated, avg_duration}` + - Autonomous training: `{cycles_completed, best_accuracy, performance_history[], dataset_inventory}` +- **Monitoring scripts**: + - `scripts/status_dashboard.py` — Unified view of all orchestrators (supports --watch, --export) + - `scripts/resource_monitor.py` — CPU/memory/disk/GPU with threshold alerts + - `scripts/system_health_check.py` — Comprehensive health report (venvs, Azure Functions, datasets) + - `scripts/training_analytics.py` — Performance trends, improvement rates, plateau detection +- **Performance degradation alerts**: Auto-detect >5% accuracy drops between cycles +- **Metrics tracked**: mean_accuracy, median_accuracy, max_accuracy, successful_count, exceptional_models +- **Notification config**: `config/notification_config.yaml` (email/SMTP/local alerts) + +## Where to Edit + +| Change | File(s) | +|--------|---------| +| Add/modify API endpoint | `function_app.py` | +| Quantum LLM pipeline | `ai-projects/quantum-ml/src/quantum_llm/pipeline.py` | +| Quantum token sampler | `ai-projects/quantum-ml/src/quantum_llm/quantum_sampler.py` | +| Quantum embedding transformer | `ai-projects/quantum-ml/src/quantum_llm/quantum_embeddings.py` | +| Quantum provider router | `ai-projects/quantum-ml/src/quantum_llm/quantum_router.py` | +| Quantum LLM config | `ai-projects/quantum-ml/src/quantum_llm/config.py` | +| Chat provider logic | `ai-projects/chat-cli/src/chat_providers.py` (re-exported by `shared/chat_providers.py`) | +| Training orchestration | `scripts/autotrain.py` + root `autotrain.yaml` | +| Autonomous training behavior | `scripts/autonomous_training_orchestrator.py` + `config/autonomous_training.yaml` | +| Master orchestrator (schedules/coordination) | `scripts/master_orchestrator.py` + `config/master_orchestrator.yaml` | +| Aria automation | `scripts/aria_automation.py` (server + training + health monitoring) | +| Full repo automation | `scripts/repo_automation.py` (all components + backups + notifications) | +| Quantum jobs | `scripts/quantum_autorun.py` + root `quantum_autorun.yaml` | +| MCP server tools | `ai-projects/quantum-ml/quantum_mcp_server.py` | +| Shared DB/telemetry | `shared/sql_engine.py`, `shared/telemetry.py`, `shared/cosmos_client.py` | +| Aria character interface | `apps/aria/index.html`, `apps/aria/aria_controller.js`, `apps/aria/server.py` | +| Aria movement/gestures | `apps/aria/aria_controller.js` (command parsing & animation triggers) | +| Semantic memory/embeddings | `shared/chat_memory.py` (generate_embedding, fetch_similar, store) | +| Token management | `ai-projects/chat-cli/src/token_utils.py` (counting, pruning) | +| LLM tool generation | `ai-projects/llm-maker/src/tool_maker.py`, `tool_validator.py` | +| Website generation | `ai-projects/llm-maker/src/website_maker.py` | +| Cooking AI recipes | `ai-projects/cooking-ai/src/agents/recipe_agent.py` | +| Subscriptions/monetization | `shared/subscription_manager.py` + `setup_monetization.py` | +| DB logging (fault-tolerant) | `shared/db_logging.py` (SP wrappers) | +| Vision/expression AI | `scripts/vision_inference.py` (TinyConvNet) | +| Batch model evaluation | `scripts/batch_evaluator.py` + `config/evaluation/` | +| Monitoring dashboard | `apps/dashboard/` (hub, analytics, GPU monitor) | +| AGI reasoning | `ai-projects/chat-cli/src/agi_provider.py` | +| Test runner | `scripts/test_runner.py` (centralized suite orchestrator) | +| Request validation | `shared/request_validator.py` (JSON schema validation) | +| Artifact cleanup | `scripts/cleanup_artifacts.py` (data_out/ retention) | +| Model evaluation | `scripts/evaluate_model.py` (delegates or fallback metrics) | +| Fast validation | `scripts/fast_validate.py` (configs, providers, deps) | + +## Safety Rules + +- Always `--dry-run` orchestrators before GPU/QPU execution +- Quantum: simulate locally first, then use `azure_ionq_simulator`, only then real QPU +- Real QPU jobs require `azure_confirm_cost: true` in YAML + cost estimate review +- Never hardcode secrets — use `local.settings.json` (dev) or Azure App Settings (prod) +- Monitor DB pool via `/api/ai/status` (warns at ≥80% saturation) + +## Testing & Validation + +- **Unit tests:** `pytest tests/ -m "not slow and not azure"` or `python scripts/test_runner.py --unit` +- **Integration tests:** `python scripts/test_runner.py --integration` +- **All tests:** `python scripts/test_runner.py --all` +- **VS Code Test Explorer:** Use 🧪 icon for interactive test running +- **Markers:** `@pytest.mark.slow`, `@pytest.mark.azure`, `@pytest.mark.integration` +- **Chat dataset validation:** `python scripts/validate_datasets.py --category chat` + +## Optional Services + +**SQL persistence** (optional): +- Enable via `QAI_DB_CONN` env var (SQLite, PostgreSQL, Azure SQL) +- Pool size: `QAI_SQL_POOL_SIZE` (default: 10) +- Health: Check `/api/ai/status` for pool saturation (warns ≥80%) + +**Cosmos DB** (optional, feature-flagged): +- Enable: `QAI_ENABLE_COSMOS=true` +- Config: `COSMOS_ENDPOINT`, `COSMOS_KEY`, `COSMOS_DATABASE`, `COSMOS_CONTAINER` +- Partition key: `/session_id`, enable TTL for cost savings + +**Telemetry** (optional): +- Application Insights via `APPLICATIONINSIGHTS_CONNECTION_STRING` +- Non-blocking, gracefully degrades if unavailable + +## Modular Instructions + +This repo uses component-specific instruction files in `.github/instructions/`: +- `functions.instructions.md` — Azure Functions API endpoints +- `shared-python.instructions.md` — Shared infrastructure patterns +- `quantum-ai*.instructions.md` — Quantum ML workflows +- `talk-to-ai*.instructions.md` — Chat CLI patterns +- `lora*.instructions.md` — LoRA fine-tuning patterns +- `chat-web.instructions.md` — Frontend SSE integration +- `agi-provider.instructions.md` — AGI reasoning system +- `aria-character.instructions.md` — Interactive character system +- `aria-web.instructions.md` — Aria web server module +- `autonomous-training.instructions.md` — Autonomous training orchestration +- `training-scripts.instructions.md` — Training script patterns +- `orchestrator-configs.instructions.md` — YAML orchestrator configs +- `dashboard.instructions.md` — Monitoring dashboard +- `tests.instructions.md` — Testing infrastructure +- `llm-maker.instructions.md` — Safe tool/website generation +- `cooking-ai.instructions.md` — Cooking AI recipe agent +- `chat-providers.instructions.md` — Multi-provider chat system +- `chat-memory.instructions.md` — Semantic memory & embeddings +- `subscription.instructions.md` — Subscription/monetization +- `db-logging.instructions.md` — Fault-tolerant DB logging +- `telemetry.instructions.md` — OpenTelemetry setup +- `token-utils.instructions.md` — Token counting & context pruning +- `evaluation.instructions.md` — Batch evaluation & analytics +- `vision-inference.instructions.md` — Vision AI & CNN models + +These are automatically applied by VS Code based on file paths. Check attachment indicators to see which rules are active. + +## Custom Coding Agents + +Available agents in `.github/agents/`: + +| Agent | Purpose | +|-------|---------| +| `ai.agent.md` | Primary autonomous agent — task decomposition, multi-step execution | +| `my-agent.agent.md` | QAI specialist — quantum-AI/ML development | +| `agi-reasoning.agent.md` | Chain-of-thought reasoning, self-reflection (CoT is internal, final answer only) | +| `visible-reasoning.agent.md` | Visible step-by-step reasoning, shows CoT trace to users | +| `aria-character.agent.md` | Interactive character commands, animations | +| `autonomous-trainer.agent.md` | LoRA training lifecycle, model promotion | +| `full-stack-debugger.agent.md` | Cross-stack issue diagnosis | +| `automated-code-fixer.agent.md` | Autonomous code improvements | +| `ai-architect.agent.md` | AI pipeline design, provider integration | +| `llm-maker.agent.md` | Safe tool/website generation | +| `chat-provider.agent.md` | Multi-provider chat, streaming, memory | +| `platform-ops.agent.md` | Subscriptions, monitoring, deployment | +| `vision-ai.agent.md` | Expression/emotion classification | +| `data-pipeline.agent.md` | Batch evaluation, dataset management | + +**Mode equivalents now live in `.github/agents/`**: +- `AI_model_training.agent.md` — End-to-end LoRA training, evaluation, and model promotion +- `Aria_character_development.agent.md` — Interactive character commands, actions, world generation +- `Quantum_ML_development.agent.md` — Quantum circuits, simulation, Azure Quantum pipelines +- `Full_stack_debugging.agent.md` — Cross-stack diagnostic protocol +- `AI_chat_development.agent.md` — Multi-provider chat, streaming, memory, self-learning +- `Azure_function_codegen_and_deployment.agent.md` — Enterprise Azure Functions workflow with IaC +- `Azure_Static_Web_App.agent.md` — Static web app deployment patterns + +**Prompts** (`.github/prompts/`): +- `agi.prompt.md` — Autonomous AGI reasoning with multi-step analysis and self-correction (chain-of-thought is internal, not exposed in output) +- `reason.prompt.md` — Visible step-by-step reasoning that exposes chain-of-thought, confidence scores, and self-reflection to the user (uses `visible-reasoning` agent) +- `debug.prompt.md` — Systematic diagnostic protocol +- `review.prompt.md` — Code review (correctness, security, performance) +- `aria-command.prompt.md` — Natural language → Aria actions +- `train.prompt.md` — Training execution with safety +- `quantum.prompt.md` — Cost-aware quantum workflows +- `chat.prompt.md` — Multi-provider chat with memory +- `generate-tool.prompt.md` — Safe Python tool generation +- `generate-website.prompt.md` — Complete website generation +- `evaluate.prompt.md` — Model evaluation & benchmarking +- `deploy.prompt.md` — Model/service deployment +- `optimize.prompt.md` — Performance analysis & optimization + +**Usage**: Agents are invoked automatically based on context or explicitly selected in GitHub Copilot interfaces. + +## Coding Agent Best Practices + +**For AI Coding Agents working in this repository:** + +1. **Always Check Context** + - Read `.github/copilot-instructions.md` (this file) first + - Check for relevant `.github/instructions/*.instructions.md` files based on file paths + - Reference `.github/agents/my-agent.agent.md` for QAI-specific patterns + +2. **Safety-First Approach** + - `--dry-run` all orchestrators before GPU/QPU execution + - Never modify files in `datasets/` (read-only) + - Check `/api/ai/status` before making provider-dependent changes + - Verify test suite passes: `python scripts/test_runner.py --unit` + +3. **Follow Established Patterns** + - Provider detection chain: Azure OpenAI → OpenAI → LMStudio → Local + - Config precedence: `YAML base` < `CLI flags` < `per-job YAML` < `env vars` + - Status files: Always write to `data_out//status.json` + - Autonomous systems: Use signal-based triggers (`pkill -USR1`) for immediate execution + +4. **Testing & Validation** + - Run unit tests before committing: `python scripts/test_runner.py --unit` + - Use `scripts/fast_validate.py` for quick cross-component validation + - Check health endpoint: `curl http://localhost:7071/api/ai/status | jq` + - Monitor logs: `tail -f data_out/autonomous_training.log` + +5. **Documentation Updates** + - Update this file when adding major features or changing workflows + - Keep component-specific instructions in `.github/instructions/` in sync + - Update PR checklist if adding new safety requirements + - Document new orchestrators in "Where to Edit" table + +6. **Cost & Resource Awareness** + - Quantum: Simulate locally first, then Azure simulator, only then real QPU + - Monitor DB pool saturation via `/api/ai/status` (warns at ≥80%) + - Check GPU/CPU usage: `python scripts/resource_monitor.py --snapshot` + - Review training analytics: `python scripts/training_analytics.py` + +## PR Checklist for AI Agents & Reviewers + +Before submitting or approving PRs, verify: + +- [ ] **Dry-run orchestrators**: If modifying YAML configs or orchestrators, run `--dry-run` to validate changes before committing +- [ ] **Provider detection intact**: Changes to `shared/chat_providers.py` or `function_app.py` don't break detection chain (test with `/api/ai/status`) +- [ ] **Dataset immutability**: No modifications to `datasets/` — all outputs written to `data_out/` +- [ ] **Status.json compliance**: Orchestrator changes maintain status JSON writes to `data_out//status.json` +- [ ] **Test suite passes**: Run `python scripts/test_runner.py --unit` (or `--all` for integration tests) +- [ ] **No hardcoded secrets**: All API keys/connection strings use env vars or `local.settings.json` +- [ ] **Quantum cost gates**: QPU jobs include `azure_confirm_cost: true` in YAML configs +- [ ] **LoRA adapter validity**: If modifying training scripts, verify output includes both `adapter_config.json` + `adapter_model.safetensors` +- [ ] **Documentation sync**: Update relevant READMEs/instruction files if changing core workflows or adding features + +Full/verbose guidance and advanced examples are preserved at `.github/copilot-instructions.full.md`. Ask me to expand any area or add examples for a specific change. diff --git a/.github/prompts/agi.prompt.md b/.github/prompts/agi.prompt.md index f97ac3d60..d79e17060 100644 --- a/.github/prompts/agi.prompt.md +++ b/.github/prompts/agi.prompt.md @@ -1,5 +1,5 @@ --- -description: "Engage autonomous AGI reasoning with multi-step analysis, task decomposition, self-correction, and iterative improvement. Use when: long-running autonomous work, plan-then-implement tasks, deep reasoning with concise summaries, extended code generation, iterative debugging, repo-wide fixes, or self-correcting execution." +description: "Engage autonomous AGI reasoning with multi-step analysis, task decomposition, self-correction, and iterative improvement. Chain-of-thought reasoning runs internally — only the final answer is delivered to the user. For visible step-by-step reasoning, use reason.prompt instead." name: "AGI Reasoning" argument-hint: "Problem or task description (example: analyze the auth flow for race conditions + constraints)" agent: agi-reasoning diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 6ca5cb208..ecabcfb4f 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -26,7 +26,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup ShellCheck run: sudo apt-get update && sudo apt-get install -y shellcheck diff --git a/.github/workflows/api-health-smoke.yml b/.github/workflows/api-health-smoke.yml index a27e73656..316fb88e0 100644 --- a/.github/workflows/api-health-smoke.yml +++ b/.github/workflows/api-health-smoke.yml @@ -25,6 +25,9 @@ concurrency: group: api-health-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true # Opt into Node.js 24 for JS actions ahead of June 2026 forced migration + jobs: smoke: name: API Smoke Validation @@ -33,7 +36,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -55,7 +58,7 @@ jobs: - name: Upload smoke artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: api-health-smoke-${{ github.run_number }} retention-days: 14 diff --git a/.github/workflows/artifact-lifecycle.yml b/.github/workflows/artifact-lifecycle.yml index 6a1165e36..59c824c41 100644 --- a/.github/workflows/artifact-lifecycle.yml +++ b/.github/workflows/artifact-lifecycle.yml @@ -47,7 +47,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -75,7 +75,7 @@ jobs: - name: Upload cleanup summary if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: artifact-lifecycle-${{ github.run_number }} retention-days: 30 diff --git a/.github/workflows/autonomous-evolver.yml b/.github/workflows/autonomous-evolver.yml index f9cf14054..f2004aff1 100644 --- a/.github/workflows/autonomous-evolver.yml +++ b/.github/workflows/autonomous-evolver.yml @@ -12,9 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 20 diff --git a/.github/workflows/broken-links.yml b/.github/workflows/broken-links.yml index 9806aab1d..a80a6aa6c 100644 --- a/.github/workflows/broken-links.yml +++ b/.github/workflows/broken-links.yml @@ -26,7 +26,7 @@ jobs: timeout-minutes: 15 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Check links in docs and markdown uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2.8.0 @@ -36,7 +36,6 @@ jobs: --verbose --no-progress --max-concurrency 8 - --exclude-mail --accept 200,204,206,301,302,307,308,401,403,429 '**/*.md' 'docs/**/*.md' diff --git a/.github/workflows/coverage-report.yml b/.github/workflows/coverage-report.yml index bcd0fdd4a..6cc1887ce 100644 --- a/.github/workflows/coverage-report.yml +++ b/.github/workflows/coverage-report.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -68,7 +68,7 @@ jobs: # On main pushes → store as the next PR's baseline - name: Upload main baseline artifact if: github.ref == 'refs/heads/main' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: coverage-main-baseline path: coverage.xml @@ -79,7 +79,7 @@ jobs: # On PRs → also upload per-SHA artifact for auditability - name: Upload PR coverage artifact if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: coverage-pr-${{ github.event.pull_request.number }} path: coverage.xml @@ -122,7 +122,7 @@ jobs: - name: Post PR coverage comment if: github.event_name == 'pull_request' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 env: COV_TOTAL: ${{ steps.run.outputs.total }} COV_BASE: ${{ steps.baseline.outputs.total }} diff --git a/.github/workflows/dataset-integrity.yml b/.github/workflows/dataset-integrity.yml index 405203428..7a921eee0 100644 --- a/.github/workflows/dataset-integrity.yml +++ b/.github/workflows/dataset-integrity.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 @@ -105,7 +105,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index bfa159995..b1bade985 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -18,10 +18,10 @@ jobs: timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Dependency Review - uses: actions/dependency-review-action@v4 + uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4 with: fail-on-severity: high deny-licenses: GPL-2.0, GPL-3.0, AGPL-3.0 diff --git a/.github/workflows/devcontainer-ci.yml b/.github/workflows/devcontainer-ci.yml index 62cc85fa2..3a133267a 100644 --- a/.github/workflows/devcontainer-ci.yml +++ b/.github/workflows/devcontainer-ci.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Build and test devcontainer uses: devcontainers/ci@b63b30de439b47a52267f241112c5b453b673db5 # v0.3 diff --git a/.github/workflows/experiment-runner.yml b/.github/workflows/experiment-runner.yml index 3b05ebf69..2d9eb1a88 100644 --- a/.github/workflows/experiment-runner.yml +++ b/.github/workflows/experiment-runner.yml @@ -12,9 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 20 diff --git a/.github/workflows/integration-contract-gate.yml b/.github/workflows/integration-contract-gate.yml index b4320b526..5a5d6895c 100644 --- a/.github/workflows/integration-contract-gate.yml +++ b/.github/workflows/integration-contract-gate.yml @@ -33,6 +33,9 @@ concurrency: group: contract-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true # Opt into Node.js 24 for JS actions ahead of June 2026 forced migration + jobs: contract-gate: name: API Contract Validation @@ -40,7 +43,7 @@ jobs: timeout-minutes: 15 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -84,7 +87,7 @@ jobs: - name: Upload contract results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: contract-gate-results-${{ github.run_number }} retention-days: 14 diff --git a/.github/workflows/llm-maker-tests.yml b/.github/workflows/llm-maker-tests.yml index 4460e407c..1f00dc29e 100644 --- a/.github/workflows/llm-maker-tests.yml +++ b/.github/workflows/llm-maker-tests.yml @@ -17,10 +17,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.11' diff --git a/.github/workflows/markdown-quality.yml b/.github/workflows/markdown-quality.yml index e2bbd8517..825c6d7a1 100644 --- a/.github/workflows/markdown-quality.yml +++ b/.github/workflows/markdown-quality.yml @@ -24,10 +24,10 @@ jobs: timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Lint Markdown files - uses: DavidAnson/markdownlint-cli2-action@8305c0f1062bb0d184d09ef4493ecb9288447732 # v20 + uses: DavidAnson/markdownlint-cli2-action@6b51ade7a9e4a75a7ad929842dd298a3804ebe8b # v23 with: globs: | **/*.md diff --git a/.github/workflows/nightly-regression.yml b/.github/workflows/nightly-regression.yml index 01f78de9a..fec6d2833 100644 --- a/.github/workflows/nightly-regression.yml +++ b/.github/workflows/nightly-regression.yml @@ -31,6 +31,9 @@ concurrency: group: nightly-regression cancel-in-progress: false # Never cancel a nightly run mid-flight +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true # Opt into Node.js 24 for JS actions ahead of June 2026 forced migration + jobs: full-test-suite: name: Full Test Suite @@ -44,7 +47,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -96,7 +99,7 @@ jobs: - name: Upload test results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: nightly-test-results-${{ github.run_number }} retention-days: 30 @@ -128,7 +131,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Download previous baseline artifact id: download-prev @@ -171,7 +174,7 @@ jobs: - name: Open regression issue if: steps.compare.outputs.regression == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/platform-health-daily.yml b/.github/workflows/platform-health-daily.yml index 539ce546f..224b335d1 100644 --- a/.github/workflows/platform-health-daily.yml +++ b/.github/workflows/platform-health-daily.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -46,7 +46,7 @@ jobs: - name: Upload health artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: platform-health-${{ github.run_number }} retention-days: 30 diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index ce69a981f..476403168 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -28,6 +28,7 @@ permissions: contents: read env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true # Opt into Node.js 24 for JS actions ahead of June 2026 forced migration PYTHON_VERSION: '3.11' TEST_PATH: tests @@ -38,10 +39,10 @@ jobs: timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ env.PYTHON_VERSION }} cache: pip @@ -51,12 +52,12 @@ jobs: pyproject.toml setup.cfg - - name: Install dev dependencies + - name: Install dependencies shell: bash run: | set -euo pipefail python -m pip install --upgrade pip - + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [[ -f requirements-dev.txt ]]; then python -m pip install -r requirements-dev.txt elif [[ -f pyproject.toml ]]; then @@ -91,10 +92,9 @@ jobs: test-path: ${{ env.TEST_PATH }} extra-args: --maxfail=1 -q env: - CI: 'true' + CI: true watcher: - name: Test watcher needs: test if: >- ( @@ -108,25 +108,16 @@ jobs: image: mcr.microsoft.com/devcontainers/python:1-3.11 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Install dev dependencies (in container) - shell: bash run: | - set -euo pipefail python -m pip install --upgrade pip - - if [[ -f requirements-dev.txt ]]; then - python -m pip install -r requirements-dev.txt - else - python -m pip install pytest watchdog - fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; else pip install pytest watchdog; fi - name: Run test watcher (runs tests once then exits via timeout) - shell: bash run: | - set -euo pipefail timeout "${{ github.event.inputs.watcher_timeout_minutes || '10' }}m" \ - python3 scripts/test_watcher.py --cmd "pytest ${{ env.TEST_PATH }} -q --maxfail=1" + python3 scripts/test_watcher.py --cmd "pytest tests -q --maxfail=1" env: - CI: 'true' + CI: true diff --git a/.github/workflows/quantum-ai-smoke.yml b/.github/workflows/quantum-ai-smoke.yml index 3fce108a2..16faa02f6 100644 --- a/.github/workflows/quantum-ai-smoke.yml +++ b/.github/workflows/quantum-ai-smoke.yml @@ -23,7 +23,7 @@ jobs: timeout-minutes: 20 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -42,7 +42,7 @@ jobs: - name: Upload smoke context if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: quantum-ai-smoke-${{ github.run_number }} retention-days: 7 diff --git a/.github/workflows/secrets-scan.yml b/.github/workflows/secrets-scan.yml index c744d5125..e70cf5a6e 100644 --- a/.github/workflows/secrets-scan.yml +++ b/.github/workflows/secrets-scan.yml @@ -30,12 +30,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - name: Run Gitleaks - uses: gitleaks/gitleaks-action@v2 + uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # GITLEAKS_LICENSE is required for org-level repos on enterprise plans. @@ -49,7 +49,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 diff --git a/.github/workflows/site-bundle-validation.yml b/.github/workflows/site-bundle-validation.yml index 088dca707..e39ac1c55 100644 --- a/.github/workflows/site-bundle-validation.yml +++ b/.github/workflows/site-bundle-validation.yml @@ -23,7 +23,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -37,7 +37,7 @@ jobs: - name: Upload validation context if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: generated-site-bundles-${{ github.run_number }} retention-days: 7 diff --git a/.github/workflows/telemetry-trigger.yml b/.github/workflows/telemetry-trigger.yml index 432c68144..90cc45cdf 100644 --- a/.github/workflows/telemetry-trigger.yml +++ b/.github/workflows/telemetry-trigger.yml @@ -10,9 +10,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 20 diff --git a/.github/workflows/test-watcher.yml b/.github/workflows/test-watcher.yml index f589f2def..374086052 100644 --- a/.github/workflows/test-watcher.yml +++ b/.github/workflows/test-watcher.yml @@ -10,9 +10,9 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.11' - name: Install dependencies diff --git a/.github/workflows/training-health-report.yml b/.github/workflows/training-health-report.yml index f90dae3fc..9869c3e16 100644 --- a/.github/workflows/training-health-report.yml +++ b/.github/workflows/training-health-report.yml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python uses: ./.github/actions/setup-python-env @@ -47,7 +47,7 @@ jobs: if [ ! -f data_out/autonomous_training_status.json ]; then echo "No training status found — running 1 seed cycle to bootstrap data..." python scripts/autonomous_training_orchestrator.py \ - --cycles 1 --interval 0 --skip-quantum 2>&1 || echo "Seed training failed (likely missing dependencies or invalid config), workflow will proceed with empty data..." + --cycles 1 --interval 0 --skip-quantum 2>&1 || echo "Seed training failed — workflow will proceed with no data (fresh checkout or missing dependencies)." else echo "Existing training status found, skipping seed." fi @@ -167,7 +167,7 @@ jobs: - name: Open degradation issue if: steps.status.outputs.degraded == 'true' || github.event.inputs.open-issue == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/ai-projects/lora-training/microsoft_phi-silica-3.6_v1/scripts/train_lora.py b/ai-projects/lora-training/microsoft_phi-silica-3.6_v1/scripts/train_lora.py index 1e44c132a..6acf50b2c 100644 --- a/ai-projects/lora-training/microsoft_phi-silica-3.6_v1/scripts/train_lora.py +++ b/ai-projects/lora-training/microsoft_phi-silica-3.6_v1/scripts/train_lora.py @@ -1,940 +1,940 @@ -import argparse -import json -import math -import os -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, Iterable, List - -try: - import yaml # type: ignore -except Exception: - raise SystemExit("pyyaml is required. Install with: pip install pyyaml") from None -try: - # Optional: install torch if missing - try: - import torch # pip install torch # type: ignore[reportMissingImports] - except ImportError: - raise SystemExit( - "PyTorch is required. Install with: pip install torch" - ) from None - from datasets import load_dataset # type: ignore[import] - - try: - from peft import LoraConfig, PeftModel, get_peft_model - from transformers import (AutoModelForCausalLM, AutoTokenizer, - DataCollatorForLanguageModeling, - EarlyStoppingCallback, Trainer, - TrainerCallback, TrainingArguments) - except ImportError: - raise SystemExit( - "Transformers is required. Install with: pip install transformers" - ) from None -except Exception as e: - # Provide visibility into which dependency import failed - import traceback - - print(f"[import-debug] training dependency import failed: {e}") - traceback.print_exc() - # We'll allow dry-run without these by nulling all symbols - torch = None # type: ignore - load_dataset = None # type: ignore - AutoModelForCausalLM = AutoTokenizer = Trainer = TrainingArguments = DataCollatorForLanguageModeling = None # type: ignore - LoraConfig = get_peft_model = PeftModel = None # type: ignore - TrainerCallback = None # type: ignore - EarlyStoppingCallback = None # type: ignore - - -@dataclass -class Config: - model: str - finetune_dataset: str - save_dir: str - finetune_train_nsamples: int | None - finetune_test_nsamples: int | None - finetune_train_batch_size: int - finetune_test_batch_size: int - finetune_train_seqlen: int - finetune_test_seqlen: int - learning_rate: float - lora_dropout: float - epochs: int - eval_steps: int - save_steps: int - gradient_checkpointing: bool - seed: int - warmup_steps: int - gradient_accumulation_steps: int - max_grad_norm: float - early_stopping_patience: int - early_stopping_threshold: float - - -def read_yaml(yaml_path: Path) -> Dict[str, Any]: - with yaml_path.open("r", encoding="utf-8") as f: - return yaml.safe_load(f) - - -def resolve_path(p: str) -> Path: - # allow tokens like mount//dataset to be overridden by --dataset - return Path(p).expanduser() - - -def iter_jsonl(path: Path) -> Iterable[Dict[str, Any]]: - with path.open("r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - yield json.loads(line) - - -def count_records(path: Path) -> int: - n = 0 - for _ in iter_jsonl(path): - n += 1 - return n - - -def validate_sample(path: Path) -> Dict[str, Any]: - for obj in iter_jsonl(path): - msgs = obj.get("messages") - if isinstance(msgs, list) and len(msgs) >= 2: - return obj - raise RuntimeError(f"No valid chat records found in {path}") - - -def build_text_from_messages(messages: List[Dict[str, str]]) -> str: - """Convert messages to training text with improved formatting and end tokens.""" - parts: List[str] = [] - for m in messages: - role = m.get("role", "").lower() - content = m.get("content", "").strip() - if not content: # Skip empty messages - continue - if role == "system": - parts.append(f"<|system|>\n{content}<|end|>\n") - elif role == "user": - parts.append(f"<|user|>\n{content}<|end|>\n") - elif role == "assistant": - parts.append(f"<|assistant|>\n{content}<|end|>\n") - return "".join(parts) - - -def make_hf_dataset_from_files( - train_files: List[str], eval_files: List[str], streaming: bool = True -): - if load_dataset is None: - raise RuntimeError( - "HuggingFace datasets not available. Install 'datasets' to load datasets." - ) - data_files = {"train": train_files, "validation": eval_files} - ds = load_dataset("json", data_files=data_files, streaming=streaming) - return ds - - -def _read_text_source(path_or_url: str) -> Iterable[str]: - if path_or_url.startswith("http://") or path_or_url.startswith("https://"): - import urllib.request - - with urllib.request.urlopen(path_or_url) as resp: # nosec B310 - for line in resp.read().decode("utf-8").splitlines(): - yield line - else: - p = Path(path_or_url) - with p.open("r", encoding="utf-8") as f: - for line in f: - yield line.rstrip("\n") - - -def parse_manifest(path_or_url: str) -> List[str]: - urls: List[str] = [] - lower = path_or_url.lower() - if lower.endswith(".json"): - import json as _json - - if path_or_url.startswith("http://") or path_or_url.startswith("https://"): - import urllib.request - - with urllib.request.urlopen(path_or_url) as resp: # nosec B310 - obj = _json.loads(resp.read().decode("utf-8")) - else: - with Path(path_or_url).open("r", encoding="utf-8") as f: - obj = _json.load(f) - if isinstance(obj, dict): - for key in ("train", "validation", "urls", "files"): - v = obj.get(key) - if isinstance(v, list): - urls.extend([str(x) for x in v]) - if not urls and "url" in obj: - urls.append(str(obj["url"])) - elif isinstance(obj, list): - urls.extend([str(x) for x in obj]) - elif lower.endswith(".jsonl"): - import json as _json - - for line in _read_text_source(path_or_url): - if not line.strip(): - continue - try: - rec = _json.loads(line) - if isinstance(rec, str): - urls.append(rec) - elif isinstance(rec, dict) and "url" in rec: - urls.append(str(rec["url"])) - except Exception: - urls.append(line.strip()) - else: - for line in _read_text_source(path_or_url): - if line.strip(): - urls.append(line.strip()) - # Dedupe - seen = set() - uniq: List[str] = [] - for u in urls: - if u not in seen: - seen.add(u) - uniq.append(u) - return uniq - - -def main(): - ap = argparse.ArgumentParser( - description="Train LoRA on chat dataset using lora.yaml config" - ) - ap.add_argument( - "--config", - default=str(Path(__file__).resolve().parents[1] / "lora" / "lora.yaml"), - ) - ap.add_argument( - "--dataset", default=str(Path(__file__).resolve().parents[1] / "data") - ) - ap.add_argument( - "--dry-run", - action="store_true", - help="Validate dataset/config only; no model download", - ) - ap.add_argument( - "--max-train-samples", - type=int, - default=None, - help="Limit train examples (for smoke test)", - ) - ap.add_argument( - "--max-eval-samples", - type=int, - default=None, - help="Limit eval examples (for smoke test)", - ) - ap.add_argument( - "--hf-model-id", - default=None, - help="Override HF model id (e.g., microsoft/Phi-3.5-mini-instruct)", - ) - ap.add_argument( - "--no-stream", action="store_true", help="Disable streaming mode for datasets" - ) - ap.add_argument( - "--deepspeed", - default=None, - help="Path to DeepSpeed config JSON to enable ZeRO (multi-GPU)", - ) - ap.add_argument( - "--train-manifest", - default=None, - help="Path or URL to manifest of training files (txt/json/jsonl)", - ) - ap.add_argument( - "--eval-manifest", - default=None, - help="Path or URL to manifest of eval files (txt/json/jsonl)", - ) - ap.add_argument( - "--save-dir", - default=None, - help="Override output directory (else from config or defaults)", - ) - ap.add_argument( - "--device", - default="auto", - choices=["auto", "cuda", "cpu", "directml", "mps"], - help="Device preference: auto selects best available (cuda>mps>directml>cpu)", - ) - # Optional overrides for HPO/cloud runs - ap.add_argument( - "--learning-rate", - type=float, - default=None, - help="Override learning_rate from config", - ) - ap.add_argument( - "--lora-dropout", - type=float, - default=None, - help="Override lora_dropout from config", - ) - ap.add_argument( - "--epochs", type=int, default=None, help="Override epochs from config" - ) - ap.add_argument( - "--train-batch-size", - type=int, - default=None, - help="Override finetune_train_batch_size from config", - ) - ap.add_argument( - "--eval-batch-size", - type=int, - default=None, - help="Override finetune_test_batch_size from config", - ) - ap.add_argument("--seed", type=int, default=None, help="Override seed from config") - args = ap.parse_args() - - # Initialize tracing (best-effort). This allows the optional - # OpenTelemetryTrainerCallback to get an active tracer if available. - # Optional tracing import (ignore if missing) - try: - from shared.tracing import init_tracing # type: ignore - - init_tracing(service_name="train_lora") - except Exception as _e: - print(f"[tracing] init skipped in train_lora: {_e}") - - cfg_raw = read_yaml(Path(args.config)) - cfg = Config( - model=cfg_raw.get("model") or "Phi-3.6-mini-instruct", - finetune_dataset=cfg_raw.get("finetune_dataset") or str(Path(args.dataset)), - save_dir=( - args.save_dir - or cfg_raw.get("save_dir") - or str(Path(__file__).resolve().parents[1] / "outputs") - ), - finetune_train_nsamples=cfg_raw.get("finetune_train_nsamples"), - finetune_test_nsamples=cfg_raw.get("finetune_test_nsamples"), - finetune_train_batch_size=int(cfg_raw.get("finetune_train_batch_size") or 2), - finetune_test_batch_size=int(cfg_raw.get("finetune_test_batch_size") or 2), - finetune_train_seqlen=int(cfg_raw.get("finetune_train_seqlen") or 1024), - finetune_test_seqlen=int(cfg_raw.get("finetune_test_seqlen") or 2048), - learning_rate=float(cfg_raw.get("learning_rate") or 2e-4), - lora_dropout=float(cfg_raw.get("lora_dropout") or 0.1), - epochs=int(cfg_raw.get("epochs") or 1), - eval_steps=int(cfg_raw.get("eval_steps", 64)), - save_steps=int(cfg_raw.get("save_steps") or 64), - gradient_checkpointing=bool(cfg_raw.get("gradient_checkpointing") or False), - seed=int(cfg_raw.get("seed") or 42), - warmup_steps=int(cfg_raw.get("warmup_steps") or 100), - gradient_accumulation_steps=int( - cfg_raw.get("gradient_accumulation_steps") or 4 - ), - max_grad_norm=float(cfg_raw.get("max_grad_norm") or 1.0), - early_stopping_patience=int(cfg_raw.get("early_stopping_patience") or 3), - early_stopping_threshold=float(cfg_raw.get("early_stopping_threshold") or 0.01), - ) - - # Apply CLI overrides for HPO or cloud jobs - if getattr(args, "learning_rate", None) is not None: - cfg.learning_rate = float(args.learning_rate) - if getattr(args, "lora_dropout", None) is not None: - cfg.lora_dropout = float(args.lora_dropout) - if getattr(args, "epochs", None) is not None: - cfg.epochs = int(args.epochs) - if getattr(args, "train_batch_size", None) is not None: - cfg.finetune_train_batch_size = int(args.train_batch_size) - if getattr(args, "eval_batch_size", None) is not None: - cfg.finetune_test_batch_size = int(args.eval_batch_size) - if getattr(args, "seed", None) is not None: - cfg.seed = int(args.seed) - - # Resolve data sources: manifests or local files - train_files: List[str] = [] - eval_files: List[str] = [] - # CLI overrides take precedence - train_manifest = getattr(args, "train_manifest", None) - eval_manifest = getattr(args, "eval_manifest", None) - if train_manifest or eval_manifest: - if train_manifest: - train_files = parse_manifest(train_manifest) - if eval_manifest: - eval_files = parse_manifest(eval_manifest) - if not train_files: - raise RuntimeError("No train files found from manifest") - if not eval_files: - # Fallback: use a small subset of train for eval - eval_files = train_files[:1] - else: - dataset_path = ( - Path(args.dataset) if args.dataset else resolve_path(cfg.finetune_dataset) - ) - if dataset_path.is_file(): - # Allow direct file usage (.json or .jsonl) - if dataset_path.suffix.lower() in (".json", ".jsonl"): - train_files = [str(dataset_path)] - eval_files = [str(dataset_path)] - else: - raise FileNotFoundError( - f"Unsupported dataset file type: {dataset_path}" - ) - else: - # Directory: accept train.json/test.json or train.jsonl/test.jsonl; fallback to single train file present - candidates = [ - (dataset_path / "train.json", dataset_path / "test.json"), - (dataset_path / "train.jsonl", dataset_path / "test.jsonl"), - ] - found = False - for t_path, v_path in candidates: - if t_path.exists() and v_path.exists(): - train_files = [str(t_path)] - eval_files = [str(v_path)] - found = True - break - if not found: - # Fallbacks: if any train.* exists, use it for both train/val - t_candidates = [ - dataset_path / "train.json", - dataset_path / "train.jsonl", - ] - t_use = next((p for p in t_candidates if p.exists()), None) - if t_use is None: - raise FileNotFoundError( - f"Expected dataset files at: {dataset_path}/train.json[.l] and optionally test.json[.l]" - ) - train_files = [str(t_use)] - eval_candidates = [ - dataset_path / "test.json", - dataset_path / "test.jsonl", - ] - v_use = next((p for p in eval_candidates if p.exists()), None) - eval_files = [str(v_use)] if v_use else [str(t_use)] - - # Determine and report device early (even for dry-run) - chosen_device = "cpu" - if args.device == "auto": - if ( - torch is not None - and getattr(torch, "cuda", None) - and getattr(torch.cuda, "is_available", lambda: False)() - ): - chosen_device = "cuda" - elif ( - torch is not None - and getattr(torch, "backends", None) - and getattr(torch.backends, "mps", None) - and getattr(torch.backends.mps, "is_available", lambda: False)() - ): - chosen_device = "mps" - else: - # Optional DirectML detection - try: - chosen_device = "directml" - except Exception: - chosen_device = "cpu" - else: - chosen_device = args.device - print( - f"[device] selection={args.device} resolved={chosen_device} cuda_available={(getattr(torch, 'cuda', None) and getattr(torch.cuda, 'is_available', lambda: False)() if torch else False)}" - ) - - # Dry run: count/validate records only (no model/tokenizer downloads) - if args.dry_run: - if train_manifest or eval_manifest: - print("Dry run OK.") - print( - { - "device": chosen_device, - "train_files": train_files[:5], - "eval_files": eval_files[:5], - "note": "Counting skipped for remote manifests", - } - ) - return - n_train = count_records(Path(train_files[0])) - n_test = count_records(Path(eval_files[0])) - sample = validate_sample(Path(train_files[0])) - print("Dry run OK.") - print( - { - "device": chosen_device, - "train_examples": n_train, - "test_examples": n_test, - "sample": sample, - } - ) - return - - # Real training requires heavy deps - if ( - AutoTokenizer is None - or AutoModelForCausalLM is None - or load_dataset is None - or torch is None - ): - missing = [] - if torch is None: - missing.append("torch") - if load_dataset is None: - missing.append("datasets") - if AutoTokenizer is None or AutoModelForCausalLM is None: - missing.append("transformers") - if LoraConfig is None: - missing.append("peft") - model_venv = Path(__file__).resolve().parents[1] / "venv" - root_venv = Path(__file__).resolve().parents[3] / "venv" - raise RuntimeError( - "Training dependencies not installed or import failed.\n" - f"missing={missing}\n" - f"To fix (model env): {model_venv / 'Scripts' / 'pip.exe'} install transformers datasets peft accelerate torch\n" - f"Alt (root env): {root_venv / 'Scripts' / 'pip.exe'} install transformers datasets peft accelerate torch\n" - "After installing, re-run this script." - ) - - # Resolve model id - hf_model_id = args.hf_model_id or os.environ.get("HF_MODEL_ID") - if hf_model_id is None: - # Best-effort mapping for local runs - # If the configured model isn't an HF id, default to a widely-available one - hf_model_id = { - "Phi-3.6-mini-instruct": "microsoft/Phi-3.5-mini-instruct", - }.get(cfg.model, cfg.model) - - tokenizer = AutoTokenizer.from_pretrained(hf_model_id, use_fast=True) - if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - - def preprocess(examples): - # Handle both single example and batched mapping - texts = [] - if isinstance(examples, dict) and "messages" in examples: - msgs = examples["messages"] - try: - print(f"[preprocess] messages type: {type(msgs)}") - if isinstance(msgs, dict): - print(f"[preprocess] messages dict keys: {list(msgs.keys())}") - # Attempt to reconstruct per-sample conversations if messages is a dict of lists - # Expect shape: msgs[key][i] gives ith sample's list of that field - keys = list(msgs.keys()) - batch_size = ( - len(msgs[keys[0]]) - if keys and isinstance(msgs[keys[0]], list) - else 0 - ) - print(f"[preprocess] inferred batch_size from dict: {batch_size}") - for i in range(batch_size): - # Reconstruct sample i as list of dicts using available fields - sample_messages = [] - # Try common fields 'role' and 'content' - roles = msgs.get("role", [])[i] if "role" in msgs else None - contents = ( - msgs.get("content", [])[i] if "content" in msgs else None - ) - if ( - isinstance(roles, list) - and isinstance(contents, list) - and len(roles) == len(contents) - ): - for r, c in zip(roles, contents): - sample_messages.append({"role": r, "content": c}) - else: - # Fallback: try to rebuild from a generic list of dicts if present - # msgs may have a single key representing the full objects - for k in keys: - candidate = ( - msgs[k][i] if isinstance(msgs[k], list) else None - ) - if ( - isinstance(candidate, list) - and candidate - and isinstance(candidate[0], dict) - ): - sample_messages = candidate - break - if sample_messages: - text = ( - tokenizer.apply_chat_template( - sample_messages, - tokenize=False, - add_generation_prompt=False, - ) - if hasattr(tokenizer, "apply_chat_template") - else build_text_from_messages(sample_messages) - ) - texts.append(text) - # Batched: list of lists - elif isinstance(msgs, list) and msgs and isinstance(msgs[0], list): - for obj in msgs: - if obj and isinstance(obj[0], dict): - text = ( - tokenizer.apply_chat_template( - obj, - tokenize=False, - add_generation_prompt=False, - ) - if hasattr(tokenizer, "apply_chat_template") - else build_text_from_messages(obj) - ) - texts.append(text) - # Single example: list of dicts - elif isinstance(msgs, list) and msgs and isinstance(msgs[0], dict): - text = ( - tokenizer.apply_chat_template( - msgs, - tokenize=False, - add_generation_prompt=False, - ) - if hasattr(tokenizer, "apply_chat_template") - else build_text_from_messages(msgs) - ) - texts.append(text) - except Exception as e: - print(f"[preprocess] error reconstructing messages: {e}") - # When using input_columns=["messages"], the function may receive just the messages column values - elif isinstance(examples, list): - msgs = examples - if isinstance(msgs, list) and msgs and isinstance(msgs[0], list): - for obj in msgs: - if obj and isinstance(obj[0], dict): - text = ( - tokenizer.apply_chat_template( - obj, - tokenize=False, - add_generation_prompt=False, - ) - if hasattr(tokenizer, "apply_chat_template") - else build_text_from_messages(obj) - ) - texts.append(text) - - if not texts: - # Return empty dict for batch - print("[preprocess] empty texts batch") - return {"input_ids": [], "attention_mask": []} - tokenized = tokenizer( - texts, truncation=True, max_length=cfg.finetune_train_seqlen, padding=False - ) - try: - n_in = len(texts) - n_out = len(tokenized.get("input_ids", [])) - print(f"[preprocess] batch texts={n_in} -> tokenized input_ids={n_out}") - except Exception: - pass - # Return as dict of lists for batched mapping - return tokenized - - ds = make_hf_dataset_from_files( - train_files, eval_files, streaming=not args.no_stream - ) - - # For streaming datasets, map with batched=False - train_ds = ds["train"] - eval_ds = ds["validation"] - # Debug raw dataset structure - try: - print(f"[debug] raw train columns: {train_ds.column_names}") - sample0 = train_ds[0] - print(f"[debug] raw train sample0 keys: {list(sample0.keys())}") - if "messages" in sample0: - print( - f"[debug] raw train sample0 messages type: {type(sample0['messages'])}" - ) - if isinstance(sample0["messages"], list) and sample0["messages"]: - print( - f"[debug] raw train sample0 messages[0] type: {type(sample0['messages'][0])}" - ) - except Exception as e: - print(f"[debug] error inspecting raw dataset: {e}") - - if args.max_train_samples: - train_ds = train_ds.take(args.max_train_samples) - if args.max_eval_samples: - eval_ds = eval_ds.take(args.max_eval_samples) - - # Load base model - # DType selection: prefer bfloat16 on CUDA, else float32. (MPS/directml kept at float32 for stability.) - use_cuda = chosen_device == "cuda" and torch.cuda.is_available() - dtype = torch.bfloat16 if use_cuda and hasattr(torch, "bfloat16") else torch.float32 - # Use explicit device for single GPU to avoid meta device issues with device_map="auto" - device_map_param = ( - "cuda:0" if use_cuda and torch.cuda.device_count() == 1 else "auto" - ) - base_model = AutoModelForCausalLM.from_pretrained( - hf_model_id, - torch_dtype=dtype, - device_map=device_map_param, - ) - - if cfg.gradient_checkpointing: - base_model.gradient_checkpointing_enable() - if hasattr(base_model.config, "use_cache"): - base_model.config.use_cache = False - - # Use target_modules from config if present, else default to Phi-3.5 list - default_target_modules = ["q_proj", "v_proj", "k_proj", "o_proj", "fc1", "fc2"] - config_target_modules = getattr(cfg, "target_modules", None) - lora_config = LoraConfig( - r=8, - lora_alpha=16, - target_modules=( - config_target_modules if config_target_modules else default_target_modules - ), - lora_dropout=cfg.lora_dropout, - bias="none", - task_type="CAUSAL_LM", - ) - model = get_peft_model(base_model, lora_config) - - class FilteringDataCollator(DataCollatorForLanguageModeling): - def torch_call(self, features): # type: ignore[override] - # Keep only model-relevant keys to avoid nested fields like 'messages' - filtered = [] - for f in features: - if isinstance(f, dict): - filtered.append( - { - k: v - for k, v in f.items() - if k in ("input_ids", "attention_mask", "labels") - } - ) - else: - filtered.append(f) - return super().torch_call(filtered) - - data_collator = FilteringDataCollator(tokenizer=tokenizer, mlm=False) - - out_dir = Path(cfg.save_dir).expanduser() - out_dir.mkdir(parents=True, exist_ok=True) - - # Determine if dataset is streaming (IterableDataset) which lacks __len__ - def is_iterable_dataset(ds) -> bool: - # Robustly check for streaming dataset - try: - from datasets import IterableDataset - - if isinstance(ds, IterableDataset): - return True - except ImportError: - pass # datasets not installed, fallback to attribute check - # Fallback: has __iter__ but not __len__ (common for streaming datasets) - return hasattr(ds, "__iter__") and not hasattr(ds, "__len__") - - streaming_train = is_iterable_dataset(train_ds) - # If streaming, Trainer requires max_steps for LR scheduler. Heuristic: derive from max-train-samples if provided. - max_steps_override = None - if streaming_train: - # Steps per epoch based on desired sample count and batch size - # Use max_train_samples if provided, else finetune_train_nsamples from config, else fallback to 1000 - target_samples = ( - args.max_train_samples - or getattr(cfg, "finetune_train_nsamples", None) - or 1000 - ) - steps_per_epoch = max( - 1, math.ceil(target_samples / max(1, cfg.finetune_train_batch_size)) - ) - max_steps_override = max(1, steps_per_epoch * max(1, cfg.epochs)) - - # Precision flags: enable bf16 if supported, otherwise leave fp16 False on CPU to avoid errors - bf16_flag = use_cuda and getattr(torch.cuda, "is_bf16_supported", lambda: False)() - fp16_flag = use_cuda and not bf16_flag - training_args = TrainingArguments( - output_dir=str(out_dir), - per_device_train_batch_size=cfg.finetune_train_batch_size, - per_device_eval_batch_size=cfg.finetune_test_batch_size, - eval_strategy="steps", - eval_steps=cfg.eval_steps, - save_steps=cfg.save_steps, - num_train_epochs=cfg.epochs, - max_steps=(max_steps_override if max_steps_override is not None else -1), - learning_rate=cfg.learning_rate, - logging_steps=max(1, cfg.eval_steps // 2), - bf16=bf16_flag, - fp16=fp16_flag, - gradient_checkpointing=cfg.gradient_checkpointing, - gradient_accumulation_steps=cfg.gradient_accumulation_steps, - warmup_steps=cfg.warmup_steps, - max_grad_norm=cfg.max_grad_norm, - lr_scheduler_type="cosine", - weight_decay=0.01, - remove_unused_columns=False, - save_total_limit=3, - load_best_model_at_end=True, - metric_for_best_model="eval_loss", - greater_is_better=False, - seed=cfg.seed, - deepspeed=args.deepspeed if args.deepspeed else None, - ddp_find_unused_parameters=False, - ) - - # Metrics logger - from metrics_logger import MetricsLogger # local import - - logger = MetricsLogger(out_dir) - - # Callback to log evaluation perplexity records - class PerplexityLoggingCallback(TrainerCallback if TrainerCallback else object): - # type: ignore[no-redef] - def __init__(self, logger_obj): - self._logger = logger_obj - - def on_evaluate(self, args, state, control, metrics=None, **kwargs): - if not metrics: - return - try: - if "eval_loss" in metrics and metrics["eval_loss"] is not None: - ppl = math.exp(float(metrics["eval_loss"])) - rec = { - "step": int(getattr(state, "global_step", 0)), - "eval_loss": float(metrics["eval_loss"]), - "eval_perplexity": float(ppl), - } - try: - self._logger.log(rec) - except Exception: - # Don't allow logging issues to break training - pass - except Exception: - # Swallow unexpected callback errors - pass - - is_streaming = is_iterable_dataset(train_ds) - # Remove 'messages' column so only tokenized output is kept - # Note: IterableDataset.map() has limited parameter support compared to Dataset.map() - is_streaming = ( - hasattr(train_ds, "__class__") and "Iterable" in train_ds.__class__.__name__ - ) - - map_kwargs_train = { - "batched": True, - "input_columns": ["messages"], - } - map_kwargs_eval = { - "batched": True, - "input_columns": ["messages"], - } - - # Only add these parameters for non-streaming datasets - if not is_streaming: - map_kwargs_train["load_from_cache_file"] = False - map_kwargs_train["desc"] = "Tokenizing train" - map_kwargs_eval["load_from_cache_file"] = False - map_kwargs_eval["desc"] = "Tokenizing eval" - - train_dataset = ( - train_ds.map(preprocess, **map_kwargs_train) - if hasattr(train_ds, "map") - else train_ds - ) - eval_dataset = ( - eval_ds.map(preprocess, **map_kwargs_eval) - if hasattr(eval_ds, "map") - else eval_ds - ) - # Remove all non-model columns to avoid DataCollator confusion - # Note: IterableDataset doesn't support column_names or remove_columns - keep_cols = {"input_ids", "attention_mask"} - if ( - hasattr(train_dataset, "column_names") - and train_dataset.column_names is not None - ): - drop_train = [c for c in train_dataset.column_names if c not in keep_cols] - if drop_train: - train_dataset = train_dataset.remove_columns(drop_train) - if hasattr(eval_dataset, "column_names") and eval_dataset.column_names is not None: - drop_eval = [c for c in eval_dataset.column_names if c not in keep_cols] - if drop_eval: - eval_dataset = eval_dataset.remove_columns(drop_eval) - # Debug dataset sizes and sample - try: - print(f"[debug] train_dataset len: {len(train_dataset)}") - print(f"[debug] eval_dataset len: {len(eval_dataset)}") - # Show first sample keys if available - if len(train_dataset) > 0: - first = train_dataset[0] - print(f"[debug] first train sample keys: {list(first.keys())}") - for k in ("input_ids", "attention_mask"): - if k in first: - print(f"[debug] first train sample {k} len: {len(first[k])}") - except Exception as e: - print(f"[debug] dataset inspection error: {e}") - - # Initialize early stopping callback to prevent overfitting - callbacks_list = [] - if TrainerCallback is not None: - try: - # Early stopping: configured via YAML/config values - early_stopping = EarlyStoppingCallback( - early_stopping_patience=cfg.early_stopping_patience, - early_stopping_threshold=cfg.early_stopping_threshold, - ) - callbacks_list.append(early_stopping) - print( - f"[training] Early stopping enabled (patience={cfg.early_stopping_patience}, threshold={cfg.early_stopping_threshold})" - ) - except Exception as e: - print(f"[debug] Failed to configure EarlyStoppingCallback: {e}") - - trainer = Trainer( - model=model, - args=training_args, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - tokenizer=tokenizer, - data_collator=data_collator, - callbacks=callbacks_list, - ) - if TrainerCallback is not None: - # Add perplexity logging callback - trainer.add_callback(PerplexityLoggingCallback(logger)) - # Add OpenTelemetry tracing callback if available and compatible - try: - from otel_callback import \ - OpenTelemetryTrainerCallback # type: ignore - - if hasattr(OpenTelemetryTrainerCallback, "on_prediction_step"): - trainer.add_callback(OpenTelemetryTrainerCallback()) - else: - print( - "[debug] Skipping OpenTelemetryTrainerCallback: missing on_prediction_step" - ) - except Exception as e: - print(f"[debug] Skipping OpenTelemetryTrainerCallback: {e}") - - # Pre-training evaluation (perplexity) - pre_metrics = trainer.evaluate() - if "eval_loss" in pre_metrics and pre_metrics["eval_loss"] is not None: - try: - ppl = math.exp(float(pre_metrics["eval_loss"])) - rec = { - "phase": "pre", - "eval_loss": float(pre_metrics["eval_loss"]), - "eval_perplexity": float(ppl), - } - print(rec) - logger.log(rec) - except Exception: - pass - - trainer.train() - # Post-training evaluation (perplexity) - post_metrics = trainer.evaluate() - if "eval_loss" in post_metrics and post_metrics["eval_loss"] is not None: - try: - ppl = math.exp(float(post_metrics["eval_loss"])) - rec = { - "phase": "post", - "eval_loss": float(post_metrics["eval_loss"]), - "eval_perplexity": float(ppl), - } - print(rec) - logger.log(rec) - except Exception: - pass - # Save adapters only - trainer.model.save_pretrained(str(out_dir / "lora_adapter")) - tokenizer.save_pretrained(str(out_dir / "tokenizer")) - - print(f"Training complete. Artifacts saved to: {out_dir}") - - -if __name__ == "__main__": - main() +import argparse +import json +import math +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, List + +try: + import yaml # type: ignore +except Exception: + raise SystemExit("pyyaml is required. Install with: pip install pyyaml") from None +try: + # Optional: install torch if missing + try: + import torch # pip install torch # type: ignore[reportMissingImports] + except ImportError: + raise SystemExit( + "PyTorch is required. Install with: pip install torch" + ) from None + from datasets import load_dataset # type: ignore[import] + + try: + from peft import LoraConfig, PeftModel, get_peft_model + from transformers import (AutoModelForCausalLM, AutoTokenizer, + DataCollatorForLanguageModeling, + EarlyStoppingCallback, Trainer, + TrainerCallback, TrainingArguments) + except ImportError: + raise SystemExit( + "Transformers is required. Install with: pip install transformers" + ) from None +except Exception as e: + # Provide visibility into which dependency import failed + import traceback + + print(f"[import-debug] training dependency import failed: {e}") + traceback.print_exc() + # We'll allow dry-run without these by nulling all symbols + torch = None # type: ignore + load_dataset = None # type: ignore + AutoModelForCausalLM = AutoTokenizer = Trainer = TrainingArguments = DataCollatorForLanguageModeling = None # type: ignore + LoraConfig = get_peft_model = PeftModel = None # type: ignore + TrainerCallback = None # type: ignore + EarlyStoppingCallback = None # type: ignore + + +@dataclass +class Config: + model: str + finetune_dataset: str + save_dir: str + finetune_train_nsamples: int | None + finetune_test_nsamples: int | None + finetune_train_batch_size: int + finetune_test_batch_size: int + finetune_train_seqlen: int + finetune_test_seqlen: int + learning_rate: float + lora_dropout: float + epochs: int + eval_steps: int + save_steps: int + gradient_checkpointing: bool + seed: int + warmup_steps: int + gradient_accumulation_steps: int + max_grad_norm: float + early_stopping_patience: int + early_stopping_threshold: float + + +def read_yaml(yaml_path: Path) -> Dict[str, Any]: + with yaml_path.open("r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def resolve_path(p: str) -> Path: + # allow tokens like mount//dataset to be overridden by --dataset + return Path(p).expanduser() + + +def iter_jsonl(path: Path) -> Iterable[Dict[str, Any]]: + with path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + yield json.loads(line) + + +def count_records(path: Path) -> int: + n = 0 + for _ in iter_jsonl(path): + n += 1 + return n + + +def validate_sample(path: Path) -> Dict[str, Any]: + for obj in iter_jsonl(path): + msgs = obj.get("messages") + if isinstance(msgs, list) and len(msgs) >= 2: + return obj + raise RuntimeError(f"No valid chat records found in {path}") + + +def build_text_from_messages(messages: List[Dict[str, str]]) -> str: + """Convert messages to training text with improved formatting and end tokens.""" + parts: List[str] = [] + for m in messages: + role = m.get("role", "").lower() + content = m.get("content", "").strip() + if not content: # Skip empty messages + continue + if role == "system": + parts.append(f"<|system|>\n{content}<|end|>\n") + elif role == "user": + parts.append(f"<|user|>\n{content}<|end|>\n") + elif role == "assistant": + parts.append(f"<|assistant|>\n{content}<|end|>\n") + return "".join(parts) + + +def make_hf_dataset_from_files( + train_files: List[str], eval_files: List[str], streaming: bool = True +): + if load_dataset is None: + raise RuntimeError( + "HuggingFace datasets not available. Install 'datasets' to load datasets." + ) + data_files = {"train": train_files, "validation": eval_files} + ds = load_dataset("json", data_files=data_files, streaming=streaming) + return ds + + +def _read_text_source(path_or_url: str) -> Iterable[str]: + if path_or_url.startswith("http://") or path_or_url.startswith("https://"): + import urllib.request + + with urllib.request.urlopen(path_or_url) as resp: # nosec B310 + for line in resp.read().decode("utf-8").splitlines(): + yield line + else: + p = Path(path_or_url) + with p.open("r", encoding="utf-8") as f: + for line in f: + yield line.rstrip("\n") + + +def parse_manifest(path_or_url: str) -> List[str]: + urls: List[str] = [] + lower = path_or_url.lower() + if lower.endswith(".json"): + import json as _json + + if path_or_url.startswith("http://") or path_or_url.startswith("https://"): + import urllib.request + + with urllib.request.urlopen(path_or_url) as resp: # nosec B310 + obj = _json.loads(resp.read().decode("utf-8")) + else: + with Path(path_or_url).open("r", encoding="utf-8") as f: + obj = _json.load(f) + if isinstance(obj, dict): + for key in ("train", "validation", "urls", "files"): + v = obj.get(key) + if isinstance(v, list): + urls.extend([str(x) for x in v]) + if not urls and "url" in obj: + urls.append(str(obj["url"])) + elif isinstance(obj, list): + urls.extend([str(x) for x in obj]) + elif lower.endswith(".jsonl"): + import json as _json + + for line in _read_text_source(path_or_url): + if not line.strip(): + continue + try: + rec = _json.loads(line) + if isinstance(rec, str): + urls.append(rec) + elif isinstance(rec, dict) and "url" in rec: + urls.append(str(rec["url"])) + except Exception: + urls.append(line.strip()) + else: + for line in _read_text_source(path_or_url): + if line.strip(): + urls.append(line.strip()) + # Dedupe + seen = set() + uniq: List[str] = [] + for u in urls: + if u not in seen: + seen.add(u) + uniq.append(u) + return uniq + + +def main(): + ap = argparse.ArgumentParser( + description="Train LoRA on chat dataset using lora.yaml config" + ) + ap.add_argument( + "--config", + default=str(Path(__file__).resolve().parents[1] / "lora" / "lora.yaml"), + ) + ap.add_argument( + "--dataset", default=str(Path(__file__).resolve().parents[1] / "data") + ) + ap.add_argument( + "--dry-run", + action="store_true", + help="Validate dataset/config only; no model download", + ) + ap.add_argument( + "--max-train-samples", + type=int, + default=None, + help="Limit train examples (for smoke test)", + ) + ap.add_argument( + "--max-eval-samples", + type=int, + default=None, + help="Limit eval examples (for smoke test)", + ) + ap.add_argument( + "--hf-model-id", + default=None, + help="Override HF model id (e.g., microsoft/Phi-3.5-mini-instruct)", + ) + ap.add_argument( + "--no-stream", action="store_true", help="Disable streaming mode for datasets" + ) + ap.add_argument( + "--deepspeed", + default=None, + help="Path to DeepSpeed config JSON to enable ZeRO (multi-GPU)", + ) + ap.add_argument( + "--train-manifest", + default=None, + help="Path or URL to manifest of training files (txt/json/jsonl)", + ) + ap.add_argument( + "--eval-manifest", + default=None, + help="Path or URL to manifest of eval files (txt/json/jsonl)", + ) + ap.add_argument( + "--save-dir", + default=None, + help="Override output directory (else from config or defaults)", + ) + ap.add_argument( + "--device", + default="auto", + choices=["auto", "cuda", "cpu", "directml", "mps"], + help="Device preference: auto selects best available (cuda>mps>directml>cpu)", + ) + # Optional overrides for HPO/cloud runs + ap.add_argument( + "--learning-rate", + type=float, + default=None, + help="Override learning_rate from config", + ) + ap.add_argument( + "--lora-dropout", + type=float, + default=None, + help="Override lora_dropout from config", + ) + ap.add_argument( + "--epochs", type=int, default=None, help="Override epochs from config" + ) + ap.add_argument( + "--train-batch-size", + type=int, + default=None, + help="Override finetune_train_batch_size from config", + ) + ap.add_argument( + "--eval-batch-size", + type=int, + default=None, + help="Override finetune_test_batch_size from config", + ) + ap.add_argument("--seed", type=int, default=None, help="Override seed from config") + args = ap.parse_args() + + # Initialize tracing (best-effort). This allows the optional + # OpenTelemetryTrainerCallback to get an active tracer if available. + # Optional tracing import (ignore if missing) + try: + from shared.tracing import init_tracing # type: ignore + + init_tracing(service_name="train_lora") + except Exception as _e: + print(f"[tracing] init skipped in train_lora: {_e}") + + cfg_raw = read_yaml(Path(args.config)) + cfg = Config( + model=cfg_raw.get("model") or "Phi-3.6-mini-instruct", + finetune_dataset=cfg_raw.get("finetune_dataset") or str(Path(args.dataset)), + save_dir=( + args.save_dir + or cfg_raw.get("save_dir") + or str(Path(__file__).resolve().parents[1] / "outputs") + ), + finetune_train_nsamples=cfg_raw.get("finetune_train_nsamples"), + finetune_test_nsamples=cfg_raw.get("finetune_test_nsamples"), + finetune_train_batch_size=int(cfg_raw.get("finetune_train_batch_size") or 2), + finetune_test_batch_size=int(cfg_raw.get("finetune_test_batch_size") or 2), + finetune_train_seqlen=int(cfg_raw.get("finetune_train_seqlen") or 1024), + finetune_test_seqlen=int(cfg_raw.get("finetune_test_seqlen") or 2048), + learning_rate=float(cfg_raw.get("learning_rate") or 2e-4), + lora_dropout=float(cfg_raw.get("lora_dropout") or 0.1), + epochs=int(cfg_raw.get("epochs") or 1), + eval_steps=int(cfg_raw.get("eval_steps", 64)), + save_steps=int(cfg_raw.get("save_steps") or 64), + gradient_checkpointing=bool(cfg_raw.get("gradient_checkpointing") or False), + seed=int(cfg_raw.get("seed") or 42), + warmup_steps=int(cfg_raw.get("warmup_steps") or 100), + gradient_accumulation_steps=int( + cfg_raw.get("gradient_accumulation_steps") or 4 + ), + max_grad_norm=float(cfg_raw.get("max_grad_norm") or 1.0), + early_stopping_patience=int(cfg_raw.get("early_stopping_patience") or 3), + early_stopping_threshold=float(cfg_raw.get("early_stopping_threshold") or 0.01), + ) + + # Apply CLI overrides for HPO or cloud jobs + if getattr(args, "learning_rate", None) is not None: + cfg.learning_rate = float(args.learning_rate) + if getattr(args, "lora_dropout", None) is not None: + cfg.lora_dropout = float(args.lora_dropout) + if getattr(args, "epochs", None) is not None: + cfg.epochs = int(args.epochs) + if getattr(args, "train_batch_size", None) is not None: + cfg.finetune_train_batch_size = int(args.train_batch_size) + if getattr(args, "eval_batch_size", None) is not None: + cfg.finetune_test_batch_size = int(args.eval_batch_size) + if getattr(args, "seed", None) is not None: + cfg.seed = int(args.seed) + + # Resolve data sources: manifests or local files + train_files: List[str] = [] + eval_files: List[str] = [] + # CLI overrides take precedence + train_manifest = getattr(args, "train_manifest", None) + eval_manifest = getattr(args, "eval_manifest", None) + if train_manifest or eval_manifest: + if train_manifest: + train_files = parse_manifest(train_manifest) + if eval_manifest: + eval_files = parse_manifest(eval_manifest) + if not train_files: + raise RuntimeError("No train files found from manifest") + if not eval_files: + # Fallback: use a small subset of train for eval + eval_files = train_files[:1] + else: + dataset_path = ( + Path(args.dataset) if args.dataset else resolve_path(cfg.finetune_dataset) + ) + if dataset_path.is_file(): + # Allow direct file usage (.json or .jsonl) + if dataset_path.suffix.lower() in (".json", ".jsonl"): + train_files = [str(dataset_path)] + eval_files = [str(dataset_path)] + else: + raise FileNotFoundError( + f"Unsupported dataset file type: {dataset_path}" + ) + else: + # Directory: accept train.json/test.json or train.jsonl/test.jsonl; fallback to single train file present + candidates = [ + (dataset_path / "train.json", dataset_path / "test.json"), + (dataset_path / "train.jsonl", dataset_path / "test.jsonl"), + ] + found = False + for t_path, v_path in candidates: + if t_path.exists() and v_path.exists(): + train_files = [str(t_path)] + eval_files = [str(v_path)] + found = True + break + if not found: + # Fallbacks: if any train.* exists, use it for both train/val + t_candidates = [ + dataset_path / "train.json", + dataset_path / "train.jsonl", + ] + t_use = next((p for p in t_candidates if p.exists()), None) + if t_use is None: + raise FileNotFoundError( + f"Expected dataset files at: {dataset_path}/train.json[.l] and optionally test.json[.l]" + ) + train_files = [str(t_use)] + eval_candidates = [ + dataset_path / "test.json", + dataset_path / "test.jsonl", + ] + v_use = next((p for p in eval_candidates if p.exists()), None) + eval_files = [str(v_use)] if v_use else [str(t_use)] + + # Determine and report device early (even for dry-run) + chosen_device = "cpu" + if args.device == "auto": + if ( + torch is not None + and getattr(torch, "cuda", None) + and getattr(torch.cuda, "is_available", lambda: False)() + ): + chosen_device = "cuda" + elif ( + torch is not None + and getattr(torch, "backends", None) + and getattr(torch.backends, "mps", None) + and getattr(torch.backends.mps, "is_available", lambda: False)() + ): + chosen_device = "mps" + else: + # Optional DirectML detection + try: + chosen_device = "directml" + except Exception: + chosen_device = "cpu" + else: + chosen_device = args.device + print( + f"[device] selection={args.device} resolved={chosen_device} cuda_available={(getattr(torch, 'cuda', None) and getattr(torch.cuda, 'is_available', lambda: False)() if torch else False)}" + ) + + # Dry run: count/validate records only (no model/tokenizer downloads) + if args.dry_run: + if train_manifest or eval_manifest: + print("Dry run OK.") + print( + { + "device": chosen_device, + "train_files": train_files[:5], + "eval_files": eval_files[:5], + "note": "Counting skipped for remote manifests", + } + ) + return + n_train = count_records(Path(train_files[0])) + n_test = count_records(Path(eval_files[0])) + sample = validate_sample(Path(train_files[0])) + print("Dry run OK.") + print( + { + "device": chosen_device, + "train_examples": n_train, + "test_examples": n_test, + "sample": sample, + } + ) + return + + # Real training requires heavy deps + if ( + AutoTokenizer is None + or AutoModelForCausalLM is None + or load_dataset is None + or torch is None + ): + missing = [] + if torch is None: + missing.append("torch") + if load_dataset is None: + missing.append("datasets") + if AutoTokenizer is None or AutoModelForCausalLM is None: + missing.append("transformers") + if LoraConfig is None: + missing.append("peft") + model_venv = Path(__file__).resolve().parents[1] / "venv" + root_venv = Path(__file__).resolve().parents[3] / "venv" + raise RuntimeError( + "Training dependencies not installed or import failed.\n" + f"missing={missing}\n" + f"To fix (model env): {model_venv / 'Scripts' / 'pip.exe'} install transformers datasets peft accelerate torch\n" + f"Alt (root env): {root_venv / 'Scripts' / 'pip.exe'} install transformers datasets peft accelerate torch\n" + "After installing, re-run this script." + ) + + # Resolve model id + hf_model_id = args.hf_model_id or os.environ.get("HF_MODEL_ID") + if hf_model_id is None: + # Best-effort mapping for local runs + # If the configured model isn't an HF id, default to a widely-available one + hf_model_id = { + "Phi-3.6-mini-instruct": "microsoft/Phi-3.5-mini-instruct", + }.get(cfg.model, cfg.model) + + tokenizer = AutoTokenizer.from_pretrained(hf_model_id, use_fast=True) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + def preprocess(examples): + # Handle both single example and batched mapping + texts = [] + if isinstance(examples, dict) and "messages" in examples: + msgs = examples["messages"] + try: + print(f"[preprocess] messages type: {type(msgs)}") + if isinstance(msgs, dict): + print(f"[preprocess] messages dict keys: {list(msgs.keys())}") + # Attempt to reconstruct per-sample conversations if messages is a dict of lists + # Expect shape: msgs[key][i] gives ith sample's list of that field + keys = list(msgs.keys()) + batch_size = ( + len(msgs[keys[0]]) + if keys and isinstance(msgs[keys[0]], list) + else 0 + ) + print(f"[preprocess] inferred batch_size from dict: {batch_size}") + for i in range(batch_size): + # Reconstruct sample i as list of dicts using available fields + sample_messages = [] + # Try common fields 'role' and 'content' + roles = msgs.get("role", [])[i] if "role" in msgs else None + contents = ( + msgs.get("content", [])[i] if "content" in msgs else None + ) + if ( + isinstance(roles, list) + and isinstance(contents, list) + and len(roles) == len(contents) + ): + for r, c in zip(roles, contents): + sample_messages.append({"role": r, "content": c}) + else: + # Fallback: try to rebuild from a generic list of dicts if present + # msgs may have a single key representing the full objects + for k in keys: + candidate = ( + msgs[k][i] if isinstance(msgs[k], list) else None + ) + if ( + isinstance(candidate, list) + and candidate + and isinstance(candidate[0], dict) + ): + sample_messages = candidate + break + if sample_messages: + text = ( + tokenizer.apply_chat_template( + sample_messages, + tokenize=False, + add_generation_prompt=False, + ) + if hasattr(tokenizer, "apply_chat_template") + else build_text_from_messages(sample_messages) + ) + texts.append(text) + # Batched: list of lists + elif isinstance(msgs, list) and msgs and isinstance(msgs[0], list): + for obj in msgs: + if obj and isinstance(obj[0], dict): + text = ( + tokenizer.apply_chat_template( + obj, + tokenize=False, + add_generation_prompt=False, + ) + if hasattr(tokenizer, "apply_chat_template") + else build_text_from_messages(obj) + ) + texts.append(text) + # Single example: list of dicts + elif isinstance(msgs, list) and msgs and isinstance(msgs[0], dict): + text = ( + tokenizer.apply_chat_template( + msgs, + tokenize=False, + add_generation_prompt=False, + ) + if hasattr(tokenizer, "apply_chat_template") + else build_text_from_messages(msgs) + ) + texts.append(text) + except Exception as e: + print(f"[preprocess] error reconstructing messages: {e}") + # When using input_columns=["messages"], the function may receive just the messages column values + elif isinstance(examples, list): + msgs = examples + if isinstance(msgs, list) and msgs and isinstance(msgs[0], list): + for obj in msgs: + if obj and isinstance(obj[0], dict): + text = ( + tokenizer.apply_chat_template( + obj, + tokenize=False, + add_generation_prompt=False, + ) + if hasattr(tokenizer, "apply_chat_template") + else build_text_from_messages(obj) + ) + texts.append(text) + + if not texts: + # Return empty dict for batch + print("[preprocess] empty texts batch") + return {"input_ids": [], "attention_mask": []} + tokenized = tokenizer( + texts, truncation=True, max_length=cfg.finetune_train_seqlen, padding=False + ) + try: + n_in = len(texts) + n_out = len(tokenized.get("input_ids", [])) + print(f"[preprocess] batch texts={n_in} -> tokenized input_ids={n_out}") + except Exception: + pass + # Return as dict of lists for batched mapping + return tokenized + + ds = make_hf_dataset_from_files( + train_files, eval_files, streaming=not args.no_stream + ) + + # For streaming datasets, map with batched=False + train_ds = ds["train"] + eval_ds = ds["validation"] + # Debug raw dataset structure + try: + print(f"[debug] raw train columns: {train_ds.column_names}") + sample0 = train_ds[0] + print(f"[debug] raw train sample0 keys: {list(sample0.keys())}") + if "messages" in sample0: + print( + f"[debug] raw train sample0 messages type: {type(sample0['messages'])}" + ) + if isinstance(sample0["messages"], list) and sample0["messages"]: + print( + f"[debug] raw train sample0 messages[0] type: {type(sample0['messages'][0])}" + ) + except Exception as e: + print(f"[debug] error inspecting raw dataset: {e}") + + if args.max_train_samples: + train_ds = train_ds.take(args.max_train_samples) + if args.max_eval_samples: + eval_ds = eval_ds.take(args.max_eval_samples) + + # Load base model + # DType selection: prefer bfloat16 on CUDA, else float32. (MPS/directml kept at float32 for stability.) + use_cuda = chosen_device == "cuda" and torch.cuda.is_available() + dtype = torch.bfloat16 if use_cuda and hasattr(torch, "bfloat16") else torch.float32 + # Use explicit device for single GPU to avoid meta device issues with device_map="auto" + device_map_param = ( + "cuda:0" if use_cuda and torch.cuda.device_count() == 1 else "auto" + ) + base_model = AutoModelForCausalLM.from_pretrained( + hf_model_id, + torch_dtype=dtype, + device_map=device_map_param, + ) + + if cfg.gradient_checkpointing: + base_model.gradient_checkpointing_enable() + if hasattr(base_model.config, "use_cache"): + base_model.config.use_cache = False + + # Use target_modules from config if present, else default to Phi-3.5 list + default_target_modules = ["q_proj", "v_proj", "k_proj", "o_proj", "fc1", "fc2"] + config_target_modules = getattr(cfg, "target_modules", None) + lora_config = LoraConfig( + r=8, + lora_alpha=16, + target_modules=( + config_target_modules if config_target_modules else default_target_modules + ), + lora_dropout=cfg.lora_dropout, + bias="none", + task_type="CAUSAL_LM", + ) + model = get_peft_model(base_model, lora_config) + + class FilteringDataCollator(DataCollatorForLanguageModeling): + def torch_call(self, features): # type: ignore[override] + # Keep only model-relevant keys to avoid nested fields like 'messages' + filtered = [] + for f in features: + if isinstance(f, dict): + filtered.append( + { + k: v + for k, v in f.items() + if k in ("input_ids", "attention_mask", "labels") + } + ) + else: + filtered.append(f) + return super().torch_call(filtered) + + data_collator = FilteringDataCollator(tokenizer=tokenizer, mlm=False) + + out_dir = Path(cfg.save_dir).expanduser() + out_dir.mkdir(parents=True, exist_ok=True) + + # Determine if dataset is streaming (IterableDataset) which lacks __len__ + def is_iterable_dataset(ds) -> bool: + # Robustly check for streaming dataset + try: + from datasets import IterableDataset + + if isinstance(ds, IterableDataset): + return True + except ImportError: + pass # datasets not installed, fallback to attribute check + # Fallback: has __iter__ but not __len__ (common for streaming datasets) + return hasattr(ds, "__iter__") and not hasattr(ds, "__len__") + + streaming_train = is_iterable_dataset(train_ds) + # If streaming, Trainer requires max_steps for LR scheduler. Heuristic: derive from max-train-samples if provided. + max_steps_override = None + if streaming_train: + # Steps per epoch based on desired sample count and batch size + # Use max_train_samples if provided, else finetune_train_nsamples from config, else fallback to 1000 + target_samples = ( + args.max_train_samples + or getattr(cfg, "finetune_train_nsamples", None) + or 1000 + ) + steps_per_epoch = max( + 1, math.ceil(target_samples / max(1, cfg.finetune_train_batch_size)) + ) + max_steps_override = max(1, steps_per_epoch * max(1, cfg.epochs)) + + # Precision flags: enable bf16 if supported, otherwise leave fp16 False on CPU to avoid errors + bf16_flag = use_cuda and getattr(torch.cuda, "is_bf16_supported", lambda: False)() + fp16_flag = use_cuda and not bf16_flag + training_args = TrainingArguments( + output_dir=str(out_dir), + per_device_train_batch_size=cfg.finetune_train_batch_size, + per_device_eval_batch_size=cfg.finetune_test_batch_size, + eval_strategy="steps", + eval_steps=cfg.eval_steps, + save_steps=cfg.save_steps, + num_train_epochs=cfg.epochs, + max_steps=(max_steps_override if max_steps_override is not None else -1), + learning_rate=cfg.learning_rate, + logging_steps=max(1, cfg.eval_steps // 2), + bf16=bf16_flag, + fp16=fp16_flag, + gradient_checkpointing=cfg.gradient_checkpointing, + gradient_accumulation_steps=cfg.gradient_accumulation_steps, + warmup_steps=cfg.warmup_steps, + max_grad_norm=cfg.max_grad_norm, + lr_scheduler_type="cosine", + weight_decay=0.01, + remove_unused_columns=False, + save_total_limit=3, + load_best_model_at_end=True, + metric_for_best_model="eval_loss", + greater_is_better=False, + seed=cfg.seed, + deepspeed=args.deepspeed if args.deepspeed else None, + ddp_find_unused_parameters=False, + ) + + # Metrics logger + from metrics_logger import MetricsLogger # local import + + logger = MetricsLogger(out_dir) + + # Callback to log evaluation perplexity records + class PerplexityLoggingCallback(TrainerCallback if TrainerCallback else object): + # type: ignore[no-redef] + def __init__(self, logger_obj): + self._logger = logger_obj + + def on_evaluate(self, args, state, control, metrics=None, **kwargs): + if not metrics: + return + try: + if "eval_loss" in metrics and metrics["eval_loss"] is not None: + ppl = math.exp(float(metrics["eval_loss"])) + rec = { + "step": int(getattr(state, "global_step", 0)), + "eval_loss": float(metrics["eval_loss"]), + "eval_perplexity": float(ppl), + } + try: + self._logger.log(rec) + except Exception: + # Don't allow logging issues to break training + pass + except Exception: + # Swallow unexpected callback errors + pass + + is_streaming = is_iterable_dataset(train_ds) + # Remove 'messages' column so only tokenized output is kept + # Note: IterableDataset.map() has limited parameter support compared to Dataset.map() + is_streaming = ( + hasattr(train_ds, "__class__") and "Iterable" in train_ds.__class__.__name__ + ) + + map_kwargs_train = { + "batched": True, + "input_columns": ["messages"], + } + map_kwargs_eval = { + "batched": True, + "input_columns": ["messages"], + } + + # Only add these parameters for non-streaming datasets + if not is_streaming: + map_kwargs_train["load_from_cache_file"] = False + map_kwargs_train["desc"] = "Tokenizing train" + map_kwargs_eval["load_from_cache_file"] = False + map_kwargs_eval["desc"] = "Tokenizing eval" + + train_dataset = ( + train_ds.map(preprocess, **map_kwargs_train) + if hasattr(train_ds, "map") + else train_ds + ) + eval_dataset = ( + eval_ds.map(preprocess, **map_kwargs_eval) + if hasattr(eval_ds, "map") + else eval_ds + ) + # Remove all non-model columns to avoid DataCollator confusion + # Note: IterableDataset doesn't support column_names or remove_columns + keep_cols = {"input_ids", "attention_mask"} + if ( + hasattr(train_dataset, "column_names") + and train_dataset.column_names is not None + ): + drop_train = [c for c in train_dataset.column_names if c not in keep_cols] + if drop_train: + train_dataset = train_dataset.remove_columns(drop_train) + if hasattr(eval_dataset, "column_names") and eval_dataset.column_names is not None: + drop_eval = [c for c in eval_dataset.column_names if c not in keep_cols] + if drop_eval: + eval_dataset = eval_dataset.remove_columns(drop_eval) + # Debug dataset sizes and sample + try: + print(f"[debug] train_dataset len: {len(train_dataset)}") + print(f"[debug] eval_dataset len: {len(eval_dataset)}") + # Show first sample keys if available + if len(train_dataset) > 0: + first = train_dataset[0] + print(f"[debug] first train sample keys: {list(first.keys())}") + for k in ("input_ids", "attention_mask"): + if k in first: + print(f"[debug] first train sample {k} len: {len(first[k])}") + except Exception as e: + print(f"[debug] dataset inspection error: {e}") + + # Initialize early stopping callback to prevent overfitting + callbacks_list = [] + if TrainerCallback is not None: + try: + # Early stopping: configured via YAML/config values + early_stopping = EarlyStoppingCallback( + early_stopping_patience=cfg.early_stopping_patience, + early_stopping_threshold=cfg.early_stopping_threshold, + ) + callbacks_list.append(early_stopping) + print( + f"[training] Early stopping enabled (patience={cfg.early_stopping_patience}, threshold={cfg.early_stopping_threshold})" + ) + except Exception as e: + print(f"[debug] Failed to configure EarlyStoppingCallback: {e}") + + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + tokenizer=tokenizer, + data_collator=data_collator, + callbacks=callbacks_list, + ) + if TrainerCallback is not None: + # Add perplexity logging callback + trainer.add_callback(PerplexityLoggingCallback(logger)) + # Add OpenTelemetry tracing callback if available and compatible + try: + from otel_callback import \ + OpenTelemetryTrainerCallback # type: ignore + + if hasattr(OpenTelemetryTrainerCallback, "on_prediction_step"): + trainer.add_callback(OpenTelemetryTrainerCallback()) + else: + print( + "[debug] Skipping OpenTelemetryTrainerCallback: missing on_prediction_step" + ) + except Exception as e: + print(f"[debug] Skipping OpenTelemetryTrainerCallback: {e}") + + # Pre-training evaluation (perplexity) + pre_metrics = trainer.evaluate() + if "eval_loss" in pre_metrics and pre_metrics["eval_loss"] is not None: + try: + ppl = math.exp(float(pre_metrics["eval_loss"])) + rec = { + "phase": "pre", + "eval_loss": float(pre_metrics["eval_loss"]), + "eval_perplexity": float(ppl), + } + print(rec) + logger.log(rec) + except Exception: + pass + + trainer.train() + # Post-training evaluation (perplexity) + post_metrics = trainer.evaluate() + if "eval_loss" in post_metrics and post_metrics["eval_loss"] is not None: + try: + ppl = math.exp(float(post_metrics["eval_loss"])) + rec = { + "phase": "post", + "eval_loss": float(post_metrics["eval_loss"]), + "eval_perplexity": float(ppl), + } + print(rec) + logger.log(rec) + except Exception: + pass + # Save adapters only + trainer.model.save_pretrained(str(out_dir / "lora_adapter")) + tokenizer.save_pretrained(str(out_dir / "tokenizer")) + + print(f"Training complete. Artifacts saved to: {out_dir}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_orchestrator_health_in_status_endpoint.py b/tests/test_orchestrator_health_in_status_endpoint.py index aba0cb3d0..512b479d5 100644 --- a/tests/test_orchestrator_health_in_status_endpoint.py +++ b/tests/test_orchestrator_health_in_status_endpoint.py @@ -16,7 +16,11 @@ def _load_function_app() -> ModuleType: raise RuntimeError("Failed to load function_app module") module = importlib.util.module_from_spec(spec) sys.modules["function_app"] = module - spec.loader.exec_module(module) + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop("function_app", None) + raise return module diff --git a/tests/test_orchestrator_health_integration.py b/tests/test_orchestrator_health_integration.py index 1bde720cd..3f52879b5 100644 --- a/tests/test_orchestrator_health_integration.py +++ b/tests/test_orchestrator_health_integration.py @@ -25,7 +25,11 @@ def app_module(): raise RuntimeError("Failed to load function_app module") module = importlib.util.module_from_spec(spec) sys.modules["function_app"] = module - spec.loader.exec_module(module) + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop("function_app", None) + raise return module