diff --git a/README.md b/README.md index dc02e6c5..c76760a2 100644 --- a/README.md +++ b/README.md @@ -893,8 +893,10 @@ config: max_attempts: 3 # Retry failed graders up to 3 times (default: 1, no retries) timeout_seconds: 300 parallel: false - executor: mock # or copilot-sdk + executor: mock # or copilot-sdk, openai-compatible model: claude-sonnet-4-20250514 + endpoint: http://127.0.0.1:1234 # Required for openai-compatible unless set in .waza.yaml + api_key: "" # Optional per-eval override; prefer OPENAI_API_KEY when auth is required group_by: model # Group results by model (or other dimension) instruction_files: - .github/instructions/project.instructions.md @@ -1250,6 +1252,7 @@ jobs: | **Go Version** | 1.26 or higher | | **Executor** | Use `mock` executor for CI (no API keys needed) | | **Copilot Auth** | Required for the default `copilot-sdk` route; set `GITHUB_TOKEN` in CI. Custom providers can be configured with `COPILOT_BASE_URL` or `COPILOT_PROVIDER_BASE_URL` instead. | +| **OpenAI-compatible endpoint** | Required for `openai-compatible` unless set as `.waza.yaml` `defaults.endpoint`; for local LM Studio use `http://127.0.0.1:1234`. | | **Exit Codes** | 0=success, 1=test failure, 2=config error | #### Expected Skill Structure @@ -1288,9 +1291,34 @@ Supported environment variables: | `COPILOT_WIRE_API` or `COPILOT_PROVIDER_WIRE_API` | Wire format passed through to the SDK, for example `responses` or `completions`, depending on provider. | | `COPILOT_API_KEY` or `COPILOT_PROVIDER_API_KEY` | API key for the custom provider, if required. | | `COPILOT_BEARER_TOKEN` or `COPILOT_PROVIDER_BEARER_TOKEN` | Bearer token for the custom provider, if required. | +| `OPENAI_API_KEY` | Bearer token sent by the `openai-compatible` executor when the endpoint requires authentication. | When a custom provider is active, the CLI usage summary labels the SDK request counter as `Provider Requests` instead of `Premium Requests`. Result JSON records `usage.provider: "custom"` and a sanitized `usage.provider_host`; it does not store the full provider URL. +#### OpenAI-Compatible Executor + +Use `executor: openai-compatible` to run directly against a local or hosted OpenAI-compatible Chat Completions API without the Copilot SDK. The `endpoint` may be a base URL, `/v1` URL, or full `/v1/chat/completions` URL. `model` is optional and defaults to `local-model`. + +```yaml +config: + trials_per_task: 1 + timeout_seconds: 300 + executor: openai-compatible + endpoint: http://127.0.0.1:1234 + model: local-model + api_key: "" # Optional per-eval override; prefer OPENAI_API_KEY when auth is required +``` + +You can also put the executor and endpoint in `.waza.yaml`: + +```yaml +defaults: + engine: openai-compatible + endpoint: http://127.0.0.1:1234 +``` + +When the endpoint requires bearer-token authentication, set `OPENAI_API_KEY`. `config.api_key` remains available as a per-eval override, but project-level config does not store API keys. + ### For Waza Repository This repository includes reusable workflows: diff --git a/cmd/waza/cmd_run.go b/cmd/waza/cmd_run.go index a8e45118..e08d997f 100644 --- a/cmd/waza/cmd_run.go +++ b/cmd/waza/cmd_run.go @@ -473,6 +473,9 @@ func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []str if err != nil { return nil, fmt.Errorf("failed to load spec: %w", err) } + if cfg, err := projectconfig.Load(filepath.Dir(specPath)); err == nil && cfg != nil { + applyProjectDefaultsToSpec(spec, cfg) + } // CLI flags override spec config if parallel { @@ -594,6 +597,21 @@ func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []str return allResults, nil } +func applyProjectDefaultsToSpec(spec *models.EvalSpec, cfg *projectconfig.ProjectConfig) { + if spec == nil || cfg == nil { + return + } + if spec.Config.EngineType == "" { + spec.Config.EngineType = cfg.Defaults.Engine + } + if spec.Config.ModelID == "" && (spec.Config.EngineType != "openai-compatible" || cfg.Defaults.Model != projectconfig.DefaultModel) { + spec.Config.ModelID = cfg.Defaults.Model + } + if spec.Config.Endpoint == "" { + spec.Config.Endpoint = cfg.Defaults.Endpoint + } +} + // runSingleModel executes a benchmark for one model and returns the outcome. // It prints the per-model summary and saves output for single-model runs. func runSingleModel(cmd *cobra.Command, spec *models.EvalSpec, specPath string, defaultSkills []string) (*models.EvaluationOutcome, error) { @@ -655,6 +673,7 @@ func runSingleModel(cmd *cobra.Command, spec *models.EvalSpec, specPath string, // Create engine based on spec var engine execution.AgentEngine + var err error switch spec.Config.EngineType { case "mock": @@ -663,6 +682,11 @@ func runSingleModel(cmd *cobra.Command, spec *models.EvalSpec, specPath string, engine = execution.NewCopilotEngineBuilder(spec.Config.ModelID, &execution.CopilotEngineBuilderOptions{ NewCopilotClient: newCopilotClientFn, // if nil, uses the real function, otherwise overridable for tests. }).Build() + case "openai-compatible": + engine, err = execution.NewOpenAICompatibleEngine(spec.Config.Endpoint, spec.Config.ModelID, spec.Config.APIKey) + if err != nil { + return nil, err + } default: return nil, fmt.Errorf("unknown engine type: %s", spec.Config.EngineType) } diff --git a/cmd/waza/cmd_run_test.go b/cmd/waza/cmd_run_test.go index 81a20047..ad355e04 100644 --- a/cmd/waza/cmd_run_test.go +++ b/cmd/waza/cmd_run_test.go @@ -9,6 +9,8 @@ import ( "io" "io/fs" "maps" + "net/http" + "net/http/httptest" "os" "path/filepath" "slices" @@ -716,6 +718,113 @@ tasks: assert.Contains(t, err.Error(), "unknown engine type") } +func TestRunCommand_OpenAICompatibleEngine(t *testing.T) { + resetRunGlobals() + + var hitPath string + var gotAuth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"openai compatible response"}}]}`)) + })) + defer server.Close() + + dir := t.TempDir() + taskDir := filepath.Join(dir, "tasks") + require.NoError(t, os.MkdirAll(taskDir, 0o755)) + task := `id: t1 +name: OpenAI Compatible Task +inputs: + prompt: "Say hello" +expected: + output_contains: + - "openai compatible response" +` + require.NoError(t, os.WriteFile(filepath.Join(taskDir, "task.yaml"), []byte(task), 0o644)) + spec := `name: openai-compatible-test +skill: test-skill +config: + trials_per_task: 1 + timeout_seconds: 30 + executor: openai-compatible + endpoint: ` + server.URL + ` + api_key: lm-studio +tasks: + - "tasks/*.yaml" +` + specPath := filepath.Join(dir, "eval.yaml") + require.NoError(t, os.WriteFile(specPath, []byte(spec), 0o644)) + + cmd := newRunCommand() + cmd.SetArgs([]string{specPath}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + + err := cmd.Execute() + require.NoError(t, err) + require.Equal(t, "/v1/chat/completions", hitPath) + require.Equal(t, "Bearer lm-studio", gotAuth) +} + +func TestRunCommand_OpenAICompatibleDefaultsFromWazaYaml(t *testing.T) { + resetRunGlobals() + + var hitPath string + var gotBody struct { + Model string `json:"model"` + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitPath = r.URL.Path + require.Empty(t, r.Header.Get("Authorization")) + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody)) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"defaulted response"}}]}`)) + })) + defer server.Close() + + dir := t.TempDir() + wazaYAML := `defaults: + engine: openai-compatible + endpoint: ` + server.URL + ` +` + require.NoError(t, os.WriteFile(filepath.Join(dir, ".waza.yaml"), []byte(wazaYAML), 0o644)) + + taskDir := filepath.Join(dir, "tasks") + require.NoError(t, os.MkdirAll(taskDir, 0o755)) + task := `id: t1 +name: OpenAI Compatible Defaults Task +inputs: + prompt: "Say hello" +expected: + output_contains: + - "defaulted response" +` + require.NoError(t, os.WriteFile(filepath.Join(taskDir, "task.yaml"), []byte(task), 0o644)) + spec := `name: openai-compatible-defaults-test +skill: test-skill +config: + trials_per_task: 1 + timeout_seconds: 30 +tasks: + - "tasks/*.yaml" +` + specPath := filepath.Join(dir, "eval.yaml") + require.NoError(t, os.WriteFile(specPath, []byte(spec), 0o644)) + t.Setenv("OPENAI_API_KEY", "") + + cmd := newRunCommand() + cmd.SetArgs([]string{specPath}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + + err := cmd.Execute() + require.NoError(t, err) + require.Equal(t, "/v1/chat/completions", hitPath) + require.Equal(t, "local-model", gotBody.Model) +} + // --------------------------------------------------------------------------- // --format flag // --------------------------------------------------------------------------- diff --git a/internal/execution/copilot.go b/internal/execution/copilot.go index 8204b258..2f2bdc7e 100644 --- a/internal/execution/copilot.go +++ b/internal/execution/copilot.go @@ -628,6 +628,10 @@ func (e *CopilotEngine) extractReqParams(req *ExecutionRequest) (modelID string, } func (*CopilotEngine) getSkillDirs(cwd string, req *ExecutionRequest) []string { + return skillDirsForRequest(cwd, req) +} + +func skillDirsForRequest(cwd string, req *ExecutionRequest) []string { skillDirs := []string{cwd} seen := map[string]bool{ diff --git a/internal/execution/openai_compatible.go b/internal/execution/openai_compatible.go new file mode 100644 index 00000000..059bebed --- /dev/null +++ b/internal/execution/openai_compatible.go @@ -0,0 +1,342 @@ +package execution + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + copilot "github.com/github/copilot-sdk/go" + "github.com/microsoft/waza/internal/models" +) + +const defaultOpenAICompatibleModel = "local-model" + +// OpenAICompatibleEngine executes prompts against an OpenAI-compatible chat completions API. +type OpenAICompatibleEngine struct { + endpoint string + modelID string + apiKey string + client *http.Client + + mu sync.Mutex + usage map[string]*models.UsageStats + counter int + + workspaces []string + keepWorkspace bool + shutdownCalled bool +} + +func NewOpenAICompatibleEngine(endpoint, modelID, apiKey string) (*OpenAICompatibleEngine, error) { + chatURL, err := normalizeOpenAICompatibleEndpoint(endpoint) + if err != nil { + return nil, err + } + if strings.TrimSpace(modelID) == "" { + modelID = defaultOpenAICompatibleModel + } + return &OpenAICompatibleEngine{ + endpoint: chatURL, + modelID: modelID, + apiKey: strings.TrimSpace(apiKey), + client: &http.Client{Timeout: 5 * time.Minute}, + usage: make(map[string]*models.UsageStats), + }, nil +} + +func normalizeOpenAICompatibleEndpoint(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", fmt.Errorf("openai-compatible endpoint is required") + } + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return "", fmt.Errorf("invalid openai-compatible endpoint %q: must include scheme and host", raw) + } + u.RawQuery = "" + u.Fragment = "" + path := strings.TrimRight(u.Path, "/") + switch { + case strings.HasSuffix(path, "/chat/completions"): + u.Path = path + case strings.HasSuffix(path, "/v1"): + u.Path = path + "/chat/completions" + case path == "": + u.Path = "/v1/chat/completions" + default: + u.Path = path + "/v1/chat/completions" + } + return u.String(), nil +} + +func (e *OpenAICompatibleEngine) Initialize(context.Context) error { + return nil +} + +func (e *OpenAICompatibleEngine) SetKeepWorkspace(keep bool) { + e.mu.Lock() + defer e.mu.Unlock() + e.keepWorkspace = keep +} + +func (e *OpenAICompatibleEngine) Execute(ctx context.Context, req *ExecutionRequest) (*ExecutionResponse, error) { + if req == nil { + return nil, fmt.Errorf("nil req was passed to OpenAICompatibleEngine.Execute") + } + if err := ctx.Err(); err != nil { + return nil, err + } + var err error + sourceDir := req.SourceDir + if sourceDir == "" { + sourceDir, err = os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get current directory: %w", err) + } + } + + modelID := e.modelID + if strings.TrimSpace(req.ModelID) != "" { + modelID = strings.TrimSpace(req.ModelID) + } + start := time.Now() + + workspaceDir := req.WorkspaceDir + if workspaceDir == "" { + workspaceDir, err = os.MkdirTemp("", "waza-openai-*") + if err != nil { + return nil, fmt.Errorf("failed to create openai-compatible workspace: %w", err) + } + if err := setupWorkspaceResources(workspaceDir, req.Resources); err != nil { + _ = os.RemoveAll(workspaceDir) + return nil, fmt.Errorf("failed to setup openai-compatible workspace resources: %w", err) + } + e.mu.Lock() + e.workspaces = append(e.workspaces, workspaceDir) + e.mu.Unlock() + } + if _, err := ResolveWorkDir(workspaceDir, req.WorkDir); err != nil { + return nil, err + } + + output, usage, errMsg, success, err := e.sendChatCompletion(ctx, modelID, req, sourceDir) + if err != nil { + if workspaceDir != req.WorkspaceDir { + _ = os.RemoveAll(workspaceDir) + } + return nil, err + } + + sessionID := req.SessionID + if sessionID == "" { + sessionID = e.nextSessionID() + } + if usage != nil { + e.mu.Lock() + e.usage[sessionID] = usage + e.mu.Unlock() + } + + resp := &ExecutionResponse{ + FinalOutput: output, + Events: []copilot.SessionEvent{}, + ModelID: modelID, + DurationMs: time.Since(start).Milliseconds(), + ToolCalls: []models.ToolCall{}, + ErrorMsg: errMsg, + Success: success, + SessionID: sessionID, + WorkspaceDir: workspaceDir, + WorkspaceFiles: captureWorkspaceFiles(workspaceDir), + Usage: usage, + } + if req.SkipWorkspaceCapture { + resp.WorkspaceFiles = nil + } + return resp, nil +} + +func (e *OpenAICompatibleEngine) nextSessionID() string { + e.mu.Lock() + defer e.mu.Unlock() + e.counter++ + return fmt.Sprintf("openai-compatible-%d", e.counter) +} + +func (e *OpenAICompatibleEngine) sendChatCompletion(ctx context.Context, modelID string, req *ExecutionRequest, sourceDir string) (string, *models.UsageStats, string, bool, error) { + payload := openAICompatibleChatRequest{ + Model: modelID, + Messages: buildOpenAICompatibleMessages(req, sourceDir), + Stream: false, + } + body, err := json.Marshal(payload) + if err != nil { + return "", nil, "", false, fmt.Errorf("encoding openai-compatible request: %w", err) + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, e.endpoint, bytes.NewReader(body)) + if err != nil { + return "", nil, "", false, fmt.Errorf("creating openai-compatible request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + if apiKey := e.apiKey; apiKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+apiKey) + } else if apiKey := os.Getenv("OPENAI_API_KEY"); apiKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+apiKey) + } + + resp, err := e.client.Do(httpReq) + if err != nil { + return "", nil, "", false, fmt.Errorf("calling openai-compatible endpoint: %w", err) + } + defer resp.Body.Close() //nolint:errcheck + + data, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return "", nil, "", false, fmt.Errorf("reading openai-compatible response: %w", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", nil, string(data), false, nil + } + + var decoded openAICompatibleChatResponse + if err := json.Unmarshal(data, &decoded); err != nil { + return "", nil, "", false, fmt.Errorf("decoding openai-compatible response: %w", err) + } + var output string + if len(decoded.Choices) > 0 { + output = decoded.Choices[0].Message.Content + } + usage := decoded.Usage.toUsageStats(modelID, e.endpoint) + return output, usage, "", true, nil +} + +func buildOpenAICompatibleMessages(req *ExecutionRequest, sourceDir string) []openAICompatibleMessage { + var system []string + if !req.NoSkills { + skillDirs := skillDirsForRequest(sourceDir, req) + if msg := buildSkillSystemMessage(skillDirs, req.SkillName, !req.SuppressSkillBody); msg != "" { + system = append(system, msg) + } + } + for _, f := range req.Instructions { + if len(f.Content) > 0 { + system = append(system, fmt.Sprintf("# %s\n%s", f.Path, string(f.Content))) + } + } + if len(req.Context) > 0 { + if b, err := json.MarshalIndent(req.Context, "", " "); err == nil { + system = append(system, "Context:\n"+string(b)) + } + } + if len(req.Resources) > 0 { + var b strings.Builder + b.WriteString("Files available to the task:\n") + for _, r := range req.Resources { + fmt.Fprintf(&b, "\n## %s\n%s\n", r.Path, string(r.Content)) + } + system = append(system, b.String()) + } + + messages := make([]openAICompatibleMessage, 0, 2) + if len(system) > 0 { + messages = append(messages, openAICompatibleMessage{ + Role: "system", + Content: strings.Join(system, "\n\n"), + }) + } + messages = append(messages, openAICompatibleMessage{ + Role: "user", + Content: req.Message, + }) + return messages +} + +func (e *OpenAICompatibleEngine) Shutdown(context.Context) error { + e.mu.Lock() + if e.shutdownCalled { + e.mu.Unlock() + return nil + } + e.shutdownCalled = true + keep := e.keepWorkspace + workspaces := append([]string(nil), e.workspaces...) + e.workspaces = nil + e.mu.Unlock() + + if keep { + for _, workspace := range workspaces { + fmt.Fprintf(os.Stderr, "Workspace preserved: %s\n", workspace) + } + return nil + } + for _, workspace := range workspaces { + if err := os.RemoveAll(workspace); err != nil { + return fmt.Errorf("failed to remove openai-compatible workspace %s: %w", workspace, err) + } + } + return nil +} + +func (e *OpenAICompatibleEngine) SessionUsage(sessionID string) *models.UsageStats { + e.mu.Lock() + defer e.mu.Unlock() + return e.usage[sessionID] +} + +type openAICompatibleChatRequest struct { + Model string `json:"model"` + Messages []openAICompatibleMessage `json:"messages"` + Stream bool `json:"stream"` +} + +type openAICompatibleMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type openAICompatibleChatResponse struct { + Choices []struct { + Message openAICompatibleMessage `json:"message"` + } `json:"choices"` + Usage openAICompatibleUsage `json:"usage"` +} + +type openAICompatibleUsage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` +} + +func (u openAICompatibleUsage) toUsageStats(modelID, endpoint string) *models.UsageStats { + if u.PromptTokens == 0 && u.CompletionTokens == 0 && u.TotalTokens == 0 { + return nil + } + host := "" + if parsed, err := url.Parse(endpoint); err == nil { + host = parsed.Host + } + stats := &models.UsageStats{ + Turns: 1, + InputTokens: u.PromptTokens, + OutputTokens: u.CompletionTokens, + PremiumRequests: 1, + Provider: models.UsageProviderCustom, + ProviderHost: host, + ModelMetrics: map[string]models.ModelUsage{ + modelID: { + InputTokens: u.PromptTokens, + OutputTokens: u.CompletionTokens, + RequestCount: 1, + }, + }, + } + return stats +} diff --git a/internal/execution/openai_compatible_test.go b/internal/execution/openai_compatible_test.go new file mode 100644 index 00000000..e27bf1c4 --- /dev/null +++ b/internal/execution/openai_compatible_test.go @@ -0,0 +1,161 @@ +package execution + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeOpenAICompatibleEndpoint(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {name: "host root", in: "http://127.0.0.1:1234", want: "http://127.0.0.1:1234/v1/chat/completions"}, + {name: "v1 base", in: "http://127.0.0.1:1234/v1", want: "http://127.0.0.1:1234/v1/chat/completions"}, + {name: "full chat URL", in: "http://127.0.0.1:1234/v1/chat/completions", want: "http://127.0.0.1:1234/v1/chat/completions"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := normalizeOpenAICompatibleEndpoint(tt.in) + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestOpenAICompatibleEngineExecute(t *testing.T) { + var gotPath string + var gotAuth string + var gotBody openAICompatibleChatRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody)) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "choices": [{"message": {"role": "assistant", "content": "hello from lm studio"}}], + "usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10} + }`)) + })) + defer server.Close() + t.Setenv("OPENAI_API_KEY", "test-key") + + engine, err := NewOpenAICompatibleEngine(server.URL, "", "") + require.NoError(t, err) + require.NoError(t, engine.Initialize(t.Context())) + resp, err := engine.Execute(t.Context(), &ExecutionRequest{ + Message: "Say hello", + Context: map[string]any{"case": "unit"}, + Resources: []ResourceFile{ + {Path: "example.txt", Content: []byte("fixture content")}, + }, + }) + require.NoError(t, err) + require.True(t, resp.Success) + require.Equal(t, "hello from lm studio", resp.FinalOutput) + require.Equal(t, defaultOpenAICompatibleModel, resp.ModelID) + require.Equal(t, "/v1/chat/completions", gotPath) + require.Equal(t, "Bearer test-key", gotAuth) + require.Equal(t, defaultOpenAICompatibleModel, gotBody.Model) + require.Len(t, gotBody.Messages, 2) + require.Contains(t, gotBody.Messages[0].Content, "fixture content") + require.Equal(t, "Say hello", gotBody.Messages[1].Content) + require.Equal(t, 7, resp.Usage.InputTokens) + require.Equal(t, 3, resp.Usage.OutputTokens) + require.NotEmpty(t, resp.WorkspaceDir) + + require.NoError(t, engine.Shutdown(t.Context())) + _, statErr := os.Stat(resp.WorkspaceDir) + require.True(t, os.IsNotExist(statErr), "shutdown should remove temporary workspace") +} + +func TestOpenAICompatibleEngineHTTPErrorReturnsExecutionResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "bad model", http.StatusBadRequest) + })) + defer server.Close() + + engine, err := NewOpenAICompatibleEngine(server.URL, "custom-model", "") + require.NoError(t, err) + defer func() { + require.NoError(t, engine.Shutdown(t.Context())) + }() + resp, err := engine.Execute(t.Context(), &ExecutionRequest{Message: "hello"}) + require.NoError(t, err) + require.False(t, resp.Success) + require.Contains(t, resp.ErrorMsg, "bad model") + require.Equal(t, "custom-model", resp.ModelID) +} + +func TestOpenAICompatibleEngineConfiguredAPIKeyOverridesEnvironment(t *testing.T) { + var gotAuth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"ok"}}]}`)) + })) + defer server.Close() + t.Setenv("OPENAI_API_KEY", "env-key") + + engine, err := NewOpenAICompatibleEngine(server.URL, "local-model", "configured-key") + require.NoError(t, err) + defer func() { + require.NoError(t, engine.Shutdown(t.Context())) + }() + resp, err := engine.Execute(t.Context(), &ExecutionRequest{Message: "hello"}) + require.NoError(t, err) + require.True(t, resp.Success) + require.Equal(t, "Bearer configured-key", gotAuth) +} + +func TestOpenAICompatibleEngineInjectsSkillContext(t *testing.T) { + var gotBody openAICompatibleChatRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody)) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"ok"}}]}`)) + })) + defer server.Close() + + skillRoot := t.TempDir() + skillDir := filepath.Join(skillRoot, "test-skill") + require.NoError(t, os.MkdirAll(skillDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(`--- +name: test-skill +description: Test skill. +--- + +# Test Skill + +Always mention injected skill context. +`), 0o644)) + + engine, err := NewOpenAICompatibleEngine(server.URL, "local-model", "") + require.NoError(t, err) + defer func() { + require.NoError(t, engine.Shutdown(t.Context())) + }() + + resp, err := engine.Execute(t.Context(), &ExecutionRequest{ + Message: "hello", + SkillName: "test-skill", + SkillPaths: []string{ + skillRoot, + }, + SourceDir: t.TempDir(), + }) + require.NoError(t, err) + require.True(t, resp.Success) + require.NotEmpty(t, gotBody.Messages) + require.Equal(t, "system", gotBody.Messages[0].Role) + require.Contains(t, gotBody.Messages[0].Content, "") + require.Contains(t, gotBody.Messages[0].Content, "Always mention injected skill context.") + require.Contains(t, gotBody.Messages[0].Content, "") +} diff --git a/internal/models/spec.go b/internal/models/spec.go index 12af5616..4f84dc0b 100644 --- a/internal/models/spec.go +++ b/internal/models/spec.go @@ -42,6 +42,8 @@ type Config struct { StopOnError bool `yaml:"fail_fast,omitempty" json:"stop_on_error,omitempty"` EngineType string `yaml:"executor" json:"engine_type"` ModelID string `yaml:"model" json:"model_id"` + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + APIKey string `yaml:"api_key,omitempty" json:"api_key,omitempty"` SkillPaths []string `yaml:"skill_directories,omitempty" json:"skill_paths,omitempty"` InstructionFiles []string `yaml:"instruction_files,omitempty" json:"instruction_files,omitempty"` InjectSkillBody *bool `yaml:"inject_skill_body,omitempty" json:"inject_skill_body,omitempty"` diff --git a/internal/models/spec_test.go b/internal/models/spec_test.go index 5e444361..d166a31f 100644 --- a/internal/models/spec_test.go +++ b/internal/models/spec_test.go @@ -104,6 +104,60 @@ config: } } +func TestLoadEvalSpec_OpenAICompatibleEndpoint(t *testing.T) { + tempDir := t.TempDir() + yamlContent := `name: openai-compatible +skill: test-skill +config: + trials_per_task: 1 + timeout_seconds: 60 + executor: openai-compatible + endpoint: http://127.0.0.1:1234 + api_key: lm-studio +` + specPath := filepath.Join(tempDir, "spec.yaml") + if err := os.WriteFile(specPath, []byte(yamlContent), 0644); err != nil { + t.Fatalf("Failed to write spec file: %v", err) + } + + spec, err := LoadEvalSpec(specPath) + if err != nil { + t.Fatalf("Failed to load spec: %v", err) + } + if spec.Config.Endpoint != "http://127.0.0.1:1234" { + t.Errorf("Expected endpoint, got %q", spec.Config.Endpoint) + } + if spec.Config.APIKey != "lm-studio" { + t.Errorf("Expected api_key, got %q", spec.Config.APIKey) + } + if spec.Config.ModelID != "" { + t.Errorf("Expected omitted model to remain empty, got %q", spec.Config.ModelID) + } +} + +func TestLoadEvalSpec_OpenAICompatibleAllowsEndpointDefault(t *testing.T) { + tempDir := t.TempDir() + yamlContent := `name: openai-compatible +skill: test-skill +config: + trials_per_task: 1 + timeout_seconds: 60 + executor: openai-compatible +` + specPath := filepath.Join(tempDir, "spec.yaml") + if err := os.WriteFile(specPath, []byte(yamlContent), 0644); err != nil { + t.Fatalf("Failed to write spec file: %v", err) + } + + spec, err := LoadEvalSpec(specPath) + if err != nil { + t.Fatalf("Failed to load spec: %v", err) + } + if spec.Config.Endpoint != "" { + t.Errorf("Expected omitted endpoint to remain empty, got %q", spec.Config.Endpoint) + } +} + func TestTestCase_LoadFromYAML(t *testing.T) { tempDir := t.TempDir() yamlContent := `id: test-001 diff --git a/internal/projectconfig/config.go b/internal/projectconfig/config.go index b24da0e6..db1bdafd 100644 --- a/internal/projectconfig/config.go +++ b/internal/projectconfig/config.go @@ -70,6 +70,7 @@ type FilesConfig struct { type DefaultsConfig struct { Engine string `yaml:"engine,omitempty"` Model string `yaml:"model,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` JudgeModel string `yaml:"judgeModel,omitempty"` Timeout int `yaml:"timeout,omitempty"` Parallel *bool `yaml:"parallel,omitempty"` @@ -294,6 +295,9 @@ func mergeConfig(dst, src *ProjectConfig) { if src.Defaults.Model != "" { dst.Defaults.Model = src.Defaults.Model } + if src.Defaults.Endpoint != "" { + dst.Defaults.Endpoint = src.Defaults.Endpoint + } if src.Defaults.JudgeModel != "" { dst.Defaults.JudgeModel = src.Defaults.JudgeModel } diff --git a/internal/projectconfig/config_test.go b/internal/projectconfig/config_test.go index 0075c66c..2d72d1b8 100644 --- a/internal/projectconfig/config_test.go +++ b/internal/projectconfig/config_test.go @@ -68,6 +68,7 @@ files: defaults: engine: mock model: gpt-4o + endpoint: http://127.0.0.1:1234 judgeModel: claude-sonnet-4.6 timeout: 600 parallel: true @@ -109,6 +110,7 @@ graders: assertEqual(t, "Files.TaskFileSuffix", ".waza-task.yaml", cfg.Files.TaskFileSuffix) assertEqual(t, "Defaults.Engine", "mock", cfg.Defaults.Engine) assertEqual(t, "Defaults.Model", "gpt-4o", cfg.Defaults.Model) + assertEqual(t, "Defaults.Endpoint", "http://127.0.0.1:1234", cfg.Defaults.Endpoint) assertEqual(t, "Defaults.JudgeModel", "claude-sonnet-4.6", cfg.Defaults.JudgeModel) assertEqualInt(t, "Defaults.Timeout", 600, cfg.Defaults.Timeout) assertBoolPtr(t, "Defaults.Parallel", true, cfg.Defaults.Parallel) diff --git a/internal/validation/schema_test.go b/internal/validation/schema_test.go index ea7ea2ad..d9d95247 100644 --- a/internal/validation/schema_test.go +++ b/internal/validation/schema_test.go @@ -108,6 +108,57 @@ func TestValidateEvalBytes_Invalid(t *testing.T) { require.Contains(t, joined, "threshold") } +func TestValidateEvalBytes_OpenAICompatibleConfig(t *testing.T) { + valid := `name: test-eval +skill: test-skill +version: "1.0" +config: + trials_per_task: 1 + timeout_seconds: 60 + executor: openai-compatible + endpoint: http://127.0.0.1:1234 + api_key: lm-studio +metrics: + - name: accuracy + weight: 1.0 + threshold: 0.8 +tasks: + - "tasks/*.yaml" +` + require.Empty(t, ValidateEvalBytes([]byte(valid)), "openai-compatible should allow omitted model when endpoint is set") + + defaultedEndpoint := `name: test-eval +skill: test-skill +version: "1.0" +config: + trials_per_task: 1 + timeout_seconds: 60 + executor: openai-compatible +metrics: + - name: accuracy + weight: 1.0 + threshold: 0.8 +tasks: + - "tasks/*.yaml" +` + require.Empty(t, ValidateEvalBytes([]byte(defaultedEndpoint)), "openai-compatible should allow endpoint from .waza.yaml defaults") + + defaultedExecutor := `name: test-eval +skill: test-skill +version: "1.0" +config: + trials_per_task: 1 + timeout_seconds: 60 +metrics: + - name: accuracy + weight: 1.0 + threshold: 0.8 +tasks: + - "tasks/*.yaml" +` + require.Empty(t, ValidateEvalBytes([]byte(defaultedExecutor)), "eval should allow executor from .waza.yaml defaults") +} + func TestValidateTaskBytes_Valid(t *testing.T) { errs := ValidateTaskBytes([]byte(validTaskYAML)) require.Empty(t, errs, "valid task should have no errors") diff --git a/schemas/config.schema.json b/schemas/config.schema.json index 1ac5cd27..0aff4e72 100644 --- a/schemas/config.schema.json +++ b/schemas/config.schema.json @@ -56,19 +56,24 @@ "type": "object", "description": "Default values applied to evaluations. Used as fallbacks by 'waza run' and as defaults for 'waza new'.", "properties": { - "engine": { - "type": "string", - "description": "Execution engine for evaluations.", - "enum": ["copilot-sdk", "mock"], - "default": "copilot-sdk" - }, - "model": { - "type": "string", - "description": "Default model identifier for evaluations.", - "examples": ["claude-sonnet-4.6", "gpt-4o", "claude-sonnet-4"], - "default": "claude-sonnet-4.6" - }, - "judgeModel": { + "engine": { + "type": "string", + "description": "Execution engine for evaluations.", + "enum": ["copilot-sdk", "mock", "openai-compatible"], + "default": "copilot-sdk" + }, + "model": { + "type": "string", + "description": "Default model identifier for evaluations.", + "examples": ["claude-sonnet-4.6", "gpt-4o", "claude-sonnet-4"], + "default": "claude-sonnet-4.6" + }, + "endpoint": { + "type": "string", + "description": "Default endpoint for OpenAI-compatible evaluations.", + "examples": ["http://127.0.0.1:1234", "http://127.0.0.1:1234/v1/chat/completions"] + }, + "judgeModel": { "type": "string", "description": "Model used for LLM-as-judge grading. If omitted, uses the same model as the evaluation.", "examples": ["claude-sonnet-4.6", "gpt-4o"] diff --git a/schemas/eval.schema.json b/schemas/eval.schema.json index 4c4a409b..1813fe00 100644 --- a/schemas/eval.schema.json +++ b/schemas/eval.schema.json @@ -94,9 +94,26 @@ "type": "object", "required": [ "trials_per_task", - "timeout_seconds", - "executor", - "model" + "timeout_seconds" + ], + "allOf": [ + { + "if": { + "properties": { + "executor": { + "const": "copilot-sdk" + } + }, + "required": [ + "executor" + ] + }, + "then": { + "required": [ + "model" + ] + } + } ], "additionalProperties": false, "description": "Execution configuration for the evaluation.", @@ -130,9 +147,10 @@ "type": "string", "enum": [ "copilot-sdk", - "mock" + "mock", + "openai-compatible" ], - "description": "Execution engine to use. 'copilot-sdk' for real evaluations, 'mock' for testing." + "description": "Execution engine to use. 'copilot-sdk' for Copilot-backed evaluations, 'openai-compatible' for OpenAI-compatible chat completions endpoints, 'mock' for testing." }, "model": { "type": "string", @@ -144,6 +162,24 @@ "gpt-4o-mini" ] }, + "endpoint": { + "type": "string", + "minLength": 1, + "description": "OpenAI-compatible endpoint base URL or chat completions URL. Required for executor: openai-compatible unless supplied by .waza.yaml defaults.endpoint.", + "examples": [ + "http://127.0.0.1:1234", + "http://127.0.0.1:1234/v1", + "http://127.0.0.1:1234/v1/chat/completions" + ] + }, + "api_key": { + "type": "string", + "description": "Optional bearer token for executor: openai-compatible. If omitted or empty, OPENAI_API_KEY is used when set.", + "examples": [ + "lm-studio", + "sk-..." + ] + }, "max_attempts": { "type": "integer", "minimum": 1, diff --git a/site/src/content/docs/guides/eval-yaml.mdx b/site/src/content/docs/guides/eval-yaml.mdx index 33d5c00f..e4cb6725 100644 --- a/site/src/content/docs/guides/eval-yaml.mdx +++ b/site/src/content/docs/guides/eval-yaml.mdx @@ -96,8 +96,10 @@ config: parallel: false # Run tasks sequentially (true = concurrent) workers: 0 # Auto-size parallel workers if parallel: true model: claude-sonnet-4.6 # Default model (override with --model) + endpoint: http://127.0.0.1:1234 # Required for openai-compatible unless set in .waza.yaml + api_key: "" # Optional per-eval override; prefer OPENAI_API_KEY when auth is required judge_model: gpt-4o # Model for LLM-as-judge graders (optional) - executor: mock # mock (local) or copilot-sdk (real API) + executor: mock # mock, copilot-sdk, or openai-compatible inject_skill_body: true # Inject target SKILL.md/.agent.md body into the prompt instruction_files: - .github/instructions/project.instructions.md @@ -109,9 +111,11 @@ config: | `timeout_seconds` | int | 300 | Task timeout in seconds | | `parallel` | bool | false | Run tasks concurrently | | `workers` | int | 0 | Number of parallel workers; 0 auto-sizes | -| `model` | string | _required_ | Default model for tasks (override with `--model` flag) | +| `model` | string | required for `copilot-sdk` | Default model for tasks (override with `--model` flag). Optional for `openai-compatible` | +| `endpoint` | string | required for `openai-compatible` unless set in `.waza.yaml` | OpenAI-compatible base URL or `/v1/chat/completions` URL | +| `api_key` | string | — | Optional per-eval bearer token override; prefer `OPENAI_API_KEY` when auth is required | | `judge_model` | string | (same as `model`) | Model for `prompt`-type graders (LLM-as-judge) | -| `executor` | string | `copilot-sdk` | Executor: `mock` (local, echoes task metadata and file content) or `copilot-sdk` (real API) | +| `executor` | string | `copilot-sdk` | Executor: `mock`, `copilot-sdk`, or `openai-compatible` | | `max_attempts` | int | 0 | Maximum retry attempts per task on failure (0 = no retries) | | `group_by` | string | — | Group results by a field (e.g., `tags`, `task_id`) | | `fail_fast` | bool | false | Stop the entire run on first task failure | diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index b06a2cc0..b7f3537c 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -983,6 +983,7 @@ A newer version of waza is available: v0.24.0 → v0.28.0. Run: curl -fsSL ... | | `COPILOT_WIRE_API` / `COPILOT_PROVIDER_WIRE_API` | Wire format passed through to the Copilot SDK, for example `responses` or `completions`, depending on provider. | | `COPILOT_API_KEY` / `COPILOT_PROVIDER_API_KEY` | API key for the custom provider, if required. | | `COPILOT_BEARER_TOKEN` / `COPILOT_PROVIDER_BEARER_TOKEN` | Bearer token for the custom provider, if required. | +| `OPENAI_API_KEY` | Fallback bearer token for `openai-compatible` when `config.api_key` is omitted or empty. | | `WAZA_HOME` | Config directory (default: `~/.waza`) | | `WAZA_CACHE` | Cache directory (default: `.waza-cache`) | | `WAZA_NO_UPDATE_CHECK` | Set to `1` to disable automatic version check | diff --git a/site/src/content/docs/reference/schema.mdx b/site/src/content/docs/reference/schema.mdx index 3dbd0986..6bed5c2e 100644 --- a/site/src/content/docs/reference/schema.mdx +++ b/site/src/content/docs/reference/schema.mdx @@ -145,7 +145,7 @@ config: **Type:** string **Default:** (empty) -Default LLM model. Override with `--model` flag. +Default LLM model. Required for `copilot-sdk`; optional for `openai-compatible`, where it defaults to `local-model`. Override with `--model` flag. ```yaml config: @@ -156,18 +156,46 @@ config: **Type:** string **Default:** mock -**Options:** `mock`, `copilot-sdk` +**Options:** `mock`, `copilot-sdk`, `openai-compatible` Execution engine: - `mock` — Local testing (no API calls) - `copilot-sdk` — Real LLM execution +- `openai-compatible` — OpenAI-compatible Chat Completions API, such as local LM Studio ```yaml config: executor: copilot-sdk ``` +### endpoint + +**Type:** string +**Required:** when `executor` is `openai-compatible`, unless supplied by `.waza.yaml` `defaults.endpoint` + +OpenAI-compatible endpoint base URL or Chat Completions URL. A base URL such as `http://127.0.0.1:1234` is normalized to `/v1/chat/completions`. + +```yaml +config: + executor: openai-compatible + endpoint: http://127.0.0.1:1234 +``` + +### api_key + +**Type:** string +**Required:** no + +Optional per-eval bearer token override for `openai-compatible`. Prefer `OPENAI_API_KEY` when authentication is required. Local LM Studio can omit this or leave it empty. + +```yaml +config: + executor: openai-compatible + endpoint: http://127.0.0.1:1234 + api_key: "" +``` + ### max_attempts **Type:** integer diff --git a/site/src/content/docs/reference/waza-yaml.mdx b/site/src/content/docs/reference/waza-yaml.mdx index dd3c1d00..0ecd53ec 100644 --- a/site/src/content/docs/reference/waza-yaml.mdx +++ b/site/src/content/docs/reference/waza-yaml.mdx @@ -20,7 +20,9 @@ files: taskFileSuffix: .yaml defaults: + engine: copilot-sdk model: claude-sonnet-4.6 + endpoint: http://127.0.0.1:1234 timeout: 300 workers: 0 ``` @@ -62,6 +64,7 @@ Default execution parameters applied to all commands unless overridden by CLI fl |-------|------|---------|-------------| | `engine` | string | `copilot-sdk` | Execution engine | | `model` | string | `claude-sonnet-4.6` | Default model for execution | +| `endpoint` | string | | Default OpenAI-compatible endpoint. API keys should come from `OPENAI_API_KEY`. | | `judgeModel` | string | | Model for LLM-as-judge graders | | `timeout` | int | `300` | Task timeout in seconds | | `parallel` | bool | `false` | Enable parallel execution | @@ -173,6 +176,7 @@ files: defaults: engine: copilot-sdk model: claude-sonnet-4.6 + endpoint: http://127.0.0.1:1234 timeout: 300 parallel: false workers: 0