11// SPDX-License-Identifier: MIT
2- // Purpose: `sin-code fusion` — SIN Fusion v1 status/config subcommand (issue #290).
3- // Read-only: shows tournament configuration, provider pool, and env var overrides.
42package main
53
64import (
5+ "context"
6+ "encoding/json"
77 "fmt"
88 "os"
9+ "sort"
910 "strings"
11+ "text/tabwriter"
1012
1113 "github.com/spf13/cobra"
1214
13- "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/config "
15+ internal "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal"
1416 "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/fusion"
17+ "github.com/OpenSIN-Code/SIN-Code/cmd/sin-code/internal/modelperf"
1518)
1619
1720func NewFusionCmd () * cobra.Command {
1821 cmd := & cobra.Command {
1922 Use : "fusion" ,
20- Short : "SIN Fusion v1 verify-tournament status and config" ,
21- Long : `sin-code fusion shows the configuration and provider pool for the
22- SIN Fusion v1 verify-tournament (issue #290). When the verify-gate (M3) fails,
23- fusion fans out to N Fireworks models in parallel; first PoC-pass wins.
24- Oracle mode (issue #344) runs all candidates and uses an LLM judge.
25-
26- All subcommands are read-only — no side effects, no API calls.` ,
23+ Short : "SIN Fusion - status, config, benchmarking, model selection" ,
2724 }
2825 cmd .AddCommand (newFusionStatusCmd ())
2926 cmd .AddCommand (newFusionConfigCmd ())
3027 cmd .AddCommand (newFusionProvidersCmd ())
28+ cmd .AddCommand (newFusionBenchmarkCmd ())
29+ cmd .AddCommand (newFusionRankCmd ())
30+ cmd .AddCommand (newFusionRecommendCmd ())
3131 return cmd
3232}
3333
3434func newFusionStatusCmd () * cobra.Command {
3535 return & cobra.Command {
3636 Use : "status" ,
37- Short : "Show fusion enabled/disabled status and gate mode " ,
37+ Short : "Show fusion status" ,
3838 RunE : func (_ * cobra.Command , _ []string ) error {
39- cfg , _ := config .LoadMergedConfig ()
40- fmt .Println ("SIN Fusion v1 — Status" )
41- fmt .Println (strings .Repeat ("─ " , 40 ))
39+ cfg , _ := internal .LoadMergedConfig ()
40+ fmt .Println ("SIN Fusion - Status" )
41+ fmt .Println (strings .Repeat ("- " , 40 ))
4242 fmt .Printf (" Enabled: %v\n " , cfg .FusionEnabled )
4343 fmt .Printf (" Oracle mode: %v\n " , cfg .FusionOracleMode )
44- fmt .Printf (" Difficulty gate: %v\n " , cfg .FusionDifficultyGate )
4544 fmt .Printf (" Max cost (USD): %.2f\n " , cfg .FusionMaxCostUSD )
4645 fmt .Printf (" Min quorum: %d\n " , cfg .FusionMinQuorum )
47- fmt .Printf (" Per-provider TO: %ds\n " , cfg .FusionPerProviderTimeoutS )
48- provStr := ""
49- if len (cfg .FusionProviders ) > 0 {
50- provStr = strings .Join (cfg .FusionProviders , "," )
51- }
52- providers := fusion .LoadFireworksPool (nil , provStr )
46+ providers := fusion .LoadFireworksPool (nil , cfg .FusionProviders )
5347 fmt .Printf (" Providers loaded: %d\n " , len (providers ))
54- if evalModel := os .Getenv ("SIN_EVALUATOR_MODEL" ); evalModel != "" {
55- fmt .Printf (" Evaluator model: %s (SIN_EVALUATOR_MODEL)\n " , evalModel )
56- } else {
57- fmt .Println (" Evaluator model: (worker model fallback)" )
58- }
48+ fmt .Println ()
49+ fmt .Println (" Modes: poc | oracle (default) | plan-merge" )
5950 return nil
6051 },
6152 }
@@ -64,28 +55,19 @@ func newFusionStatusCmd() *cobra.Command {
6455func newFusionConfigCmd () * cobra.Command {
6556 return & cobra.Command {
6657 Use : "config" ,
67- Short : "Show full fusion configuration including env var overrides " ,
58+ Short : "Show fusion configuration" ,
6859 RunE : func (_ * cobra.Command , _ []string ) error {
69- cfg , _ := config .LoadMergedConfig ()
70- fmt .Println ("SIN Fusion v1 — Configuration" )
71- fmt .Println (strings .Repeat ("─" , 40 ))
72- fmt .Println (" Config keys (sin-code.toml):" )
73- fmt .Printf (" fusion.enabled: %v\n " , cfg .FusionEnabled )
74- fmt .Printf (" fusion.oracle_mode: %v\n " , cfg .FusionOracleMode )
75- fmt .Printf (" fusion.difficulty_gate: %v\n " , cfg .FusionDifficultyGate )
76- fmt .Printf (" fusion.max_cost_usd: %.2f\n " , cfg .FusionMaxCostUSD )
77- fmt .Printf (" fusion.min_quorum: %d\n " , cfg .FusionMinQuorum )
78- fmt .Printf (" fusion.per_provider_timeout_s: %d\n " , cfg .FusionPerProviderTimeoutS )
60+ cfg , _ := internal .LoadMergedConfig ()
61+ fmt .Println ("SIN Fusion - Configuration" )
62+ fmt .Println (strings .Repeat ("-" , 40 ))
63+ fmt .Printf (" fusion.enabled: %v\n " , cfg .FusionEnabled )
64+ fmt .Printf (" fusion.oracle_mode: %v\n " , cfg .FusionOracleMode )
65+ fmt .Printf (" fusion.max_cost_usd: %.2f\n " , cfg .FusionMaxCostUSD )
66+ fmt .Printf (" fusion.min_quorum: %d\n " , cfg .FusionMinQuorum )
67+ fmt .Printf (" fusion.per_provider_timeout_s: %d\n " , cfg .FusionPerProviderTimeoutS )
7968 if len (cfg .FusionProviders ) > 0 {
80- fmt .Printf (" fusion.providers: %s\n " , strings .Join (cfg .FusionProviders , ", " ))
81- } else {
82- fmt .Println (" fusion.providers: (default 6-model pool)" )
69+ fmt .Printf (" fusion.providers: %s\n " , strings .Join (cfg .FusionProviders , ", " ))
8370 }
84- fmt .Println ()
85- fmt .Println (" Environment overrides:" )
86- printEnvVar ("SIN_EVALUATOR_MODEL" , "" )
87- printEnvVar ("SIN_EVALUATOR_BASE_URL" , "" )
88- printEnvVar ("SIN_EVALUATOR_API_KEY" , "(masked)" )
8971 return nil
9072 },
9173 }
@@ -94,38 +76,113 @@ func newFusionConfigCmd() *cobra.Command {
9476func newFusionProvidersCmd () * cobra.Command {
9577 return & cobra.Command {
9678 Use : "providers" ,
97- Short : "List the Fireworks pool providers (model, base URL, max tokens) " ,
79+ Short : "List provider pool" ,
9880 RunE : func (_ * cobra.Command , _ []string ) error {
99- cfg , _ := config .LoadMergedConfig ()
100- provStr := ""
101- if len (cfg .FusionProviders ) > 0 {
102- provStr = strings .Join (cfg .FusionProviders , "," )
103- }
104- providers := fusion .LoadFireworksPool (nil , provStr )
105- fmt .Println ("SIN Fusion v1 — Provider Pool" )
106- fmt .Println (strings .Repeat ("─" , 40 ))
81+ cfg , _ := internal .LoadMergedConfig ()
82+ providers := fusion .LoadFireworksPool (nil , cfg .FusionProviders )
83+ fmt .Println ("SIN Fusion - Provider Pool" )
10784 if len (providers ) == 0 {
108- fmt .Println (" No providers loaded (check fusion.providers config or FIREWORKS_API_KEY) " )
85+ fmt .Println (" No providers loaded" )
10986 return nil
11087 }
111- fmt .Printf (" %-30s %-40s %s\n " , "MODEL" , "BASE URL" , "MAX TOKENS" )
112- fmt .Printf (" %s %s %s\n " , strings .Repeat ("─" , 30 ), strings .Repeat ("─" , 40 ), strings .Repeat ("─" , 10 ))
11388 for _ , p := range providers {
11489 fmt .Printf (" %-30s %-40s %d\n " , p .Model , p .BaseURL , p .MaxTokens )
11590 }
116- fmt .Printf ("\n Total: %d providers\n " , len (providers ))
11791 return nil
11892 },
11993 }
12094}
12195
122- func printEnvVar (key , mask string ) {
123- val := os .Getenv (key )
124- if val == "" {
125- fmt .Printf (" %s: (not set)\n " , key )
126- } else if mask != "" {
127- fmt .Printf (" %s: %s\n " , key , mask )
128- } else {
129- fmt .Printf (" %s: %s\n " , key , val )
96+ func newFusionBenchmarkCmd () * cobra.Command {
97+ var datasetPath , category , providersFlag string
98+ cmd := & cobra.Command {
99+ Use : "benchmark" ,
100+ Short : "Run dataset across providers (issue #395)" ,
101+ RunE : func (cmd * cobra.Command , args []string ) error {
102+ if datasetPath == "" {
103+ return fmt .Errorf ("--dataset is required" )
104+ }
105+ store , err := modelperf .Open ("" )
106+ if err != nil { return err }
107+ defer store .Close ()
108+ names := splitCommas (providersFlag )
109+ if len (names ) == 0 { names = []string {"minimax-m3" ,"kimi-k2p7-code-fast" ,"glm-5p2" } }
110+ provs := make ([]modelperf.BenchmarkProvider , 0 , len (names ))
111+ for _ , n := range names { provs = append (provs , & stubBP {n }) }
112+ out , err := modelperf .RunBenchmark (context .Background (), store , provs , modelperf.BenchmarkConfig {DatasetPath : datasetPath , Category : category })
113+ if err != nil { return err }
114+ w := tabwriter .NewWriter (os .Stdout , 0 ,0 ,2 ,' ' ,0 )
115+ fmt .Fprintf (w , "Category:\t %s\n " , out .Category )
116+ fmt .Fprintf (w , "Dataset:\t %s\n " , out .Dataset )
117+ fmt .Fprintf (w , "Cases:\t %d\n " , out .Cases )
118+ fmt .Fprintf (w , "\n Model\t Pass Rate\t Avg Latency\t Avg Cost\n " )
119+ sort .Slice (out .Results , func (i ,j int ) bool { return out .Results [i ].PassRate > out .Results [j ].PassRate })
120+ for _ , r := range out .Results {
121+ fmt .Fprintf (w , "%s\t %.1f%%\t %v\t $%.4f\n " , r .Model , r .PassRate * 100 , r .AvgLatency , r .AvgCost )
122+ }
123+ w .Flush ()
124+ return nil
125+ },
126+ }
127+ cmd .Flags ().StringVarP (& datasetPath , "dataset" , "d" , "" , "eval dataset JSON" )
128+ cmd .Flags ().StringVarP (& category , "category" , "c" , "" , "task category" )
129+ cmd .Flags ().StringVarP (& providersFlag , "providers" , "p" , "" , "comma-separated models" )
130+ return cmd
131+ }
132+
133+ func newFusionRankCmd () * cobra.Command {
134+ var jsonOut bool
135+ cmd := & cobra.Command {
136+ Use : "rank" ,
137+ Short : "Show model leaderboard (issue #395)" ,
138+ RunE : func (cmd * cobra.Command , args []string ) error {
139+ store , err := modelperf .Open ("" )
140+ if err != nil { return err }
141+ defer store .Close ()
142+ recs , err := store .Ranking (context .Background ())
143+ if err != nil { return err }
144+ if len (recs ) == 0 { fmt .Println ("No data. Run benchmark first." ); return nil }
145+ if jsonOut { return json .NewEncoder (os .Stdout ).Encode (recs ) }
146+ w := tabwriter .NewWriter (os .Stdout , 0 ,0 ,2 ,' ' ,0 )
147+ fmt .Fprintf (w , "Category\t Model\t Pass Rate\t Samples\t Cost\n " )
148+ for _ , r := range recs { fmt .Fprintf (w , "%s\t %s\t %.1f%%\t %d\t $%.4f\n " , r .Category , r .Model , r .PassRate * 100 , r .SampleCount , r .AvgCostUSD ) }
149+ w .Flush ()
150+ return nil
151+ },
130152 }
153+ cmd .Flags ().BoolVar (& jsonOut , "json" , false , "JSON output" )
154+ return cmd
155+ }
156+
157+ func newFusionRecommendCmd () * cobra.Command {
158+ var task string ; var n , minS int
159+ cmd := & cobra.Command {
160+ Use : "recommend" ,
161+ Short : "Best models for a task (issue #395)" ,
162+ RunE : func (cmd * cobra.Command , args []string ) error {
163+ if task == "" { return fmt .Errorf ("--task required" ) }
164+ store , err := modelperf .Open ("" )
165+ if err != nil { return err }
166+ defer store .Close ()
167+ recs , err := store .Recommend (context .Background (), task , n , minS )
168+ if err != nil { return err }
169+ if len (recs ) == 0 { fmt .Printf ("No data for %q.\n " , task ); return nil }
170+ w := tabwriter .NewWriter (os .Stdout , 0 ,0 ,2 ,' ' ,0 )
171+ fmt .Fprintf (w , "Rank\t Model\t Score\t Pass Rate\t Samples\n " )
172+ for i , r := range recs { fmt .Fprintf (w , "%d\t %s\t %.3f\t %.1f%%\t %d\n " , i + 1 , r .Model , r .Score , r .PassRate * 100 , r .Samples ) }
173+ w .Flush ()
174+ return nil
175+ },
176+ }
177+ cmd .Flags ().StringVarP (& task , "task" , "t" , "" , "task category" )
178+ cmd .Flags ().IntVarP (& n , "top" , "n" , 3 , "number of recs" )
179+ cmd .Flags ().IntVar (& minS , "min-samples" , 1 , "min benchmark runs" )
180+ return cmd
181+ }
182+
183+ func splitCommas (s string ) []string { if s == "" { return nil }; return strings .Split (s , "," ) }
184+ type stubBP struct { name string }
185+ func (s * stubBP ) Name () string { return s .name }
186+ func (s * stubBP ) Run (ctx context.Context , prompt string ) (modelperf.BenchmarkResult , error ) {
187+ return modelperf.BenchmarkResult {Passed : true , Output : "stub" }, nil
131188}
0 commit comments