Temperature configuration

modal-deploy/sunflower-grpo-inference-vllm/fastapi_app.py

@@ -110,7 +110,7 @@ class GenerateResponse(BaseModel):
 class ProductionRequest(BaseModel):
     instruction: str = Field(..., description="User prompt / instruction")
     model_type: str = Field("qwen", description="Production model type, e.g. `qwen`")
-    temperature: float = Field(0.1, ge=0.0, le=1.0)
+    temperature: float = Field(0.6, ge=0.0, le=1.0)
     system_message: str = Field("", description="Optional system message override")
 
 
@@ -236,6 +236,7 @@ async def generate(req: GenerateRequest):
 async def generate_stream(req: GenerateRequest):
     client: httpx.AsyncClient = app.state.http
 
+    logging.info(f"Received /generate_stream request with body: {req.model_dump_json()}")
     async def passthrough() -> AsyncIterator[bytes]:
         try:
             async with client.stream(
@@ -335,6 +336,7 @@ async def generate_openai_stream(req: GenerateRequest):
 
     async def passthrough() -> AsyncIterator[bytes]:
         try:
+            logging.info(f"Requesting OpenAI stream with body: {json.dumps(_openai_chat_body(req, stream=True))}")
             async with client.stream(
                 "POST", "/chat/completions", json=_openai_chat_body(req, stream=True)
             ) as upstream:

modal-deploy/sunflower-grpo-inference-vllm/web/app/page.tsx

@@ -11,12 +11,15 @@ import {
   type PanelState,
   type ProdMode,
 } from "@/components/compare-panel";
+import { TemperatureSlider } from "@/components/temperature-slider";
 import { buildTranslationPrompt } from "@/lib/prompt";
 import { generateProduction, streamGenerate } from "@/lib/api";
 
 type Mode = "chat" | "translate";
 
 const EMPTY: PanelState = { text: "", status: "idle" };
+const CHAT_TEMP = 0.6;
+const TRANSLATE_TEMP = 0.1;
 
 export default function Page() {
   const [mode, setMode] = useState<Mode>("chat");
@@ -25,13 +28,20 @@ export default function Page() {
   const [grpoBusy, setGrpoBusy] = useState(false);
   const [prodBusy, setProdBusy] = useState(false);
   const [prodMode, setProdMode] = useState<ProdMode>("stream");
+  const [temperature, setTemperature] = useState<number>(CHAT_TEMP);
   const [lastInstruction, setLastInstruction] = useState<string | null>(null);
   const grpoCtrlRef = useRef<AbortController | null>(null);
   const prodCtrlRef = useRef<AbortController | null>(null);
 
   const busy = grpoBusy || prodBusy;
 
-  async function runGrpo(instruction: string) {
+  function handleModeChange(next: Mode) {
+    if (next === mode) return;
+    setMode(next);
+    setTemperature(next === "translate" ? TRANSLATE_TEMP : CHAT_TEMP);
+  }
+
+  async function runGrpo(instruction: string, temp: number) {
     grpoCtrlRef.current?.abort();
     const controller = new AbortController();
     grpoCtrlRef.current = controller;
@@ -63,7 +73,7 @@ export default function Page() {
             setGrpo({ text: "", status: "error", error: err.message });
           },
         },
-        { temperature: 0.2, maxTokens: 1024 },
+        { temperature: temp, maxTokens: 1024 },
       );
     } catch (err) {
       if ((err as Error).name === "AbortError") return;
@@ -77,7 +87,11 @@ export default function Page() {
     }
   }
 
-  async function runProd(instruction: string, forceMode?: ProdMode) {
+  async function runProd(
+    instruction: string,
+    temp: number,
+    forceMode?: ProdMode,
+  ) {
     prodCtrlRef.current?.abort();
     const controller = new AbortController();
     prodCtrlRef.current = controller;
@@ -112,14 +126,15 @@ export default function Page() {
             },
           },
           {
-            temperature: 0.2,
+            temperature: temp,
             maxTokens: 1024,
             endpoint: "/generate_openai_stream",
           },
         );
       } else {
         const data = await generateProduction({
           instruction,
+          temperature: temp,
           signal: controller.signal,
         });
         const text =
@@ -148,20 +163,23 @@ export default function Page() {
     if (next === prodMode) return;
     setProdMode(next);
     if (lastInstruction && !prodBusy) {
-      runProd(lastInstruction, next);
+      runProd(lastInstruction, temperature, next);
     }
   }
 
   async function runComparison(instruction: string) {
     setLastInstruction(instruction);
-    await Promise.all([runGrpo(instruction), runProd(instruction)]);
+    await Promise.all([
+      runGrpo(instruction, temperature),
+      runProd(instruction, temperature),
+    ]);
   }
 
   const retryGrpo = lastInstruction
-    ? () => runGrpo(lastInstruction)
+    ? () => runGrpo(lastInstruction, temperature)
     : undefined;
   const retryProd = lastInstruction
-    ? () => runProd(lastInstruction)
+    ? () => runProd(lastInstruction, temperature)
     : undefined;
 
   const hasResults = grpo.status !== "idle" || prod.status !== "idle";
@@ -193,19 +211,31 @@ export default function Page() {
           <ChatForm
             busy={busy}
             onSubmit={(text) => runComparison(text)}
-            onEnterTranslate={() => setMode("translate")}
+            onEnterTranslate={() => handleModeChange("translate")}
           />
         ) : (
           <TranslateForm
             busy={busy}
             onSubmit={({ source, target, text }) =>
               runComparison(buildTranslationPrompt({ source, target, text }))
             }
-            onExit={() => setMode("chat")}
+            onExit={() => handleModeChange("chat")}
           />
         )}
       </section>
 
+      <section className="mt-3">
+        <TemperatureSlider
+          value={temperature}
+          onChange={setTemperature}
+          hint={
+            mode === "translate"
+              ? "0.1 recommended for translations"
+              : "0.6 recommended for general prompts"
+          }
+        />
+      </section>
+
       {hasResults && (
         <section className="mt-10">
           <ComparePanel

modal-deploy/sunflower-grpo-inference-vllm/web/components/temperature-slider.tsx

@@ -0,0 +1,48 @@
+"use client";
+
+import { Thermometer } from "lucide-react";
+
+interface Props {
+  value: number;
+  onChange: (value: number) => void;
+  hint?: string;
+  min?: number;
+  max?: number;
+  step?: number;
+}
+
+export function TemperatureSlider({
+  value,
+  onChange,
+  hint,
+  min = 0,
+  max = 1,
+  step = 0.05,
+}: Props) {
+  return (
+    <div className="flex flex-col gap-2 rounded-2xl border border-border bg-card px-4 py-3 shadow-sm sm:flex-row sm:items-center sm:gap-4">
+      <div className="flex items-center gap-2">
+        <Thermometer className="h-3.5 w-3.5 text-muted-foreground" />
+        <span className="text-xs font-medium text-foreground">Temperature</span>
+      </div>
+      <input
+        type="range"
+        min={min}
+        max={max}
+        step={step}
+        value={value}
+        onChange={(e) => onChange(parseFloat(e.target.value))}
+        aria-label="Sampling temperature"
+        className="flex-1 accent-accent"
+      />
+      <div className="flex items-center justify-between gap-3 sm:justify-end">
+        <span className="font-mono text-xs tabular-nums text-foreground">
+          {value.toFixed(2)}
+        </span>
+        {hint && (
+          <span className="text-xs text-muted-foreground">{hint}</span>
+        )}
+      </div>
+    </div>
+  );
+}

modal-deploy/sunflower-grpo-inference-vllm/web/lib/api.ts

@@ -87,7 +87,7 @@ export interface ProductionResponse {
 export async function generateProduction({
   instruction,
   modelType = "qwen",
-  temperature = 0.1,
+  temperature = 0.6,
   systemMessage = "",
   signal,
 }: {