Feat/explanation quality l2r

eval/loaders/fast_dvlm_sglang_v3.py

@@ -268,6 +268,35 @@ def generate(bundle, image_paths, question, max_new_tokens=None, temperature=0.0
     # Build per-position vocab gates. Length matches padded template.
     gates = _build_v3_position_gates(tokenizer, slot_info, n_template)
     json_bad = list(bundle.get("json_blacklist") or [])
+
+    # --- nav-conditional lateral gate ---------------------------------------
+    # navigation_command is an INPUT (we are told the maneuver), so the lateral
+    # verb is not something to "predict" — constrain it. This makes nav accuracy
+    # deterministic and robust to steps_per_chunk / slot-length tuning (the soft
+    # nav-injection alone is fragile: at steps=16 it flips GO_RIGHT to "keep
+    # lane"). We must *force* the maneuver, not merely forbid the opposite
+    # direction: allowing "lane" lets GO_RIGHT escape to "keep lane". So for a
+    # turn command we drop "keep"/"lane" entirely (lat_w1 ∈ {turn,change},
+    # lat_w2 ∈ {left|right}); for GO_STRAIGHT we pin lat_w1=keep, lat_w2=lane.
+    if nav_command:
+        enc1 = lambda s: (tokenizer.encode(s, add_special_tokens=False) or [-1])[0]
+        ids = {w: enc1(w) for w in ("left", "right", "lane", "keep", "turn", "change")}
+        nav = nav_command.upper()
+        if "LEFT" in nav:
+            w1_allow, w2_allow = {ids["turn"], ids["change"]}, {ids["left"]}
+        elif "RIGHT" in nav:
+            w1_allow, w2_allow = {ids["turn"], ids["change"]}, {ids["right"]}
+        else:  # GO_STRAIGHT / unknown → keep lane
+            w1_allow, w2_allow = {ids["keep"]}, {ids["lane"]}
+        kind_allow = {"lat_w1": w1_allow, "lat_w2": w2_allow}
+        for local_pos, kind in slot_info:
+            allow = kind_allow.get(kind)
+            if allow and local_pos < n_template:
+                base = gates[local_pos]
+                gates[local_pos] = sorted(
+                    allow if base is None else (set(base) & allow) or allow
+                )
+
     # Also exclude JSON-meta tokens from gated allowlists in case any sneak in.
     json_bad_set = set(json_bad)
     for i, allowed in enumerate(gates):
@@ -298,10 +327,30 @@ def generate(bundle, image_paths, question, max_new_tokens=None, temperature=0.0
         # rep_penalty * count_so_far. Plus within-step dedup ensures top-K
         # commits in one step pick distinct tokens.
         "dllm_template_rep_penalty_positions": rep_positions,
-        "dllm_template_rep_penalty": 2.0,
-        # Steps per chunk (caller override → default 4). 4 = fast (1.7-2s)
-        # but coarser ADE; 8 = closer to transformers loader's ADE at 2-3s.
-        "dllm_template_steps_per_chunk": kwargs.get("steps_per_chunk", 4),
+        # rep_penalty default depends on the explanation fill mode:
+        #   parallel top-K → 4.0: needed to suppress the `upup`/`speedspeed`
+        #     token-repetition collapse when many adjacent masks commit at once.
+        #   L2R/AR → 0.0: one-at-a-time decoding already conditions on committed
+        #     left context, so it doesn't repeat; a nonzero penalty there HURTS
+        #     — it pushes the natural token off (e.g. "overcast"→"scast",
+        #     orphan "ists") to produce subword fragments.
+        "dllm_template_rep_penalty": kwargs.get(
+            "rep_penalty", 0.0 if kwargs.get("explanation_l2r", True) else 4.0,
+        ),
+        # steps_per_chunk 16 (was 4): the dominant lever for explanation
+        # coherence. More refinement passes → each committed token conditions
+        # on more already-decided neighbors, so the prose reads as sentences
+        # rather than word-salad. At the full 100-mask slot this lands at
+        # ~2.3-2.5s; nav stays 5/5 via the conditional lateral gate above
+        # (not the fragile soft-injection).
+        "dllm_template_steps_per_chunk": kwargs.get("steps_per_chunk", 16),
+        # Fill the explanation slot strictly left-to-right, one token per
+        # forward (AR), so each token conditions on its committed left context.
+        # Removes the parallel-commit BPE-boundary glue ("cyclistsists") that
+        # top-K diffusion produces on the long free-text run. Structured slots
+        # (critical/behavior/trajectory) still fill in parallel. Costs ~1 extra
+        # forward per explanation token (latency ↑), so it's the quality knob.
+        "dllm_template_explanation_l2r": kwargs.get("explanation_l2r", True),
     }
 
     torch.cuda.synchronize()
@@ -317,9 +366,44 @@ def generate(bundle, image_paths, question, max_new_tokens=None, temperature=0.0
     if isinstance(out, list):
         out = out[0]
     text = out.get("text", "") if isinstance(out, dict) else str(out)
+    text = _truncate_explanation_bleed(text)
     return text, latency
 
 
+# Markers that signal the explanation has stopped being real prose and started
+# bleeding the next JSON field's name / the trajectory numbers into the slot.
+# The model produces coherent content for ~50-70 tokens, then (forced to fill
+# the remaining masks) drifts into this. Banning the tokens just makes it route
+# around them (mangled / foreign-script variants), so instead we KEEP the good
+# prefix and drop the bleed tail from the decoded string.
+import re as _re
+
+_BLEED_MARKERS = _re.compile(
+    r"(future[_ ]?meta|meta[_ ]?behavior|_behavior|behaviorfuture|futurefuture|"
+    r"longitudinal|lateral\s*:|trajectory|waypoint|\bego\b|未来|"
+    r"\d+(?:\.\d+)?\s*s\s*:|forward\s*=|lateral\s*=)",
+    _re.IGNORECASE,
+)
+
+
+def _truncate_explanation_bleed(text: str) -> str:
+    """Cut the `explanation` value at the first bleed marker, keeping the
+    coherent prefix. Leaves the rest of the JSON (behavior, trajectory) intact."""
+    m = _re.search(r'("explanation":\s*")(.*?)("\s*,\s*"(?:navigation_command|future_meta_behavior)")',
+                   text, _re.DOTALL)
+    if not m:
+        return text
+    exp = m.group(2)
+    bleed = _BLEED_MARKERS.search(exp)
+    if not bleed:
+        return text
+    clean = exp[:bleed.start()]
+    # Trim a dangling partial word and trailing separators/space.
+    clean = _re.sub(r"\s+\S*$", "", clean) if " " in clean else clean
+    clean = clean.rstrip(" ,.;:-_")
+    return text[:m.start(2)] + clean + text[m.end(2):]
+
+
 def shutdown(bundle):
     engine = bundle.get("engine")
     if engine is not None:

eval/template_v3.py

@@ -13,6 +13,7 @@
 """
 from __future__ import annotations
 import math
+import os
 import re
 import json
 from typing import List, Tuple
@@ -30,7 +31,10 @@
 # critical_objects: 2 mask tokens per category — just enough for "none" or a
 # short 2-token phrase like "black car" / "green light".
 N_CRITICAL_TOKENS = 2
-N_EXPLANATION_TOKENS = 100
+# Explanation slot length. Overridable via env for tuning: the real content
+# is ~2-3 sentences (~50-70 tokens); a longer slot lets the model run out of
+# things to say and drift into bleeding the next JSON fields into the prose.
+N_EXPLANATION_TOKENS = int(os.environ.get("DVLA_N_EXPLANATION_TOKENS", "100"))
 
 # Trajectory: 10 waypoints @ 2 Hz (0.5 s spacing) covering 5 s horizon.
 # Each waypoint = `<sign><tens><ones>.<frac>,<sign><tens><ones>.<frac>` with

examples/test_image_run/meta.json

@@ -1,5 +1,5 @@
 {
-  "image": "/weka/home/ext-yingzima/dVLA-AD/examples/test_image.png",
+  "image": "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21/examples/test_image.png",
   "scenario": "Highway, vehicle on fire ahead, orange cones diverting traffic. Ego cruising at 15 m/s, lightly decelerating, nav=GO_STRAIGHT.",
   "mock_speed_m_s": 15.0,
   "mock_accel_m_s2": -0.5,

examples/test_image_run/output_sglang_template.json

@@ -1,5 +1,5 @@
 {
   "path": "Fast_dVLM_3B via modified SGLang fork; template-fill mdm (vocab gates + JSON blacklist + rep penalty + nav injection)",
-  "latency_s": 1.563105583190918,
-  "output_text": "{\"critical_objects\": {\"nearby_vehicle\": \"black car\", \"pedestrian\": \"none found\", \"cyclist\": \" found here\", \"construction\": \"none seen\", \"traffic_element\": \"orange cone\", \"weather_condition\": \"overcast\", \"road_hazard\": \"fire smoke\", \"emergency_vehicle\": \"not seen\", \"animal\": \"none visible\", \"special_vehicle\": \"none detected\", \"conflicting_vehicle\": \"none detected\", \"door_opening_vehicle\": \"none found\"}, \"complexity\": \"complex\", \"explanation\": \"The scene shows a highway with a fire of, indicated by smoke and the presence of. road lane cones. The black driver is to on the straight going 1  speed instructions ahead, However there up down in front path lane turning or left turn right as current as not hazard traffic future safety lateral behavior long future meta stop now go keep forward no vehicle slowinging collision within pedestrian next change left avoid right need s5sl_futuremetafuture_meta_behaviorlongitud:2 verbspeed=50\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"slow down\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+05.5m, lateral=+00.0m\n1.0s: forward=+05.5m, lateral=+00.0m\n1.5s: forward=+05.5m, lateral=+00.0m\n2.0s: forward=+05.5m, lateral=+00.0m\n2.5s: forward=+05.5m, lateral=+00.0m\n3.0s: forward=+05.5m, lateral=+00.0m\n3.5s: forward=+05.5m, lateral=+00.0m\n4.0s: forward=+55.5m, lateral=+00.0m\n4.5s: forward=+55.5m, lateral=+00.0m\n5.0s: forward=+55.5m, lateral=+00.0m\"}"
+  "latency_s": 2.698542833328247,
+  "output_text": "{\"critical_objects\": {\"nearby_vehicle\": \"none car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic cyclist\", \"construction\": \"none zone\", \"traffic_element\": \"none road\", \"weather_condition\": \"overcast\", \"road_hazard\": \"fire smoke\", \"emergency_vehicle\": \"none vehicle\", \"animal\": \"none traffic\", \"special_vehicle\": \"none vehicle\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"complex\", \"explanation\": \"The scene shows a multi-view image of a highway with a fire or smoke ahead, which is a hazard. The driver's instruction is to go straight, but the current speed and the presence of the hazard require caution. The driver should slow down and keep a distance from the hazard. The road is a highway, and the weather is overcast. There are no nearby vehicles or pedestrians, and the traffic is not complex The road hazard is a fire or smoke ahead, which requires caution and slowing down\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"slow down\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+35.5m, lateral=+00.0m\n1.0s: forward=+45.0m, lateral=+00.0m\n1.5s: forward=+55.5m, lateral=+00.0m\n2.0s: forward=+65.0m, lateral=+00.0m\n2.5s: forward=+75.0m, lateral=+00.0m\n3.0s: forward=+85.5m, lateral=+00.0m\n3.5s: forward=+95.0m, lateral=+00.0m\n4.0s: forward=+00.5m, lateral=+00.0m\n4.5s: forward=+10.0m, lateral=+00.0m\n5.0s: forward=+20.0m, lateral=+00.0m\"}"
 }

examples/test_image_run/prompt_full.txt

@@ -56,18 +56,34 @@ OUTPUT FORMAT REQUIREMENTS:
      safety overrides the high-level nav.
 
 7. trajectory: 10 future ego waypoints at 0.5 s spacing (t = 0.5, 1.0, ...
-   5.0 s) — one line per waypoint in the form
+   5.0 s) — one line per waypoint:
        `<t>s: forward=<sign><tens><ones>.<frac>m, lateral=<sign><tens><ones>.<frac>m`
-   • `forward` is the ego-frame +x distance (meters, +forward / -reverse)
-   • `lateral` is the ego-frame +y offset (meters, +left / -right)
-   • each coordinate has a sign (`+`/`-`), two integer digits, ONE decimal
-   Example output (current speed 15.0 m/s, going straight, no turn):
-       0.5s: forward=+00.5m, lateral=+00.0m
-       1.0s: forward=+05.5m, lateral=+00.0m
-       ...
-   At 15.0 m/s the forward distance grows by ~7.50 m per
-   0.5 s step. Lateral offset stays near 0 when going straight; grows
-   negative on a right turn, positive on a left turn.
+
+   Coordinate frame:
+   • `forward` = ego-frame x. POSITIVE = ahead.
+   • `lateral` = ego-frame y. POSITIVE = LEFT of the car;
+     NEGATIVE = RIGHT of the car.
+
+   The lateral SIGN encodes the turn direction; the lateral MAGNITUDE
+   encodes how aggressively the ego deviates from the original heading:
+
+     GO_LEFT     → lateral signs are POSITIVE, magnitude grows over time
+                   (e.g. +00.3m → +00.8m → +01.5m → +02.4m)
+     GO_RIGHT    → lateral signs are NEGATIVE, magnitude grows over time
+                   (e.g. -00.3m → -00.8m → -01.5m → -02.4m)
+     GO_STRAIGHT → lateral stays near zero (sign +, magnitude 00.0m)
+
+   Worked example for the CURRENT sample (instruction=GO_STRAIGHT,
+   speed 15.0 m/s) — your output should look like:
+       0.5s: forward=+07.5m, lateral=+00.0m
+       1.0s: forward=+15.0m, lateral=+00.0m
+       2.5s: forward=+37.5m, lateral=+00.0m
+       5.0s: forward=+75.0m, lateral=+00.0m
+       (lateral stays 00.0 for all 10 lines)
+
+   At 15.0 m/s the forward distance grows ~7.50 m per 0.5 s
+   step. Turning lateral grows ~0.3 m per 0.5 s step at low speed,
+   ~0.8 m per step at moderate speed.
 
 TEMPLATE (fill the <|mdm_mask|> positions only — keep all other characters
 verbatim):

examples/test_image_run/prompt_to_model.txt

@@ -56,15 +56,31 @@ OUTPUT FORMAT REQUIREMENTS:
      safety overrides the high-level nav.
 
 7. trajectory: 10 future ego waypoints at 0.5 s spacing (t = 0.5, 1.0, ...
-   5.0 s) — one line per waypoint in the form
+   5.0 s) — one line per waypoint:
        `<t>s: forward=<sign><tens><ones>.<frac>m, lateral=<sign><tens><ones>.<frac>m`
-   • `forward` is the ego-frame +x distance (meters, +forward / -reverse)
-   • `lateral` is the ego-frame +y offset (meters, +left / -right)
-   • each coordinate has a sign (`+`/`-`), two integer digits, ONE decimal
-   Example output (current speed 15.0 m/s, going straight, no turn):
-       0.5s: forward=+00.5m, lateral=+00.0m
-       1.0s: forward=+05.5m, lateral=+00.0m
-       ...
-   At 15.0 m/s the forward distance grows by ~7.50 m per
-   0.5 s step. Lateral offset stays near 0 when going straight; grows
-   negative on a right turn, positive on a left turn.
+
+   Coordinate frame:
+   • `forward` = ego-frame x. POSITIVE = ahead.
+   • `lateral` = ego-frame y. POSITIVE = LEFT of the car;
+     NEGATIVE = RIGHT of the car.
+
+   The lateral SIGN encodes the turn direction; the lateral MAGNITUDE
+   encodes how aggressively the ego deviates from the original heading:
+
+     GO_LEFT     → lateral signs are POSITIVE, magnitude grows over time
+                   (e.g. +00.3m → +00.8m → +01.5m → +02.4m)
+     GO_RIGHT    → lateral signs are NEGATIVE, magnitude grows over time
+                   (e.g. -00.3m → -00.8m → -01.5m → -02.4m)
+     GO_STRAIGHT → lateral stays near zero (sign +, magnitude 00.0m)
+
+   Worked example for the CURRENT sample (instruction=GO_STRAIGHT,
+   speed 15.0 m/s) — your output should look like:
+       0.5s: forward=+07.5m, lateral=+00.0m
+       1.0s: forward=+15.0m, lateral=+00.0m
+       2.5s: forward=+37.5m, lateral=+00.0m
+       5.0s: forward=+75.0m, lateral=+00.0m
+       (lateral stays 00.0 for all 10 lines)
+
+   At 15.0 m/s the forward distance grows ~7.50 m per 0.5 s
+   step. Turning lateral grows ~0.3 m per 0.5 s step at low speed,
+   ~0.8 m per step at moderate speed.