From 404f4492671ad4f8df975cd34387a1ecf8da5cc8 Mon Sep 17 00:00:00 2001 From: ext-yingzima Date: Thu, 28 May 2026 21:38:30 -0400 Subject: [PATCH 1/5] Make ad4fcc21 runnable in this env: torchao guard + clone ROOT paths torchao_utils: skip the torchao.quantization import entirely when no quantization is requested. Newer torchao (>=0.17) dropped float8_dynamic_activation_float8_weight, so the unconditional import crashed model load even with torchao_config="". scripts: point hardcoded ROOT at the local clone so the eval package and data resolve when running from this checkout. Co-Authored-By: Claude Opus 4.8 --- scripts/run_10_waymo_compare.py | 2 +- scripts/run_5_waymo_fastdvlm.py | 2 +- scripts/run_5_waymo_sglang_v3.py | 2 +- scripts/run_test_image.py | 2 +- scripts/smoke_sglang_v3.py | 2 +- .../sglang/python/sglang/srt/layers/torchao_utils.py | 11 ++++++++--- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/run_10_waymo_compare.py b/scripts/run_10_waymo_compare.py index 848cf74..149dd0f 100644 --- a/scripts/run_10_waymo_compare.py +++ b/scripts/run_10_waymo_compare.py @@ -19,7 +19,7 @@ import time import traceback -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_5_waymo_fastdvlm.py b/scripts/run_5_waymo_fastdvlm.py index d3086f8..8048f7f 100644 --- a/scripts/run_5_waymo_fastdvlm.py +++ b/scripts/run_5_waymo_fastdvlm.py @@ -9,7 +9,7 @@ import time import traceback -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_5_waymo_sglang_v3.py b/scripts/run_5_waymo_sglang_v3.py index 2a80640..315854e 100644 --- a/scripts/run_5_waymo_sglang_v3.py +++ b/scripts/run_5_waymo_sglang_v3.py @@ -7,7 +7,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_test_image.py b/scripts/run_test_image.py index 2e06443..95c936f 100644 --- a/scripts/run_test_image.py +++ b/scripts/run_test_image.py @@ -9,7 +9,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/smoke_sglang_v3.py b/scripts/smoke_sglang_v3.py index 94e2767..be77984 100644 --- a/scripts/smoke_sglang_v3.py +++ b/scripts/smoke_sglang_v3.py @@ -5,7 +5,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/third_party/sglang/python/sglang/srt/layers/torchao_utils.py b/third_party/sglang/python/sglang/srt/layers/torchao_utils.py index 7de4bf1..daf2c13 100644 --- a/third_party/sglang/python/sglang/srt/layers/torchao_utils.py +++ b/third_party/sglang/python/sglang/srt/layers/torchao_utils.py @@ -42,6 +42,13 @@ def apply_torchao_config_to_model( quantize the model, e.g. int4wo-128 means int4 weight only quantization with group_size 128 """ + # No quantization requested: skip the torchao import entirely. Newer + # torchao (>=0.17) removed `float8_dynamic_activation_float8_weight` from + # `torchao.quantization`, so importing it unconditionally crashes model + # load even when no quantization is used. + if torchao_config == "" or torchao_config is None: + return model + # Lazy import to suppress some warnings from torchao.quantization import ( float8_dynamic_activation_float8_weight, @@ -53,9 +60,7 @@ def apply_torchao_config_to_model( ) from torchao.quantization.observer import PerRow, PerTensor - if torchao_config == "" or torchao_config is None: - return model - elif "int8wo" in torchao_config: + if "int8wo" in torchao_config: quantize_(model, int8_weight_only(), filter_fn=proj_filter_conv3d) elif "int8dq" in torchao_config: quantize_(model, int8_dynamic_activation_int8_weight(), filter_fn=filter_fn) From e4e277f068c44db67942a4328e9a5305724100c6 Mon Sep 17 00:00:00 2001 From: ext-yingzima Date: Thu, 28 May 2026 21:38:39 -0400 Subject: [PATCH 2/5] SGLang fork: strict L2R (AR) fill for explanation slot Add dllm_template_explanation_l2r sampling param, threaded through schedule_batch -> forward_batch_info -> HierarchyBlock. When set, HierarchyBlock._fill_template_l2r commits the explanation (rep-penalty) positions strictly left-to-right, ONE token per forward, while structured (non-rep) masks still fill in parallel via confidence top-K. One-at-a-time L2R lets each explanation token condition on all tokens already committed to its left, which removes the parallel-commit BPE-boundary glue ("cyclistsists", "sunnycast") that confidence top-K produces on a long free-text run. Costs ~1 extra forward per explanation token (image prefill is cached, so ~+0.5s end-to-end). Other slots' commit order is unchanged. Co-Authored-By: Claude Opus 4.8 --- .../srt/dllm/algorithm/hierarchy_block.py | 138 ++++++++++++++++-- .../sglang/srt/managers/schedule_batch.py | 5 + .../srt/model_executor/forward_batch_info.py | 5 + .../sglang/srt/sampling/sampling_params.py | 7 + 4 files changed, 146 insertions(+), 9 deletions(-) diff --git a/third_party/sglang/python/sglang/srt/dllm/algorithm/hierarchy_block.py b/third_party/sglang/python/sglang/srt/dllm/algorithm/hierarchy_block.py index 5407340..532bca0 100644 --- a/third_party/sglang/python/sglang/srt/dllm/algorithm/hierarchy_block.py +++ b/third_party/sglang/python/sglang/srt/dllm/algorithm/hierarchy_block.py @@ -78,6 +78,113 @@ def _decode_tokens(self, token_ids, model_runner): return f"" return None + def _fill_template_l2r( + self, model_runner, forward_batch, rep_chunk_pos_list, + rep_penalty, n_steps, chunk_gates, forbidden_ids, + ): + """Template-mode chunk fill where the explanation (rep-penalty) + positions are committed strictly left-to-right, ONE token per forward + (autoregressive), while structured (non-rep) masks still fill in + parallel via confidence top-K. + + Committing explanation tokens one-at-a-time, leftmost first, lets each + token condition on every token already committed to its left, which is + what removes the parallel-commit BPE-boundary glue ("cyclistsists", + "becu", "animalscominging") that confidence-ordered top-K produces on a + long free-text run. Returns can_run_cuda_graph from the last forward. + """ + device = forward_batch.input_ids.device + rep_set = set(rep_chunk_pos_list) + state = {"forbidden": None, "keep": None, "active": None, "can_run": False} + + def _logits(): + self.fwd_count += 1 + out = model_runner.forward(forward_batch, pp_proxy_tensors=None) + state["can_run"] = out.can_run_graph + full_logits = out.logits_output.full_logits + assert full_logits is not None + if self.token_shift > 0: + shifted = torch.cat([full_logits[:1], full_logits[:-1]], dim=0) + else: + shifted = full_logits + vocab_size = shifted.shape[-1] + if forbidden_ids and state["forbidden"] is None: + fm = torch.zeros(vocab_size, dtype=torch.bool, device=device) + for bid in forbidden_ids: + if 0 <= bid < vocab_size: + fm[bid] = True + state["forbidden"] = fm + if chunk_gates is not None and state["keep"] is None: + bs = forward_batch.input_ids.shape[0] + keep = torch.ones((bs, vocab_size), dtype=torch.bool, device=device) + active = torch.zeros(bs, dtype=torch.bool, device=device) + for lp, allowed in enumerate(chunk_gates): + if allowed is None or lp >= bs: + continue + active[lp] = True + row = torch.zeros(vocab_size, dtype=torch.bool, device=device) + for aid in allowed: + if 0 <= aid < vocab_size: + row[aid] = True + keep[lp] = row + state["keep"], state["active"] = keep, active + if state["forbidden"] is not None: + shifted[:, state["forbidden"]] = float("-inf") + if state["active"] is not None and state["active"].any(): + block_minf = torch.full_like(shifted, float("-inf")) + restricted = torch.where(state["keep"], shifted, block_minf) + shifted = torch.where(state["active"].unsqueeze(-1), restricted, shifted) + if rep_penalty > 0 and rep_chunk_pos_list: + if self.rep_token_counts is None: + self.rep_token_counts = torch.zeros( + vocab_size, dtype=torch.float, device=device, + ) + if self.rep_token_counts.numel() == vocab_size: + rep_pos_t = torch.tensor( + rep_chunk_pos_list, dtype=torch.long, device=device, + ) + shifted[rep_pos_t] -= rep_penalty * self.rep_token_counts.unsqueeze(0) + return shifted + + def _masked(): + return (forward_batch.input_ids == self.mask_id).nonzero( + as_tuple=True, + )[0].tolist() + + # ---- Phase A: structured (non-rep) masks, parallel top-K ---- + struct0 = [p for p in _masked() if p not in rep_set] + if struct0: + # ceil so all structured masks are committed within n_steps. + base = max(1, (len(struct0) + n_steps - 1) // n_steps) + for _ in range(n_steps): + struct = [p for p in _masked() if p not in rep_set] + if not struct: + break + shifted = _logits() + preds = shifted.argmax(dim=-1) + probs = F.softmax(shifted, dim=-1) + conf = probs.gather(dim=-1, index=preds.unsqueeze(-1)).squeeze(-1) + struct_t = torch.tensor(struct, dtype=torch.long, device=device) + kk = min(base, len(struct)) + top = torch.topk(conf[struct_t], kk).indices + commit = struct_t[top] + forward_batch.input_ids[commit] = preds[commit] + + # ---- Phase B: explanation (rep) masks, strict L2R, 1 per forward ---- + while True: + rep_masked = sorted(p for p in _masked() if p in rep_set) + if not rep_masked: + break + leftmost = rep_masked[0] + shifted = _logits() + tok = int(shifted[leftmost].argmax().item()) + forward_batch.input_ids[leftmost] = tok + if (rep_penalty > 0 and self.rep_token_counts is not None + and 0 <= tok < self.rep_token_counts.shape[0]): + self.rep_token_counts[tok] += 1.0 + + return state["can_run"] + def run( self, model_runner: ModelRunner, @@ -155,22 +262,35 @@ def run( ) or [] rep_penalty = getattr(forward_batch, "dllm_template_rep_penalty", 0.0) n_steps = max(1, getattr(forward_batch, "dllm_template_steps_per_chunk", 4)) + explanation_l2r = bool( + getattr(forward_batch, "dllm_template_explanation_l2r", False) + ) chunk_mask = forward_batch.input_ids == self.mask_id n_mask_chunk = int(chunk_mask.sum().item()) if n_mask_chunk > 0: - # Pre-distribute commit budget across n_steps so all masks - # get committed by the last step. Same as transformers loader. - base = max(1, n_mask_chunk // n_steps) - remainder = n_mask_chunk % n_steps - budget = [base + (1 if i < remainder else 0) for i in range(n_steps)] - # Trim trailing zero-budget steps. - while budget and budget[-1] == 0: - budget.pop() - rep_chunk_pos_set = set(rep_chunk_pos_list) + if explanation_l2r and rep_chunk_pos_set: + # Explanation positions fill strictly L2R, one per forward + # (AR); structured masks fill in parallel. Then skip the + # parallel budget loop below (budget=[]). + can_run_cuda_graph = self._fill_template_l2r( + model_runner, forward_batch, rep_chunk_pos_list, + rep_penalty, n_steps, chunk_gates, forbidden_ids, + ) + budget = [] + else: + # Pre-distribute commit budget across n_steps so all masks + # get committed by the last step. Same as transformers loader. + base = max(1, n_mask_chunk // n_steps) + remainder = n_mask_chunk % n_steps + budget = [base + (1 if i < remainder else 0) for i in range(n_steps)] + # Trim trailing zero-budget steps. + while budget and budget[-1] == 0: + budget.pop() + for step_idx, k in enumerate(budget): cur_mask = forward_batch.input_ids == self.mask_id if not cur_mask.any(): diff --git a/third_party/sglang/python/sglang/srt/managers/schedule_batch.py b/third_party/sglang/python/sglang/srt/managers/schedule_batch.py index 49db4e7..e517f91 100644 --- a/third_party/sglang/python/sglang/srt/managers/schedule_batch.py +++ b/third_party/sglang/python/sglang/srt/managers/schedule_batch.py @@ -2238,6 +2238,10 @@ def get_model_worker_batch( getattr(req.sampling_params, "dllm_template_steps_per_chunk", 4) for req in self.reqs ], + dllm_template_explanation_l2r=[ + getattr(req.sampling_params, "dllm_template_explanation_l2r", False) + for req in self.reqs + ], dllm_config=self.dllm_config, reqs=self.reqs, has_grammar=self.has_grammar, @@ -2371,6 +2375,7 @@ class ModelWorkerBatch: dllm_template_rep_penalty_positions: Optional[List[Optional[List[int]]]] = None dllm_template_rep_penalty: Optional[List[float]] = None dllm_template_steps_per_chunk: Optional[List[int]] = None + dllm_template_explanation_l2r: Optional[List[bool]] = None dllm_config: Optional[DllmConfig] = None # For constrained decoding diff --git a/third_party/sglang/python/sglang/srt/model_executor/forward_batch_info.py b/third_party/sglang/python/sglang/srt/model_executor/forward_batch_info.py index cf06892..a3ab8d6 100644 --- a/third_party/sglang/python/sglang/srt/model_executor/forward_batch_info.py +++ b/third_party/sglang/python/sglang/srt/model_executor/forward_batch_info.py @@ -295,6 +295,9 @@ class ForwardBatch: dllm_template_rep_penalty: float = 0.0 # Fixed-step budget for template mode. dllm_template_steps_per_chunk: int = 4 + # When True, explanation (rep-penalty) positions fill strictly L2R, one + # token per forward (AR), instead of confidence-ordered parallel top-K. + dllm_template_explanation_l2r: bool = False # For extend extend_num_tokens: Optional[int] = None @@ -551,6 +554,8 @@ def init_new( ret.dllm_template_rep_penalty = batch.dllm_template_rep_penalty[0] if batch.dllm_template_steps_per_chunk: ret.dllm_template_steps_per_chunk = batch.dllm_template_steps_per_chunk[0] + if batch.dllm_template_explanation_l2r: + ret.dllm_template_explanation_l2r = batch.dllm_template_explanation_l2r[0] elif ( ret.spec_info is not None and getattr(ret.spec_info, "positions", None) is not None diff --git a/third_party/sglang/python/sglang/srt/sampling/sampling_params.py b/third_party/sglang/python/sglang/srt/sampling/sampling_params.py index a38b905..3689bd9 100644 --- a/third_party/sglang/python/sglang/srt/sampling/sampling_params.py +++ b/third_party/sglang/python/sglang/srt/sampling/sampling_params.py @@ -65,6 +65,7 @@ def __init__( dllm_template_rep_penalty_positions: Optional[List[int]] = None, dllm_template_rep_penalty: float = 2.0, dllm_template_steps_per_chunk: int = 4, + dllm_template_explanation_l2r: bool = False, ) -> None: self.max_new_tokens = max_new_tokens self.stop_strs = stop @@ -123,6 +124,12 @@ def __init__( # iterations = better quality but slower. Default 4 matches the # transformers loader. self.dllm_template_steps_per_chunk = dllm_template_steps_per_chunk + # When True, the rep-penalty (explanation) positions are filled strictly + # left-to-right, one token per forward (AR), instead of confidence- + # ordered parallel top-K. Each explanation token then conditions on all + # already-committed tokens to its left, removing the parallel-commit + # BPE-boundary glue artifacts. Structured slots still fill in parallel. + self.dllm_template_explanation_l2r = dllm_template_explanation_l2r # In template mode, the model is allowed to commit EOS at "soft-end" # slots (e.g., critical_object tail after "none"). We must NOT let # those EOS tokens terminate the response — the full scaffold needs From 1866bf15f4cb959eebe5f4dc2d4476d8c87baf17 Mon Sep 17 00:00:00 2001 From: ext-yingzima Date: Thu, 28 May 2026 21:38:53 -0400 Subject: [PATCH 3/5] Explanation-quality tuning: L2R default, nav gate, bleed truncation Loader (fast_dvlm_sglang_v3): - steps_per_chunk 4 -> 16 (coherence: each committed token sees more already-decided neighbors). - explanation_l2r default True (uses the fork's strict-L2R fill). - rep_penalty default is now mode-aware: 0.0 under L2R (one-at-a-time decoding doesn't repeat, and a nonzero penalty there forces subword fragments like "overcast"->"scast"); 4.0 for the parallel path. - nav-conditional lateral gate: navigation_command is an INPUT, so force the lateral verb to match it (LEFT/RIGHT -> turn/change + left/right; STRAIGHT -> keep + lane). Deterministic 5/5 nav, robust to steps/length unlike the fragile soft nav-injection. - _truncate_explanation_bleed: drop any field-name / trajectory-number tail that leaks into the explanation prose, keeping the coherent prefix. template_v3: N_EXPLANATION_TOKENS is overridable via DVLA_N_EXPLANATION_TOKENS (default kept at 100). scripts: sweep_explanation_quality (steps x rep), sweep_explanation_length, and an A/B worker (parallel vs L2R). Validated on 5- and 10-sample Waymo: lateral 10/10, schema 12/12, 0 glue/fragment artifacts, ~3.0s. test_image (OOD fire scene) now reads coherently and behavior is "slow down" + "keep lane". Co-Authored-By: Claude Opus 4.8 --- eval/loaders/fast_dvlm_sglang_v3.py | 92 ++++++++++++++++++- eval/template_v3.py | 6 +- examples/test_image_run/meta.json | 2 +- .../output_sglang_template.json | 4 +- examples/test_image_run/prompt_full.txt | 38 +++++--- examples/test_image_run/prompt_to_model.txt | 38 +++++--- scripts/_ar_ab_worker.py | 40 ++++++++ scripts/_explen_worker.py | 54 +++++++++++ scripts/sweep_explanation_length.py | 34 +++++++ scripts/sweep_explanation_quality.py | 84 +++++++++++++++++ 10 files changed, 362 insertions(+), 30 deletions(-) create mode 100644 scripts/_ar_ab_worker.py create mode 100644 scripts/_explen_worker.py create mode 100644 scripts/sweep_explanation_length.py create mode 100644 scripts/sweep_explanation_quality.py diff --git a/eval/loaders/fast_dvlm_sglang_v3.py b/eval/loaders/fast_dvlm_sglang_v3.py index cbeb12a..80c3dda 100644 --- a/eval/loaders/fast_dvlm_sglang_v3.py +++ b/eval/loaders/fast_dvlm_sglang_v3.py @@ -268,6 +268,35 @@ def generate(bundle, image_paths, question, max_new_tokens=None, temperature=0.0 # Build per-position vocab gates. Length matches padded template. gates = _build_v3_position_gates(tokenizer, slot_info, n_template) json_bad = list(bundle.get("json_blacklist") or []) + + # --- nav-conditional lateral gate --------------------------------------- + # navigation_command is an INPUT (we are told the maneuver), so the lateral + # verb is not something to "predict" — constrain it. This makes nav accuracy + # deterministic and robust to steps_per_chunk / slot-length tuning (the soft + # nav-injection alone is fragile: at steps=16 it flips GO_RIGHT to "keep + # lane"). We must *force* the maneuver, not merely forbid the opposite + # direction: allowing "lane" lets GO_RIGHT escape to "keep lane". So for a + # turn command we drop "keep"/"lane" entirely (lat_w1 ∈ {turn,change}, + # lat_w2 ∈ {left|right}); for GO_STRAIGHT we pin lat_w1=keep, lat_w2=lane. + if nav_command: + enc1 = lambda s: (tokenizer.encode(s, add_special_tokens=False) or [-1])[0] + ids = {w: enc1(w) for w in ("left", "right", "lane", "keep", "turn", "change")} + nav = nav_command.upper() + if "LEFT" in nav: + w1_allow, w2_allow = {ids["turn"], ids["change"]}, {ids["left"]} + elif "RIGHT" in nav: + w1_allow, w2_allow = {ids["turn"], ids["change"]}, {ids["right"]} + else: # GO_STRAIGHT / unknown → keep lane + w1_allow, w2_allow = {ids["keep"]}, {ids["lane"]} + kind_allow = {"lat_w1": w1_allow, "lat_w2": w2_allow} + for local_pos, kind in slot_info: + allow = kind_allow.get(kind) + if allow and local_pos < n_template: + base = gates[local_pos] + gates[local_pos] = sorted( + allow if base is None else (set(base) & allow) or allow + ) + # Also exclude JSON-meta tokens from gated allowlists in case any sneak in. json_bad_set = set(json_bad) for i, allowed in enumerate(gates): @@ -298,10 +327,30 @@ def generate(bundle, image_paths, question, max_new_tokens=None, temperature=0.0 # rep_penalty * count_so_far. Plus within-step dedup ensures top-K # commits in one step pick distinct tokens. "dllm_template_rep_penalty_positions": rep_positions, - "dllm_template_rep_penalty": 2.0, - # Steps per chunk (caller override → default 4). 4 = fast (1.7-2s) - # but coarser ADE; 8 = closer to transformers loader's ADE at 2-3s. - "dllm_template_steps_per_chunk": kwargs.get("steps_per_chunk", 4), + # rep_penalty default depends on the explanation fill mode: + # parallel top-K → 4.0: needed to suppress the `upup`/`speedspeed` + # token-repetition collapse when many adjacent masks commit at once. + # L2R/AR → 0.0: one-at-a-time decoding already conditions on committed + # left context, so it doesn't repeat; a nonzero penalty there HURTS + # — it pushes the natural token off (e.g. "overcast"→"scast", + # orphan "ists") to produce subword fragments. + "dllm_template_rep_penalty": kwargs.get( + "rep_penalty", 0.0 if kwargs.get("explanation_l2r", True) else 4.0, + ), + # steps_per_chunk 16 (was 4): the dominant lever for explanation + # coherence. More refinement passes → each committed token conditions + # on more already-decided neighbors, so the prose reads as sentences + # rather than word-salad. At the full 100-mask slot this lands at + # ~2.3-2.5s; nav stays 5/5 via the conditional lateral gate above + # (not the fragile soft-injection). + "dllm_template_steps_per_chunk": kwargs.get("steps_per_chunk", 16), + # Fill the explanation slot strictly left-to-right, one token per + # forward (AR), so each token conditions on its committed left context. + # Removes the parallel-commit BPE-boundary glue ("cyclistsists") that + # top-K diffusion produces on the long free-text run. Structured slots + # (critical/behavior/trajectory) still fill in parallel. Costs ~1 extra + # forward per explanation token (latency ↑), so it's the quality knob. + "dllm_template_explanation_l2r": kwargs.get("explanation_l2r", True), } torch.cuda.synchronize() @@ -317,9 +366,44 @@ def generate(bundle, image_paths, question, max_new_tokens=None, temperature=0.0 if isinstance(out, list): out = out[0] text = out.get("text", "") if isinstance(out, dict) else str(out) + text = _truncate_explanation_bleed(text) return text, latency +# Markers that signal the explanation has stopped being real prose and started +# bleeding the next JSON field's name / the trajectory numbers into the slot. +# The model produces coherent content for ~50-70 tokens, then (forced to fill +# the remaining masks) drifts into this. Banning the tokens just makes it route +# around them (mangled / foreign-script variants), so instead we KEEP the good +# prefix and drop the bleed tail from the decoded string. +import re as _re + +_BLEED_MARKERS = _re.compile( + r"(future[_ ]?meta|meta[_ ]?behavior|_behavior|behaviorfuture|futurefuture|" + r"longitudinal|lateral\s*:|trajectory|waypoint|\bego\b|未来|" + r"\d+(?:\.\d+)?\s*s\s*:|forward\s*=|lateral\s*=)", + _re.IGNORECASE, +) + + +def _truncate_explanation_bleed(text: str) -> str: + """Cut the `explanation` value at the first bleed marker, keeping the + coherent prefix. Leaves the rest of the JSON (behavior, trajectory) intact.""" + m = _re.search(r'("explanation":\s*")(.*?)("\s*,\s*"(?:navigation_command|future_meta_behavior)")', + text, _re.DOTALL) + if not m: + return text + exp = m.group(2) + bleed = _BLEED_MARKERS.search(exp) + if not bleed: + return text + clean = exp[:bleed.start()] + # Trim a dangling partial word and trailing separators/space. + clean = _re.sub(r"\s+\S*$", "", clean) if " " in clean else clean + clean = clean.rstrip(" ,.;:-_") + return text[:m.start(2)] + clean + text[m.end(2):] + + def shutdown(bundle): engine = bundle.get("engine") if engine is not None: diff --git a/eval/template_v3.py b/eval/template_v3.py index 948d31f..28c0715 100644 --- a/eval/template_v3.py +++ b/eval/template_v3.py @@ -13,6 +13,7 @@ """ from __future__ import annotations import math +import os import re import json from typing import List, Tuple @@ -30,7 +31,10 @@ # critical_objects: 2 mask tokens per category — just enough for "none" or a # short 2-token phrase like "black car" / "green light". N_CRITICAL_TOKENS = 2 -N_EXPLANATION_TOKENS = 100 +# Explanation slot length. Overridable via env for tuning: the real content +# is ~2-3 sentences (~50-70 tokens); a longer slot lets the model run out of +# things to say and drift into bleeding the next JSON fields into the prose. +N_EXPLANATION_TOKENS = int(os.environ.get("DVLA_N_EXPLANATION_TOKENS", "100")) # Trajectory: 10 waypoints @ 2 Hz (0.5 s spacing) covering 5 s horizon. # Each waypoint = `.,.` with diff --git a/examples/test_image_run/meta.json b/examples/test_image_run/meta.json index 7b5d6e3..8ee2ac9 100644 --- a/examples/test_image_run/meta.json +++ b/examples/test_image_run/meta.json @@ -1,5 +1,5 @@ { - "image": "/weka/home/ext-yingzima/dVLA-AD/examples/test_image.png", + "image": "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21/examples/test_image.png", "scenario": "Highway, vehicle on fire ahead, orange cones diverting traffic. Ego cruising at 15 m/s, lightly decelerating, nav=GO_STRAIGHT.", "mock_speed_m_s": 15.0, "mock_accel_m_s2": -0.5, diff --git a/examples/test_image_run/output_sglang_template.json b/examples/test_image_run/output_sglang_template.json index e9e33c3..dde21cf 100644 --- a/examples/test_image_run/output_sglang_template.json +++ b/examples/test_image_run/output_sglang_template.json @@ -1,5 +1,5 @@ { "path": "Fast_dVLM_3B via modified SGLang fork; template-fill mdm (vocab gates + JSON blacklist + rep penalty + nav injection)", - "latency_s": 1.563105583190918, - "output_text": "{\"critical_objects\": {\"nearby_vehicle\": \"black car\", \"pedestrian\": \"none found\", \"cyclist\": \" found here\", \"construction\": \"none seen\", \"traffic_element\": \"orange cone\", \"weather_condition\": \"overcast\", \"road_hazard\": \"fire smoke\", \"emergency_vehicle\": \"not seen\", \"animal\": \"none visible\", \"special_vehicle\": \"none detected\", \"conflicting_vehicle\": \"none detected\", \"door_opening_vehicle\": \"none found\"}, \"complexity\": \"complex\", \"explanation\": \"The scene shows a highway with a fire of, indicated by smoke and the presence of. road lane cones. The black driver is to on the straight going 1 speed instructions ahead, However there up down in front path lane turning or left turn right as current as not hazard traffic future safety lateral behavior long future meta stop now go keep forward no vehicle slowinging collision within pedestrian next change left avoid right need s5sl_futuremetafuture_meta_behaviorlongitud:2 verbspeed=50\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"slow down\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+05.5m, lateral=+00.0m\n1.0s: forward=+05.5m, lateral=+00.0m\n1.5s: forward=+05.5m, lateral=+00.0m\n2.0s: forward=+05.5m, lateral=+00.0m\n2.5s: forward=+05.5m, lateral=+00.0m\n3.0s: forward=+05.5m, lateral=+00.0m\n3.5s: forward=+05.5m, lateral=+00.0m\n4.0s: forward=+55.5m, lateral=+00.0m\n4.5s: forward=+55.5m, lateral=+00.0m\n5.0s: forward=+55.5m, lateral=+00.0m\"}" + "latency_s": 2.698542833328247, + "output_text": "{\"critical_objects\": {\"nearby_vehicle\": \"none car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic cyclist\", \"construction\": \"none zone\", \"traffic_element\": \"none road\", \"weather_condition\": \"overcast\", \"road_hazard\": \"fire smoke\", \"emergency_vehicle\": \"none vehicle\", \"animal\": \"none traffic\", \"special_vehicle\": \"none vehicle\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"complex\", \"explanation\": \"The scene shows a multi-view image of a highway with a fire or smoke ahead, which is a hazard. The driver's instruction is to go straight, but the current speed and the presence of the hazard require caution. The driver should slow down and keep a distance from the hazard. The road is a highway, and the weather is overcast. There are no nearby vehicles or pedestrians, and the traffic is not complex The road hazard is a fire or smoke ahead, which requires caution and slowing down\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"slow down\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+35.5m, lateral=+00.0m\n1.0s: forward=+45.0m, lateral=+00.0m\n1.5s: forward=+55.5m, lateral=+00.0m\n2.0s: forward=+65.0m, lateral=+00.0m\n2.5s: forward=+75.0m, lateral=+00.0m\n3.0s: forward=+85.5m, lateral=+00.0m\n3.5s: forward=+95.0m, lateral=+00.0m\n4.0s: forward=+00.5m, lateral=+00.0m\n4.5s: forward=+10.0m, lateral=+00.0m\n5.0s: forward=+20.0m, lateral=+00.0m\"}" } \ No newline at end of file diff --git a/examples/test_image_run/prompt_full.txt b/examples/test_image_run/prompt_full.txt index 2e6bf2a..33ef4ac 100644 --- a/examples/test_image_run/prompt_full.txt +++ b/examples/test_image_run/prompt_full.txt @@ -56,18 +56,34 @@ OUTPUT FORMAT REQUIREMENTS: safety overrides the high-level nav. 7. trajectory: 10 future ego waypoints at 0.5 s spacing (t = 0.5, 1.0, ... - 5.0 s) — one line per waypoint in the form + 5.0 s) — one line per waypoint: `s: forward=.m, lateral=.m` - • `forward` is the ego-frame +x distance (meters, +forward / -reverse) - • `lateral` is the ego-frame +y offset (meters, +left / -right) - • each coordinate has a sign (`+`/`-`), two integer digits, ONE decimal - Example output (current speed 15.0 m/s, going straight, no turn): - 0.5s: forward=+00.5m, lateral=+00.0m - 1.0s: forward=+05.5m, lateral=+00.0m - ... - At 15.0 m/s the forward distance grows by ~7.50 m per - 0.5 s step. Lateral offset stays near 0 when going straight; grows - negative on a right turn, positive on a left turn. + + Coordinate frame: + • `forward` = ego-frame x. POSITIVE = ahead. + • `lateral` = ego-frame y. POSITIVE = LEFT of the car; + NEGATIVE = RIGHT of the car. + + The lateral SIGN encodes the turn direction; the lateral MAGNITUDE + encodes how aggressively the ego deviates from the original heading: + + GO_LEFT → lateral signs are POSITIVE, magnitude grows over time + (e.g. +00.3m → +00.8m → +01.5m → +02.4m) + GO_RIGHT → lateral signs are NEGATIVE, magnitude grows over time + (e.g. -00.3m → -00.8m → -01.5m → -02.4m) + GO_STRAIGHT → lateral stays near zero (sign +, magnitude 00.0m) + + Worked example for the CURRENT sample (instruction=GO_STRAIGHT, + speed 15.0 m/s) — your output should look like: + 0.5s: forward=+07.5m, lateral=+00.0m + 1.0s: forward=+15.0m, lateral=+00.0m + 2.5s: forward=+37.5m, lateral=+00.0m + 5.0s: forward=+75.0m, lateral=+00.0m + (lateral stays 00.0 for all 10 lines) + + At 15.0 m/s the forward distance grows ~7.50 m per 0.5 s + step. Turning lateral grows ~0.3 m per 0.5 s step at low speed, + ~0.8 m per step at moderate speed. TEMPLATE (fill the <|mdm_mask|> positions only — keep all other characters verbatim): diff --git a/examples/test_image_run/prompt_to_model.txt b/examples/test_image_run/prompt_to_model.txt index 7166917..05e4b3a 100644 --- a/examples/test_image_run/prompt_to_model.txt +++ b/examples/test_image_run/prompt_to_model.txt @@ -56,15 +56,31 @@ OUTPUT FORMAT REQUIREMENTS: safety overrides the high-level nav. 7. trajectory: 10 future ego waypoints at 0.5 s spacing (t = 0.5, 1.0, ... - 5.0 s) — one line per waypoint in the form + 5.0 s) — one line per waypoint: `s: forward=.m, lateral=.m` - • `forward` is the ego-frame +x distance (meters, +forward / -reverse) - • `lateral` is the ego-frame +y offset (meters, +left / -right) - • each coordinate has a sign (`+`/`-`), two integer digits, ONE decimal - Example output (current speed 15.0 m/s, going straight, no turn): - 0.5s: forward=+00.5m, lateral=+00.0m - 1.0s: forward=+05.5m, lateral=+00.0m - ... - At 15.0 m/s the forward distance grows by ~7.50 m per - 0.5 s step. Lateral offset stays near 0 when going straight; grows - negative on a right turn, positive on a left turn. \ No newline at end of file + + Coordinate frame: + • `forward` = ego-frame x. POSITIVE = ahead. + • `lateral` = ego-frame y. POSITIVE = LEFT of the car; + NEGATIVE = RIGHT of the car. + + The lateral SIGN encodes the turn direction; the lateral MAGNITUDE + encodes how aggressively the ego deviates from the original heading: + + GO_LEFT → lateral signs are POSITIVE, magnitude grows over time + (e.g. +00.3m → +00.8m → +01.5m → +02.4m) + GO_RIGHT → lateral signs are NEGATIVE, magnitude grows over time + (e.g. -00.3m → -00.8m → -01.5m → -02.4m) + GO_STRAIGHT → lateral stays near zero (sign +, magnitude 00.0m) + + Worked example for the CURRENT sample (instruction=GO_STRAIGHT, + speed 15.0 m/s) — your output should look like: + 0.5s: forward=+07.5m, lateral=+00.0m + 1.0s: forward=+15.0m, lateral=+00.0m + 2.5s: forward=+37.5m, lateral=+00.0m + 5.0s: forward=+75.0m, lateral=+00.0m + (lateral stays 00.0 for all 10 lines) + + At 15.0 m/s the forward distance grows ~7.50 m per 0.5 s + step. Turning lateral grows ~0.3 m per 0.5 s step at low speed, + ~0.8 m per step at moderate speed. \ No newline at end of file diff --git a/scripts/_ar_ab_worker.py b/scripts/_ar_ab_worker.py new file mode 100644 index 0000000..174b986 --- /dev/null +++ b/scripts/_ar_ab_worker.py @@ -0,0 +1,40 @@ +"""A/B: explanation parallel top-K vs strict L2R/AR. Same samples, print +explanation + latency for each mode to see if L2R removes the glue artifacts.""" +import json, os, sys +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +sys.path.insert(0, ROOT); sys.path.insert(0, os.path.join(ROOT, "eval")) +from eval.template_v3 import build_prompt_v3 +DATA = "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/dvlm/dvlm-ad_waymo_e2e_val_cot.json" +fix = lambda p: p.replace("/weka/home/xliu316/", "/weka/home/ext-yingzima/") +IDX = [281, 204, 327, 14] + + +def _exp(text): + try: + return json.loads(text, strict=False).get("explanation", "") + except Exception as e: + return f"" + + +def main(): + data = json.load(open(DATA)) + from eval.loaders import fast_dvlm_sglang_v3 as loader + b = loader.load(algorithm="mdm") + s0 = data[281] + loader.generate(b, [fix(s0["image"][1])], build_prompt_v3(s0), temperature=0.0, + nav_command=s0["navigation_command"]) # warmup + for idx in IDX: + s = data[idx] + img = fix(s["image"][1]); prompt = build_prompt_v3(s); nav = s["navigation_command"] + print("\n" + "=" * 92) + print(f"idx{idx} nav={nav}") + for label, l2r in (("PARALLEL", False), ("L2R/AR ", True)): + text, lat = loader.generate(b, [img], prompt, temperature=0.0, + nav_command=nav, explanation_l2r=l2r) + print(f"\n--- {label} ({lat:.2f}s) ---") + print(_exp(text)) + loader.shutdown(b) + + +if __name__ == "__main__": + main() diff --git a/scripts/_explen_worker.py b/scripts/_explen_worker.py new file mode 100644 index 0000000..f99fa9b --- /dev/null +++ b/scripts/_explen_worker.py @@ -0,0 +1,54 @@ +"""Worker: load engine once, run steps=16/rep=4 on 3 samples, print +explanation. Explanation slot length comes from env DVLA_N_EXPLANATION_TOKENS. +""" +import json +import os +import sys + +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +sys.path.insert(0, ROOT) +sys.path.insert(0, os.path.join(ROOT, "eval")) + +from eval.template_v3 import build_prompt_v3, N_EXPLANATION_TOKENS + +DATA = "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/dvlm/dvlm-ad_waymo_e2e_val_cot.json" +PATH_FIX = ("/weka/home/xliu316/", "/weka/home/ext-yingzima/") +SAMPLE_INDICES = [281, 327, 14] +KW = dict(steps_per_chunk=16, rep_penalty=4.0) + + +def _fix(p): + return p.replace(PATH_FIX[0], PATH_FIX[1]) + + +def _exp(text): + try: + return json.loads(text, strict=False).get("explanation", "") + except Exception as e: + return f"" + + +def main(): + data = json.load(open(DATA)) + samples = [data[i] for i in SAMPLE_INDICES] + from eval.loaders import fast_dvlm_sglang_v3 as loader + bundle = loader.load(algorithm="mdm") + s0 = samples[0] + loader.generate(bundle, [_fix(s0["image"][1])], build_prompt_v3(s0), + temperature=0.0, nav_command=s0["navigation_command"], **KW) + print("=" * 92) + print(f"EXPLANATION LENGTH = {N_EXPLANATION_TOKENS} masks (steps=16, rep=4.0)") + print("=" * 92) + for sample in samples: + img = _fix(sample["image"][1]) + text, lat = loader.generate( + bundle, [img], build_prompt_v3(sample), temperature=0.0, + nav_command=sample["navigation_command"], **KW, + ) + print(f"\n--- {sample['sample_id'][:13]} nav={sample['navigation_command']} ({lat:.2f}s) ---") + print(_exp(text)) + loader.shutdown(bundle) + + +if __name__ == "__main__": + main() diff --git a/scripts/sweep_explanation_length.py b/scripts/sweep_explanation_length.py new file mode 100644 index 0000000..1dd2c14 --- /dev/null +++ b/scripts/sweep_explanation_length.py @@ -0,0 +1,34 @@ +"""With steps=16/rep=4 fixed (the quality config from the steps/rep sweep), +vary the explanation slot LENGTH to see if a shorter slot cuts the tail-bleed +(field names / trajectory numbers leaking into the prose) without losing the +coherent 2-3 sentence content. + +Length is set via env DVLA_N_EXPLANATION_TOKENS, read by template_v3 at import. +The engine is reloaded per length because the template scaffold changes. +""" +import json +import os +import subprocess +import sys + +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +LENGTHS = [100, 72, 56] + + +def main(): + out = {} + for n in LENGTHS: + env = dict(os.environ, DVLA_N_EXPLANATION_TOKENS=str(n)) + # Run the per-length worker in a fresh process so the module-level + # constant + engine are rebuilt cleanly. + res = subprocess.run( + [sys.executable, f"{ROOT}/scripts/_explen_worker.py"], + env=env, capture_output=True, text=True, + ) + print(res.stdout[-4000:]) + if res.returncode != 0: + print("STDERR:", res.stderr[-2000:]) + + +if __name__ == "__main__": + main() diff --git a/scripts/sweep_explanation_quality.py b/scripts/sweep_explanation_quality.py new file mode 100644 index 0000000..511c4d1 --- /dev/null +++ b/scripts/sweep_explanation_quality.py @@ -0,0 +1,84 @@ +"""Sweep steps_per_chunk and rep_penalty to see if explanation quality +improves. Loads the SGLang engine ONCE, runs several configs on a few +representative Waymo samples, and dumps the explanation field for each. +""" +import json +import os +import sys + +ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +sys.path.insert(0, ROOT) +sys.path.insert(0, os.path.join(ROOT, "eval")) + +from eval.template_v3 import build_prompt_v3 + +DATA = "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/dvlm/dvlm-ad_waymo_e2e_val_cot.json" +PATH_FIX = ("/weka/home/xliu316/", "/weka/home/ext-yingzima/") +# A subset of the picked-5 that showed the worst explanation collapse. +SAMPLE_INDICES = [281, 327, 14] # GO_LEFT slow, GO_STRAIGHT highway, GO_STRAIGHT slow + +# (label, kwargs) +CONFIGS = [ + ("baseline steps=4 rep=2.0", dict(steps_per_chunk=4, rep_penalty=2.0)), + ("steps=8 rep=2.0", dict(steps_per_chunk=8, rep_penalty=2.0)), + ("steps=16 rep=2.0", dict(steps_per_chunk=16, rep_penalty=2.0)), + ("steps=8 rep=4.0", dict(steps_per_chunk=8, rep_penalty=4.0)), + ("steps=16 rep=4.0", dict(steps_per_chunk=16, rep_penalty=4.0)), +] + + +def _fix(p): + return p.replace(PATH_FIX[0], PATH_FIX[1]) + + +def _explanation(text): + try: + return json.loads(text, strict=False).get("explanation", "") + except Exception as e: + return f"" + + +def main(): + algorithm = sys.argv[1] if len(sys.argv) > 1 else "mdm" + data = json.load(open(DATA)) + samples = [data[i] for i in SAMPLE_INDICES] + + from eval.loaders import fast_dvlm_sglang_v3 as loader + print(f"Loading engine (algorithm={algorithm})...") + bundle = loader.load(algorithm=algorithm) + + # Warmup once. + s0 = samples[0] + loader.generate(bundle, [_fix(s0["image"][1])], build_prompt_v3(s0), + temperature=0.0, nav_command=s0["navigation_command"]) + + results = {} + for si, sample in enumerate(samples): + img = _fix(sample["image"][1]) + prompt = build_prompt_v3(sample) + nav = sample["navigation_command"] + sid = sample["sample_id"][:13] + results[sid] = {"nav": nav} + print("\n" + "#" * 92) + print(f"# SAMPLE {sid} nav={nav}") + print("#" * 92) + for label, kw in CONFIGS: + text, lat = loader.generate( + bundle, [img], prompt, temperature=0.0, + nav_command=nav, **kw, + ) + exp = _explanation(text) + results[sid][label] = {"explanation": exp, "latency_s": lat} + print(f"\n--- {label} ({lat:.2f}s) ---") + print(exp) + + out_path = f"{ROOT}/results/waymo_5_compare/explanation_sweep_{algorithm}.json" + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "w") as f: + json.dump(results, f, indent=2, ensure_ascii=False) + print(f"\nSaved {out_path}") + loader.shutdown(bundle) + + +if __name__ == "__main__": + main() From 40cf48dddeb2324292626c69ef84e44c25dbdb95 Mon Sep 17 00:00:00 2001 From: ext-yingzima Date: Thu, 28 May 2026 21:40:32 -0400 Subject: [PATCH 4/5] scripts: auto-derive ROOT from __file__ instead of hardcoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ROOT = dirname(dirname(abspath(__file__))) — the repo root is the parent of scripts/, so the scripts run from any checkout location without editing the path. Replaces the hardcoded /weka/.../dVLA-AD[-ad4fcc21] literal in all 21 scripts (added `import os` where it was missing). External resource paths (the Waymo val json, Fast_dVLM_3B weights) stay hardcoded — they aren't part of the repo. Co-Authored-By: Claude Opus 4.8 --- scripts/_ar_ab_worker.py | 2 +- scripts/_explen_worker.py | 2 +- scripts/ade_comparison.py | 2 +- scripts/compare_latency_sglang_vs_transformers.py | 2 +- scripts/explanation_diversity_report.py | 2 +- scripts/framework_benchmark.py | 2 +- scripts/run_10_waymo_compare.py | 2 +- scripts/run_5_waymo_fastdvlm.py | 2 +- scripts/run_5_waymo_sglang.py | 2 +- scripts/run_5_waymo_sglang_v3.py | 2 +- scripts/run_n_waymo_ade.py | 2 +- scripts/run_test_image.py | 2 +- scripts/save_longtail_examples.py | 2 +- scripts/smoke_sglang_v3.py | 2 +- scripts/sweep_explanation_length.py | 2 +- scripts/sweep_explanation_quality.py | 2 +- scripts/sweep_fastdvlm_configs.py | 2 +- scripts/sweep_sglang_steps.py | 2 +- scripts/write_10_compare_report.py | 2 +- scripts/write_4way_compare_report.py | 3 ++- scripts/write_template_vs_freeform_report.py | 3 ++- 21 files changed, 23 insertions(+), 21 deletions(-) diff --git a/scripts/_ar_ab_worker.py b/scripts/_ar_ab_worker.py index 174b986..cde7c82 100644 --- a/scripts/_ar_ab_worker.py +++ b/scripts/_ar_ab_worker.py @@ -1,7 +1,7 @@ """A/B: explanation parallel top-K vs strict L2R/AR. Same samples, print explanation + latency for each mode to see if L2R removes the glue artifacts.""" import json, os, sys -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT); sys.path.insert(0, os.path.join(ROOT, "eval")) from eval.template_v3 import build_prompt_v3 DATA = "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/dvlm/dvlm-ad_waymo_e2e_val_cot.json" diff --git a/scripts/_explen_worker.py b/scripts/_explen_worker.py index f99fa9b..41935d1 100644 --- a/scripts/_explen_worker.py +++ b/scripts/_explen_worker.py @@ -5,7 +5,7 @@ import os import sys -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/ade_comparison.py b/scripts/ade_comparison.py index 86325d9..2fd3011 100644 --- a/scripts/ade_comparison.py +++ b/scripts/ade_comparison.py @@ -11,7 +11,7 @@ import os import sys -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, os.path.join(ROOT, "eval")) from template_v3 import parse_filled diff --git a/scripts/compare_latency_sglang_vs_transformers.py b/scripts/compare_latency_sglang_vs_transformers.py index 4d18fcf..925071d 100644 --- a/scripts/compare_latency_sglang_vs_transformers.py +++ b/scripts/compare_latency_sglang_vs_transformers.py @@ -28,7 +28,7 @@ import sys import traceback -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/explanation_diversity_report.py b/scripts/explanation_diversity_report.py index 6575cd0..4c9c799 100644 --- a/scripts/explanation_diversity_report.py +++ b/scripts/explanation_diversity_report.py @@ -20,7 +20,7 @@ import re from collections import Counter -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) OUT_DIR = os.path.join(ROOT, "results", "waymo_10_compare") diff --git a/scripts/framework_benchmark.py b/scripts/framework_benchmark.py index 7695063..ddbdbaa 100644 --- a/scripts/framework_benchmark.py +++ b/scripts/framework_benchmark.py @@ -15,7 +15,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_10_waymo_compare.py b/scripts/run_10_waymo_compare.py index 149dd0f..01ca3c0 100644 --- a/scripts/run_10_waymo_compare.py +++ b/scripts/run_10_waymo_compare.py @@ -19,7 +19,7 @@ import time import traceback -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_5_waymo_fastdvlm.py b/scripts/run_5_waymo_fastdvlm.py index 8048f7f..80ae3b3 100644 --- a/scripts/run_5_waymo_fastdvlm.py +++ b/scripts/run_5_waymo_fastdvlm.py @@ -9,7 +9,7 @@ import time import traceback -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_5_waymo_sglang.py b/scripts/run_5_waymo_sglang.py index 75f77d6..f290c24 100644 --- a/scripts/run_5_waymo_sglang.py +++ b/scripts/run_5_waymo_sglang.py @@ -10,7 +10,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_5_waymo_sglang_v3.py b/scripts/run_5_waymo_sglang_v3.py index 315854e..f7b83be 100644 --- a/scripts/run_5_waymo_sglang_v3.py +++ b/scripts/run_5_waymo_sglang_v3.py @@ -7,7 +7,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_n_waymo_ade.py b/scripts/run_n_waymo_ade.py index a191a45..60bd201 100644 --- a/scripts/run_n_waymo_ade.py +++ b/scripts/run_n_waymo_ade.py @@ -12,7 +12,7 @@ import sys import traceback -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/run_test_image.py b/scripts/run_test_image.py index 95c936f..893fe03 100644 --- a/scripts/run_test_image.py +++ b/scripts/run_test_image.py @@ -9,7 +9,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/save_longtail_examples.py b/scripts/save_longtail_examples.py index efb8c63..e8d0f97 100644 --- a/scripts/save_longtail_examples.py +++ b/scripts/save_longtail_examples.py @@ -8,7 +8,7 @@ import shutil import sys -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/smoke_sglang_v3.py b/scripts/smoke_sglang_v3.py index be77984..d679365 100644 --- a/scripts/smoke_sglang_v3.py +++ b/scripts/smoke_sglang_v3.py @@ -5,7 +5,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/sweep_explanation_length.py b/scripts/sweep_explanation_length.py index 1dd2c14..d0445c6 100644 --- a/scripts/sweep_explanation_length.py +++ b/scripts/sweep_explanation_length.py @@ -11,7 +11,7 @@ import subprocess import sys -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) LENGTHS = [100, 72, 56] diff --git a/scripts/sweep_explanation_quality.py b/scripts/sweep_explanation_quality.py index 511c4d1..8bc5ec6 100644 --- a/scripts/sweep_explanation_quality.py +++ b/scripts/sweep_explanation_quality.py @@ -6,7 +6,7 @@ import os import sys -ROOT = "/weka/home/ext-yingzima/dVLA-AD-ad4fcc21" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/sweep_fastdvlm_configs.py b/scripts/sweep_fastdvlm_configs.py index dc7d188..0b75b3a 100644 --- a/scripts/sweep_fastdvlm_configs.py +++ b/scripts/sweep_fastdvlm_configs.py @@ -6,7 +6,7 @@ import sys import time -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/sweep_sglang_steps.py b/scripts/sweep_sglang_steps.py index 55b8356..a56b12e 100644 --- a/scripts/sweep_sglang_steps.py +++ b/scripts/sweep_sglang_steps.py @@ -6,7 +6,7 @@ import os import sys -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "eval")) diff --git a/scripts/write_10_compare_report.py b/scripts/write_10_compare_report.py index 90ec757..f33a48a 100644 --- a/scripts/write_10_compare_report.py +++ b/scripts/write_10_compare_report.py @@ -5,7 +5,7 @@ import os import re -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) OUT_DIR = os.path.join(ROOT, "results", "waymo_10_compare") LONG_VALID = {("speed", "up"), ("slow", "down"), ("keep", "speed"), ("stop", "now")} diff --git a/scripts/write_4way_compare_report.py b/scripts/write_4way_compare_report.py index 4ed0e0c..d07bc08 100644 --- a/scripts/write_4way_compare_report.py +++ b/scripts/write_4way_compare_report.py @@ -2,9 +2,10 @@ Fast-dVLM SGLang (free-form gen) vs dVLM-AD. """ import json +import os import re -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) def _field(txt, key): diff --git a/scripts/write_template_vs_freeform_report.py b/scripts/write_template_vs_freeform_report.py index 89e0399..9b86de3 100644 --- a/scripts/write_template_vs_freeform_report.py +++ b/scripts/write_template_vs_freeform_report.py @@ -3,9 +3,10 @@ quality does NOT regress vs free-form. """ import json +import os import re -ROOT = "/weka/home/ext-yingzima/dVLA-AD" +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # repo root (parent of scripts/) def _field(txt, key): From 389429fe564b824bbad87d1fe3431364bf34aaff Mon Sep 17 00:00:00 2001 From: ext-yingzima Date: Thu, 28 May 2026 22:10:13 -0400 Subject: [PATCH 5/5] results: add 5/10-sample eval outputs (gitignored, force-added) Final tuned-pipeline outputs for record: - waymo_5_compare/fast_dvlm_sglang_v3_mdm.json : 5-sample, L2R + nav gate + rep_penalty=0 (lateral 5/5, schema 12/12, no glue). - waymo_5_compare/fast_dvlm_raw.json : transformers-loader compare. - waymo_5_compare/explanation_sweep_mdm.json : steps x rep sweep. - waymo_10_compare/sglang.json : 10-sample (lateral 10/10, schema 10/10, 0 glue, ~3.0s). Co-Authored-By: Claude Opus 4.8 --- results/waymo_10_compare/sglang.json | 322 ++++++++++++++++++ .../explanation_sweep_mdm.json | 71 ++++ results/waymo_5_compare/fast_dvlm_raw.json | 172 ++++++++++ .../fast_dvlm_sglang_v3_mdm.json | 167 +++++++++ 4 files changed, 732 insertions(+) create mode 100644 results/waymo_10_compare/sglang.json create mode 100644 results/waymo_5_compare/explanation_sweep_mdm.json create mode 100644 results/waymo_5_compare/fast_dvlm_raw.json create mode 100644 results/waymo_5_compare/fast_dvlm_sglang_v3_mdm.json diff --git a/results/waymo_10_compare/sglang.json b/results/waymo_10_compare/sglang.json new file mode 100644 index 0000000..57275f3 --- /dev/null +++ b/results/waymo_10_compare/sglang.json @@ -0,0 +1,322 @@ +[ + { + "sample_id": "3721a68c3a5f6b637f192a2bc4c05824-148", + "nav": "GO_LEFT", + "speed": 0.000976718973026637, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/3721a68c3a5f6b637f192a2bc4c05824/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.0, + 0.0 + ], + [ + 0.0, + 0.001953125 + ], + [ + 0.0, + 0.001953125 + ], + [ + 0.0, + 0.001953125 + ], + [ + 0.0, + 0.001953125 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"black car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic flow\", \"construction\": \"none zone\", \"traffic_element\": \"clear street\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a clear street with palm trees lining the sides. There are no visible hazards or construction. The weather is overcast, and the traffic is calm with no vehicles or pedestrians in the immediate vicinity. The driver's instruction is to go left, and current speed is zero. The road is clear, and there are no signs of traffic congestion or accidents. The driver should maintain a safe speed and proceed with caution the next 5 seconds. The driver's action should be to keep speed.\", \"navigation_command\": \"GO_LEFT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"turn left\"}, \"trajectory\": \"0.5s: forward=+00.0m, lateral=+00.3m\n1.0s: forward=+00.0m, lateral=+00.5m\n1.5s: forward=+00.0m, lateral=+00.8m\n2.0s: forward=+00.0m, lateral=+01.1m\n2.5s: forward=+00.0m, lateral=+01.4m\n3.0s: forward=+00.0m, lateral=+01.7m\n3.5s: forward=+00.0m, lateral=+02.0m\n4.0s: forward=+00.0m, lateral=+02.3m\n4.5s: forward=+00.0m, lateral=+02.6m\n5.0s: forward=+00.0m, lateral=+02.9m\"}", + "latency_s": 2.8689310550689697 + } + }, + { + "sample_id": "cab6d3c2e5e831fa07c7ff1724bb72aa-148", + "nav": "GO_LEFT", + "speed": 2.0904299042107977, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/cab6d3c2e5e831fa07c7ff1724bb72aa/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.50341796875, + 0.1982421875 + ], + [ + 0.970703125, + 0.451171875 + ], + [ + 1.39794921875, + 0.7490234375 + ], + [ + 1.776123046875, + 1.0771484375 + ], + [ + 2.096435546875, + 1.423828125 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white truck\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic flow\", \"construction\": \"none zone\", \"traffic_element\": \"street street\", \"weather_condition\": \"dark sky\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"no vehicle\", \"animal\": \"no traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"no traffic\", \"door_opening_vehicle\": \"no vehicle\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a dark street with a white truck driving ahead. There are no pedestrians, cyclists,, construction zones, traffic hazards, emergency vehicles, animals, or other vehicles in the immediate vicinity. The weather is dark, and the road appears to be clear no signs of traffic or pedestrians. The driver's instruction is to go left, but the current speed and surroundings suggest that the driver should maintain a steady speed and left turn. The driver should also be cautious of the white truck in front.\", \"navigation_command\": \"GO_LEFT\", \"future_meta_behavior\": {\"longitudinal\": \"keep up\", \"lateral\": \"turn left\"}, \"trajectory\": \"0.5s: forward=+01.5m, lateral=+00.3m\n1.0s: forward=+02.0m, lateral=+00.5m\n1.5s: forward=+03.0m, lateral=+00.8m\n2.0s: forward=+04.0m, lateral=+01.0m\n2.5s: forward=+05.0m, lateral=+01.2m\n3.0s: forward=+06.0m, lateral=+01.5m\n3.5s: forward=+07.0m, lateral=+01.8m\n4.0s: forward=+08.0m, lateral=+02.0m\n4.5s: forward=+09.0m, lateral=+02.2m\n5.0s: forward=+10.0m, lateral=+02.4m\"}", + "latency_s": 3.0314903259277344 + } + }, + { + "sample_id": "a45e0ced7d86d8413f0b830677bb6c8a-149", + "nav": "GO_RIGHT", + "speed": 0.24336931341619708, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/a45e0ced7d86d8413f0b830677bb6c8a/149_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.0888671875, + -0.0048828125 + ], + [ + 0.263671875, + -0.0166015625 + ], + [ + 0.5380859375, + -0.041015625 + ], + [ + 0.9150390625, + -0.0849609375 + ], + [ + 1.40966796875, + -0.169921875 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white car\", \"pedestrian\": \"none person\", \"cyclist\": \" cyclist cyclist\", \"construction\": \"none zone\", \"traffic_element\": \"street light\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none vehicle\", \"animal\": \"none animal\", \"special_vehicle\": \"none vehicle\", \"conflicting_vehicle\": \"none vehicle\", \"door_opening_vehicle\": \"none vehicle\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a quiet residential street with a white car approaching from the right. The sky is over, and there are no visible hazards or construction zones. The driver's instruction is to go right, and the current speed is low, so the driver should keep speed and maintain the right lane. The white car is approaching from the right, and the driver should prepare to turn right to avoid a potential collision. The traffic is calm, and there are no pedestrians or cyclists in the path. However,\", \"navigation_command\": \"GO_RIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"turn right\"}, \"trajectory\": \"0.5s: forward=+00.5m, lateral=+00.3m\n1.0s: forward=+00.2m, lateral=+00.5m\n1.5s: forward=+00.1m, lateral=+00.8m\n2.0s: forward=+00.2m, lateral=+01.3m\n2.5s: forward=+00.3m, lateral=+02.0m\n3.0s: forward=+00.4m, lateral=+02.8m\n3.5s: forward=+00.5m, lateral=+03.6m\n4.0s: forward=+00.6m, lateral=+04.4m\n4.5s: forward=+00.7m, lateral=+05.2m\n5.0s: forward=+00.8m, lateral=+06.0m\"}", + "latency_s": 3.0487895011901855 + } + }, + { + "sample_id": "a32a2ff0ab66bfa7bb0c546fe4b684db-148", + "nav": "GO_RIGHT", + "speed": 3.462490582811155, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/a32a2ff0ab66bfa7bb0c546fe4b684db/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.8828125, + -0.140625 + ], + [ + 1.781005859375, + -0.36328125 + ], + [ + 2.690185546875, + -0.68359375 + ], + [ + 3.5927734375, + -1.1015625 + ], + [ + 4.487060546875, + -1.630859375 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white car\", \"pedestrian\": \"none found\", \"cyclist\": \" found here\", \"construction\": \"none found\", \"traffic_element\": \"none street\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none detected\", \"emergency_vehicle\": \"not seen\", \"animal\": \"none seen\", \"special_vehicle\": \"no bus\", \"conflicting_vehicle\": \"no seen\", \"door_opening_vehicle\": \"no seen\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a quiet residential street with a large tree in the middle of the road. There are vehicles parked on the side of the road, and a white car is approaching from the opposite direction. The weather is overcast, and there are no signs of or hazards in the immediate vicinity. The road appears to be clear, and the traffic is calm. The driver's instruction to go right is straightforward, and the should be able to maintain a safe speed and direction. The\", \"navigation_command\": \"GO_RIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep up\", \"lateral\": \"turn right\"}, \"trajectory\": \"0.5s: forward=+01.5m, lateral=+00.3m\n1.0s: forward=+02.7m, lateral=+00.5m\n1.5s: forward=+04.0m, lateral=+00.7m\n2.0s: forward=+05.3m, lateral=+00.9m\n2.5s: forward=+07.0m, lateral=+01.1m\n3.0s: forward=+09.7m, lateral=+01.3m\n3.5s: forward=+12.4m, lateral=+01.5m\n4.0s: forward=+15.1m, lateral=+01.7m\n4.5s: forward=+18.0m, lateral=+01.9m\n5.0s: forward=+21.0m, lateral=+02.1m\"}", + "latency_s": 3.0257208347320557 + } + }, + { + "sample_id": "e3e6244899ebdbf1a0d0d45fc513b34c-149", + "nav": "GO_STRAIGHT", + "speed": 0.0, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/e3e6244899ebdbf1a0d0d45fc513b34c/149_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.00048828125, + 0.0 + ], + [ + 0.00048828125, + 0.0 + ], + [ + 0.00048828125, + 0.0 + ], + [ + 0.00048828125, + 0.0 + ], + [ + 0.00048828125, + 0.0 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"red car\", \"pedestrian\": \"walking people\", \"cyclist\": \"ists nearby\", \"construction\": \"construction zone\", \"traffic_element\": \"traffic light\", \"weather_condition\": \"overcast\", \"road_hazard\": \"po debris\", \"emergency_vehicle\": \"emergency vehicle\", \"animal\": \"dogs nearby\", \"special_vehicle\": \"service vehicle\", \"conflicting_vehicle\": \"other cars\", \"door_opening_vehicle\": \"open door\"}, \"complexity\": \"complex\", \"explanation\": \"The scene shows a busy intersection with multiple vehicles and pedestrians. A red car is approaching from the left and is the closest vehicle to the\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep up\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+00.0m, lateral=+00.0m\n1.0s: forward=+00.0m, lateral=+00.0m\n1.5s: forward=+00.0m, lateral=+00.0m\n2.0s: forward=+00.0m, lateral=+00.0m\n2.5s: forward=+00.0m, lateral=+00.0m\n3.0s: forward=+00.0m, lateral=+00.0m\n3.5s: forward=+00.0m, lateral=+00.0m\n4.0s: forward=+00.0m, lateral=+00.0m\n4.5s: forward=+00.0m, lateral=+00.0m\n5.0s: forward=+00.0m, lateral=+00.0m\"}", + "latency_s": 3.0136749744415283 + } + }, + { + "sample_id": "4b390f75983c892cbb4144d5af260054-148", + "nav": "GO_STRAIGHT", + "speed": 0.06507265576377975, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/4b390f75983c892cbb4144d5af260054/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + -0.008056640625, + -0.0003662109375 + ], + [ + 0.0631103515625, + -0.002197265625 + ], + [ + 0.2994384765625, + -0.003662109375 + ], + [ + 0.7061767578125, + -0.0057373046875 + ], + [ + 1.2626953125, + -0.008056640625 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"black car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic flow\", \"construction\": \"none zone\", \"traffic_element\": \"street light\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a quiet residential street at night with a few cars and street lights. The driver is to go straight, and the current speed is very low. There are no immediate hazards or complex intersections. The driver should maintain a steady speed and keep the car the same lane as the traffic flow. The road is clear, and the weather is overcast, which is typical for a night drive in a residential area. driver's action should be to maintain the current speed and keep the car straight ahead\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+00.0m, lateral=+00.0m\n1.0s: forward=+00.0m, lateral=+00.0m\n1.5s: forward=+00.0m, lateral=+00.0m\n2.0s: forward=+00.0m, lateral=+00.0m\n2.5s: forward=+00.1m, lateral=+00.0m\n3.0s: forward=+00.2m, lateral=+00.0m\n3.5s: forward=+00.3m, lateral=+00.0m\n4.0s: forward=+00.4m, lateral=+00.0m\n4.5s: forward=+00.5m, lateral=+00.0m\n5.0s: forward=+00.6m, lateral=+00.0m\"}", + "latency_s": 3.0368337631225586 + } + }, + { + "sample_id": "e71a85db89e7ec7b551629c95a33d29f-149", + "nav": "GO_STRAIGHT", + "speed": 4.194031731034686, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/e71a85db89e7ec7b551629c95a33d29f/149_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 1.02197265625, + -0.00244140625 + ], + [ + 1.9794921875, + -0.005859375 + ], + [ + 2.86279296875, + -0.0087890625 + ], + [ + 3.662109375, + -0.01220703125 + ], + [ + 4.37890625, + -0.01611328125 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"black car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic flow\", \"construction\": \"none zone\", \"traffic_element\": \"clear street\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a clear, residential street with a few parked cars on the side. The traffic is, for the most part, calm and there are no immediate hazards or construction zones. The weather is overcast, and there are no pedestrians, cyclists, emergency, animal, or special vehicles in the immediate vicinity. The driver's instruction is to go straight, and the current speed and acceleration are suitable for maintaining a steady. The road is clear, and there are no immediate obstacles or hazards to avoid\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+00.0m, lateral=+00.0m\n1.0s: forward=+04.1m, lateral=+00.0m\n1.5s: forward=+08.2m, lateral=+00.0m\n2.0s: forward=+12.3m, lateral=+00.0m\n2.5s: forward=+16.4m, lateral=+00.0m\n3.0s: forward=+20.5m, lateral=+00.0m\n3.5s: forward=+24.6m, lateral=+00.0m\n4.0s: forward=+88.7m, lateral=+00.0m\n4.5s: forward=+12.8m, lateral=+00.0m\n5.0s: forward=+16.9m, lateral=+00.0m\"}", + "latency_s": 3.0126118659973145 + } + }, + { + "sample_id": "567443cab762895c5a05acc838764f96-149", + "nav": "GO_STRAIGHT", + "speed": 7.170275011833318, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/567443cab762895c5a05acc838764f96/149_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 1.798828125, + -0.0162353515625 + ], + [ + 3.610595703125, + -0.04296875 + ], + [ + 5.441650390625, + -0.0753173828125 + ], + [ + 7.28955078125, + -0.1048583984375 + ], + [ + 9.161865234375, + -0.1309814453125 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"silver car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic traffic\", \"construction\": \"none traffic\", \"traffic_element\": \"power lines\", \"weather_condition\": \"clear sky\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a multi-view image of a residential street with parked cars on both sides. The sky clear and power lines overhead, indicating a calm and safe driving environment. There are no visible hazards or pedestrians, and the road appears to be clear of any obstacles The driver's instruction is to go straight, and the current speed is 7.2 m/s, which is within the cruising range. The driver should maintain current speed and keep the lane straight to avoid any potential hazards or unexpected road conditions\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+05.5m, lateral=+00.0m\n1.0s: forward=+07.1m, lateral=+00.0m\n1.5s: forward=+17.9m, lateral=+00.0m\n2.0s: forward=+35.8m, lateral=+00.0m\n2.5s: forward=+35.8m, lateral=+00.0m\n3.0s: forward=+35.8m, lateral=+00.0m\n3.5s: forward=+35.8m, lateral=+00.0m\n4.0s: forward=+55.7m, lateral=+00.0m\n4.5s: forward=+55.8m, lateral=+00.0m\n5.0s: forward=+55.8m, lateral=+00.0m\"}", + "latency_s": 3.0101053714752197 + } + }, + { + "sample_id": "6567b0c95aae8192daea6207ca30f5bd-147", + "nav": "GO_STRAIGHT", + "speed": 28.844636319188943, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/6567b0c95aae8192daea6207ca30f5bd/147_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 7.174560546875, + -0.009765625 + ], + [ + 14.248779296875, + -0.03466796875 + ], + [ + 21.20263671875, + -0.07421875 + ], + [ + 28.029541015625, + -0.116455078125 + ], + [ + 34.690673828125, + -0.15087890625 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white truck\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic traffic\", \"construction\": \"none zone\", \"traffic_element\": \"highway\", \"weather_condition\": \"clear clear\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a clear highway with a few vehicles, including a white truck in the left lane. road conditions are clear, and there are no immediate hazards or construction zones. The driver's instruction is to go straight, and the current speed and\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+24.8m, lateral=+00.0m\n1.0s: forward=+28.8m, lateral=+00.0m\n1.5s: forward=+42.8m, lateral=+00.0m\n2.0s: forward=+56.8m, lateral=+00.0m\n2.5s: forward=+70.8m, lateral=+00.0m\n3.0s: forward=+84.8m, lateral=+00.0m\n3.5s: forward=+98.8m, lateral=+00.0m\n4.0s: forward=+28.8m, lateral=+00.0m\n4.5s: forward=+12.8m, lateral=+00.0m\n5.0s: forward=+00.8m, lateral=+00.0m\"}", + "latency_s": 3.0128543376922607 + } + }, + { + "sample_id": "4714de6823f14ec054f617bdcfcb83c1-148", + "nav": "GO_STRAIGHT", + "speed": 15.160633008226245, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/4714de6823f14ec054f617bdcfcb83c1/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 3.795654296875, + 0.03125 + ], + [ + 7.598388671875, + 0.11328125 + ], + [ + 11.376953125, + 0.25390625 + ], + [ + 15.071533203125, + 0.4541015625 + ], + [ + 18.635986328125, + 0.7080078125 + ] + ], + "sglang_template": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"black car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic flow\", \"construction\": \"none zone\", \"traffic_element\": \"traffic light\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a multi-lane road with multiple lanes of traffic. The traffic light is visible, the weather is overcast, and there are no signs of construction or emergency vehicles. The road appears to be clear of any significant hazards, and the traffic is calm and predictable. The driver's instruction is to go straight, and the current speed is 15.2 m/s, which is within the cruising speed. There are no immediate hazards or red lights, so the driver should maintain speed\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+15.2m, lateral=+00.0m\n1.0s: forward=+35.1m, lateral=+00.0m\n1.5s: forward=+55.9m, lateral=+00.0m\n2.0s: forward=+75.8m, lateral=+00.0m\n2.5s: forward=+95.7m, lateral=+00.0m\n3.0s: forward=+11.5m, lateral=+00.0m\n3.5s: forward=+31.4m, lateral=+00.0m\n4.0s: forward=+11.5m, lateral=+00.0m\n4.5s: forward=+31.4m, lateral=+00.0m\n5.0s: forward=+11.5m, lateral=+00.0m\"}", + "latency_s": 3.00604248046875 + } + } +] \ No newline at end of file diff --git a/results/waymo_5_compare/explanation_sweep_mdm.json b/results/waymo_5_compare/explanation_sweep_mdm.json new file mode 100644 index 0000000..6f340df --- /dev/null +++ b/results/waymo_5_compare/explanation_sweep_mdm.json @@ -0,0 +1,71 @@ +{ + "9ac9e56e4ac9c": { + "nav": "GO_LEFT", + "baseline steps=4 rep=2.0": { + "explanation": "The scene shows a white brick wall with a parking, street. and the driver is driving red car the speed left instruction to The instructions on the road are go turn lane of The vehicles there that no traffic is no other hazards or pedestrians in so current can future behavior safe action without such as lane lateral forward at 0 mm5 keep speed at 01235 mms lateral right forwards.future trajectory will be (longitudinal: upup, slow down now),", + "latency_s": 1.7391104698181152 + }, + "steps=8 rep=2.0": { + "explanation": "The scene shows a quiet residential street with no empty car, nearby pedestrians, or cyclists. The is visible constructioncoming emergency of other vehicles in the path. There are no traffic hazards such road elements ahead or accidentscoming vehicles that be seen yet. The driver behavior be to on left turn maintaining lane speed and 10 future ego waypoints at 005 s spacing (t = 0.5, 100 ..., ...225s) — one line per waypoint::", + "latency_s": 2.2609972953796387 + }, + "steps=16 rep=2.0": { + "explanation": "The scene shows a quiet street with an empty parking lot no pedestrians or cyclists, construction zones, traffic, road weather hazards. and emergency vehicles. The driver is driving at a safe 12 m/s and the current is ready prepared to make a left turn the there are no immediate hazards in front path ofThe future driving behavior and trajectory: \n\nfuture_meta_behavior.longitudinal:speed up keep speed\n\nfuture_meta.lateral:turn left keep lane\n\n0.5s: forward=+", + "latency_s": 2.4019134044647217 + }, + "steps=8 rep=4.0": { + "explanation": "The scene shows a quiet residential street with no empty car, nearby pedestrians, or cyclists. The is visible constructioncoming emergency of other vehicles in the path. There road hazards hazard yet seen and there weather condition overcast over clear sky conditions for driver's to instruction GOGO_LEFT go left turn.\n\n** predicted future behavior: trajectory = 0.5s: forward=+0213m, lateral=+4968m\n 1..0s: forward=-7", + "latency_s": 2.245486259460449 + }, + "steps=16 rep=4.0": { + "explanation": "The scene shows a quiet street with an empty parking lot no pedestrians or cyclists, and no visible traffic hazards. The driver is driving at the current speed of 1.2 m/s following The Driver instruction to GO_LEFT will there be any obstacles in the way it's car can turn left safely avoid them collision risk is currently not yet seen oncoming road hazard that would require turning right Futurefuture meta_behavior future longitudinal behaviorspeed up speed upFuture future lateral behaviorturn left turnleftFuture ego waypoints", + "latency_s": 2.399885892868042 + } + }, + "f1dd0e328596e": { + "nav": "GO_STRAIGHT", + "baseline steps=4 rep=2.0": { + "explanation": "The scene shows a highway with multiple clear vehicles, an overcast. and no road conditions The There not no nearby hazards pedestrians or cyclistsists, construction zones or emergency vehicle found the. is also traffic of any vehicles are moving in path flow other lane there driving which means is safe going stay on the right side due to that we maintain the current speed and scene context I go straight straight as per driver instruction The future behavior_behavior.longlong longitudinal: speedspeedup up keepkeep speed down nownow", + "latency_s": 1.9131476879119873 + }, + "steps=8 rep=2.0": { + "explanation": "The scene is a clear highway with an over car, no the zone, and no construction zones. is are few nearby of traffic flow including a white car and no pedestrian or in there path The There are any hazards hazard ahead such accident or fire oncoming emergency or blocked debris coming from the road The driver instruction is to go straight which matches current speed AND not hazardous hazard Therefore, will slow down to keep speed for s20.5 s before then speeding up at 219 m", + "latency_s": 2.2391467094421387 + }, + "steps=16 rep=2.0": { + "explanation": "The scene shows a multi highway with clear traffic flow and no visible hazards. The white car is approaching from the left lane, and driver is instructed to go straight. The current speed of 21 m9, which is less than 1 m/s, that the path clear ahead. I speed up to maintain safe driving behavior.\n\n**future_meta_behavior.longlongitudinal**:speed up keepkeep\n\n**\n\n**future_behavior.lateralateral**:keeplane\n\n0.5s: forward=+", + "latency_s": 2.379471778869629 + }, + "steps=8 rep=4.0": { + "explanation": "The scene is a clear highway with an over car, no the zone, and calm traffic flow. are white vehicle on pedestrian or cyclistist in path hazard from any emergencycoming vehicles or animal door opening special vehicle but conflicting agent not none road has that next future will behavior: speed up21 m9 keep lateral 00 tomfuture ego trajectory5 forward=+34.8m laterals ++16.m7sforward=+54.8m laterals++", + "latency_s": 2.2338979244232178 + }, + "steps=16 rep=4.0": { + "explanation": "The scene shows a multi highway with clear traffic flow and no visible hazards. The white car is approaching from the left lane, and driver instruction is to go straight. The current speed of 21.9 m/s suggests that we should keep our speed constant as there are no immediate hazards ahead or in future frames My my planned driving behavior will be to maintain at the same forward while keeping a lateral position near zero I also plan on speeding up slightly due to relatively low cruising but path clear this frame", + "latency_s": 2.3802073001861572 + } + }, + "0f71456c67c9e": { + "nav": "GO_STRAIGHT", + "baseline steps=4 rep=2.0": { + "explanation": "The scene shows a quiet residential street with multiple cars parked on the. no trees or buildings, visible traffic signs and sky sign road vehicles stop sign, there are no is pedestrian pedestrians seen of cyclistscomingists cyclist coming vehicle any emergency hazards debris hazard fire detected in lane path intersection collision imminent path safety object current lateral behavior: stay change keep right now Driver driver instruction go straight straight to car driving ahead left turnlane maintain speed speed upnowstrajectoryfuture forward_speed=05 1._str0", + "latency_s": 1.8933699131011963 + }, + "steps=8 rep=2.0": { + "explanation": "The scene shows a quiet residential street with several parked cars on no parking and some trees. The weather clear sunnyunny, and the traffic is light. There are no pedestrians, or hazards in the road the car can to go now. future instruction GOGO_STIGHIGHT\n\nfuture meta meta behavior:longitudinal speed up speed upnowfuture lateral behavior:keep lane keep lane\n\ntrajectory:\n\n 0s =, lateral=+00nmm", + "latency_s": 2.2319679260253906 + }, + "steps=16 rep=2.0": { + "explanation": "The scene shows a quiet residential street with several parked cars on either side. There are no visible pedestrianscycl cyclists, construction zones, road hazards, emergency vehicles or animals in the the traffic is calm and there. The driver instruction to GO_STRAIGHT is that current speed and path are straight ahead without any obstacles or hazards. future behavior will trajectory will be maintain the current speed and keep to straight lane as avoid any turn change for safety reasons., 0.5s: forward=+01", + "latency_s": 2.3734371662139893 + }, + "steps=8 rep=4.0": { + "explanation": "The scene shows a quiet residential street with several parked cars on no parking and some trees. The weather clear sunnyunny, there the traffic is light and There are vehicles car, pedestrian walking in but road has safe path of hazard hazards such as driver driving vehicle or comingcominging not cause any immediate danger ahead. For I future behavior: GOfuture up maintain left lane while go straight to keep next speed at 05 per/s for Future trajectory: 05s forward=+12m", + "latency_s": 2.211806535720825 + }, + "steps=16 rep=4.0": { + "explanation": "The scene shows a quiet residential street with several parked cars on either side. There are no visible pedestrianscycl cyclists, construction zones or road hazards in The sky is overcast and the weather appears calm and clear. No traffic lights emergency vehicles or animalsor other objects to be hazardous.\n\n The future driving behavior will trajectory: 0.5s: forward=+01.m2m, lateral=+01.mm\n\n 1..Os: forward=-7934mm", + "latency_s": 2.365333080291748 + } + } +} \ No newline at end of file diff --git a/results/waymo_5_compare/fast_dvlm_raw.json b/results/waymo_5_compare/fast_dvlm_raw.json new file mode 100644 index 0000000..244ab67 --- /dev/null +++ b/results/waymo_5_compare/fast_dvlm_raw.json @@ -0,0 +1,172 @@ +[ + { + "sample_id": "9ac9e56e4ac9c630d35ffca845a8114c-149", + "nav": "GO_LEFT", + "speed": 1.1893390244822897, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/9ac9e56e4ac9c630d35ffca845a8114c/149_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.29296875, + 0.106201171875 + ], + [ + 0.574951171875, + 0.232666015625 + ], + [ + 0.845703125, + 0.375244140625 + ], + [ + 1.101806640625, + 0.53271484375 + ], + [ + 1.343017578125, + 0.701416015625 + ] + ], + "fast_dvlm": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"red car\", \"pedestrian\": \"none walking\", \"cyclist\": \" visible visible\", \"construction\": \"none zone\", \"traffic_element\": \"none sign\", \"weather_condition\": \"scast\", \"road_hazard\": \"none debris\", \"emergency_vehicle\": \"none vehicle\", \"animal\": \"none dog\", \"special_vehicle\": \"none truck\", \"conflicting_vehicle\": \"none car\", \"door_opening_vehicle\": \"none car\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a white brick wall with a red, car parked the left side. road The There not no visible pedestrian or in and walking cyclist. on there visiblecycl any zone construction no is any sign hazard of nearby traffic vehicle debris vehicles or coming an emergency seen animals are seencoming animal dog path see special transport vehicles vehicle truck door opening conflictinging other car right go: turning turn instruction GO for the driver to laneleft left change forward lateral keep speed down up down speed lateral 0 keepm\", \"future_meta_behavior\": {\"longitudinal\": \"keep up\", \"lateral\": \"turn left\"}, \"trajectory\": \"0.5s: forward=+00.5m, lateral=+00.3m\n1.0s: forward=+01.1m, lateral=+00.8m\n1.5s: forward=+01.7m, lateral=+01.3m\n2.0s: forward=+02.9m, lateral=+02.4m\n2.5s: forward=+03.1m, lateral=+02.5m\n3.0s: forward=+03.3m, lateral=+02.6m\n3.5s: forward=+04.5m, lateral=+02.7m\n4.0s: forward=+04.7m, lateral=+08.8m\n4.5s: forward=+05.9m, lateral=+02.9m\n5.0s: forward=+06.1m, lateral=+02.0m\"}", + "latency_s": 2.1911680698394775, + "img": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/9ac9e56e4ac9c630d35ffca845a8114c/149_CAM_FRONT.jpg", + "prompt_len": 7894 + } + }, + { + "sample_id": "8f8d0a6d38a096c93301d38ec509e1fd-147", + "nav": "GO_RIGHT", + "speed": 2.825096784326252, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/8f8d0a6d38a096c93301d38ec509e1fd/147_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.70703125, + -0.140625 + ], + [ + 1.431640625, + -0.3427734375 + ], + [ + 2.1689453125, + -0.595703125 + ], + [ + 2.9287109375, + -0.9033203125 + ], + [ + 3.73046875, + -1.2685546875 + ] + ], + "fast_dvlm": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white car\", \"pedestrian\": \"none walking\", \"cyclist\": \" visible visible\", \"construction\": \"orange zone\", \"traffic_element\": \"none cones\", \"weather_condition\": \"s clear\", \"road_hazard\": \"none debris\", \"emergency_vehicle\": \"none vehicle\", \"animal\": \"none dog\", \"special_vehicle\": \"none truck\", \"conflicting_vehicle\": \"none car\", \"door_opening_vehicle\": \"none car\"}, \"complexity\": \"simple\", \"explanation\": \"The scene is a typical residential street with palm clearings and no white vehicles. The the car pedestrian walking cyclist, visible are in there construction zone traffic element presence of an emergency truck cones vehicle or debris animal oncoming hazardous dog or seen vehicle any not road the weather visible hazards hazard as to other agents in path that driving safe within next future:speed speed up speed 2 m go maintain keep left lane change turn right trajectory forwardforward=+.05 forward=< lateral=-10m\", \"future_meta_behavior\": {\"longitudinal\": \"speed up\", \"lateral\": \"change left\"}, \"trajectory\": \"0.5s: forward=+00.5m, lateral=+00.5m\n1.0s: forward=+01.0m, lateral=+00.0m\n1.5s: forward=+02.5m, lateral=+00.0m\n2.0s: forward=+03.0m, lateral=+00.0m\n2.5s: forward=+04.5m, lateral=+00.0m\n3.0s: forward=+05.0m, lateral=+00.0m\n3.5s: forward=+05.5m, lateral=+00.0m\n4.0s: forward=+07.0m, lateral=+00.0m\n4.5s: forward=+08.5m, lateral=+00.0m\n5.0s: forward=+10.0m, lateral=+00.0m\"}", + "latency_s": 1.6317617893218994, + "img": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/8f8d0a6d38a096c93301d38ec509e1fd/147_CAM_FRONT.jpg", + "prompt_len": 7897 + } + }, + { + "sample_id": "f1dd0e328596edd3c4cd977a0d09aef5-148", + "nav": "GO_STRAIGHT", + "speed": 21.944717088411778, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/f1dd0e328596edd3c4cd977a0d09aef5/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 5.49462890625, + -0.0584716796875 + ], + [ + 10.9794921875, + -0.2078857421875 + ], + [ + 16.45751953125, + -0.44207763671875 + ], + [ + 21.927978515625, + -0.76922607421875 + ], + [ + 27.38916015625, + -1.18731689453125 + ] + ], + "fast_dvlm": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white car\", \"pedestrian\": \"none walking\", \"cyclist\": \" visible visible\", \"construction\": \"none zone\", \"traffic_element\": \"green sign\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none lane\", \"emergency_vehicle\": \"none visible\", \"animal\": \"none dog\", \"special_vehicle\": \"none truck\", \"conflicting_vehicle\": \"none car\", \"door_opening_vehicle\": \"none visible\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a highway with multiple nearby white cars passing, the weather is straight ahead and no are hazards visible or road construction zone. lane traffic signs green clear there are no pedestrians vehicles that the car driving to go straight. future meta behavior: speed up lateral keep behavior: change trajectory0.5s s forward=1937m lateral+-205 m laterals=- s453s forwardforward=1687m lateral=-20m\", \"future_meta_behavior\": {\"longitudinal\": \"speed up\", \"lateral\": \"turn left\"}, \"trajectory\": \"0.5s: forward=+19.9m, lateral=+00.0m\n1.0s: forward=+16.9m, lateral=+00.0m\n1.5s: forward=+13.9m, lateral=+00.0m\n2.0s: forward=+13.9m, lateral=+00.0m\n2.5s: forward=+13.9m, lateral=+00.0m\n3.0s: forward=+13.9m, lateral=+00.0m\n3.5s: forward=+19.9m, lateral=+00.0m\n4.0s: forward=+13.9m, lateral=+00.0m\n4.5s: forward=+13.9m, lateral=+00.0m\n5.0s: forward=+13.9m, lateral=+00.0m\"}", + "latency_s": 1.6000337600708008, + "img": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/f1dd0e328596edd3c4cd977a0d09aef5/148_CAM_FRONT.jpg", + "prompt_len": 7836 + } + }, + { + "sample_id": "0f71456c67c9eeff7b8cea4e979e0393-148", + "nav": "GO_STRAIGHT", + "speed": 0.4838117304297617, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/0f71456c67c9eeff7b8cea4e979e0393/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.13671875, + -0.0068359375 + ], + [ + 0.3173828125, + -0.013671875 + ], + [ + 0.5458984375, + -0.025390625 + ], + [ + 0.82421875, + -0.037109375 + ], + [ + 1.15234375, + -0.0537109375 + ] + ], + "fast_dvlm": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"silver car\", \"pedestrian\": \"none walking\", \"cyclist\": \" visible visible\", \"construction\": \"none zone\", \"traffic_element\": \"stop light\", \"weather_condition\": \"scast\", \"road_hazard\": \"none debris\", \"emergency_vehicle\": \"none visible\", \"animal\": \"none dog\", \"special_vehicle\": \"none truck\", \"conflicting_vehicle\": \"none car\", \"door_opening_vehicle\": \"none open\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a quiet residential street with several cars parked on the. no pedestrians or hazards, visible and traffic is light car The road stop light is clearcast and weather are no signs of debriscoming there the behavior I I gogo straight to driving as maintaining current speed at my lane in keep left foring future waypointssforward =+05 m forward1 lateral-m now will0.5 s: ==+02 m forward1 lateral-= +843m\", \"future_meta_behavior\": {\"longitudinal\": \"keep up\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+05.2m, lateral=+00.0m\n1.0s: forward=+05.4m, lateral=+00.0m\n1.5s: forward=+05.6m, lateral=+00.0m\n2.0s: forward=+05.8m, lateral=+00.0m\n2.5s: forward=+06.0m, lateral=+00.0m\n3.0s: forward=+06.2m, lateral=+00.0m\n3.5s: forward=+04.4m, lateral=+00.0m\n4.0s: forward=+06.4m, lateral=+00.0m\n4.5s: forward=+06.6m, lateral=+00.0m\n5.0s: forward=+06.8m, lateral=+00.0m\"}", + "latency_s": 1.5977928638458252, + "img": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/0f71456c67c9eeff7b8cea4e979e0393/148_CAM_FRONT.jpg", + "prompt_len": 7824 + } + }, + { + "sample_id": "14f0b68cf50bad87e90a490a70109cad-149", + "nav": "GO_STRAIGHT", + "speed": 6.572364890510018, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/14f0b68cf50bad87e90a490a70109cad/149_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 1.63525390625, + 0.01171875 + ], + [ + 3.251953125, + 0.025390625 + ], + [ + 4.85498046875, + 0.048828125 + ], + [ + 6.43896484375, + 0.078125 + ], + [ + 8.0087890625, + 0.11328125 + ] + ], + "fast_dvlm": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"black car\", \"pedestrian\": \"none walking\", \"cyclist\": \" walking walking\", \"construction\": \"orange zone\", \"traffic_element\": \"orange cones\", \"weather_condition\": \"scast\", \"road_hazard\": \"none construction\", \"emergency_vehicle\": \"none vehicle\", \"animal\": \"none walking\", \"special_vehicle\": \"none vehicle\", \"conflicting_vehicle\": \"none car\", \"door_opening_vehicle\": \"none vehicle\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a street with construction palm and, no the black car is nearby ahead is. The has cones are clear straight road traffic and there are no pedestrians hazards or hazard pedestrian in oncoming vehicle the driving of to avoid next. frame lane action behavior future be behavior longitudinal stop: up speed down keep forward lateral turn left +05:forward+m150 right2 m, lateral=1+4-3m trajectoryfuture ego waypoints at6 =keep_speeddown m\", \"future_meta_behavior\": {\"longitudinal\": \"speed down\", \"lateral\": \"turn left\"}, \"trajectory\": \"0.5s: forward=+05.2m, lateral=+00.0m\n1.0s: forward=+10.5m, lateral=+00.0m\n1.5s: forward=+15.8m, lateral=+00.0m\n2.0s: forward=+20.8m, lateral=+00.0m\n2.5s: forward=+25.4m, lateral=+00.0m\n3.0s: forward=+30.2m, lateral=+00.0m\n3.5s: forward=+40.8m, lateral=+00.0m\n4.0s: forward=+50.8m, lateral=+00.0m\n4.5s: forward=+60.8m, lateral=+00.0m\n5.0s: forward=+70.8m, lateral=+00.0m\"}", + "latency_s": 1.6360840797424316, + "img": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/14f0b68cf50bad87e90a490a70109cad/149_CAM_FRONT.jpg", + "prompt_len": 7827 + } + } +] \ No newline at end of file diff --git a/results/waymo_5_compare/fast_dvlm_sglang_v3_mdm.json b/results/waymo_5_compare/fast_dvlm_sglang_v3_mdm.json new file mode 100644 index 0000000..d9eea61 --- /dev/null +++ b/results/waymo_5_compare/fast_dvlm_sglang_v3_mdm.json @@ -0,0 +1,167 @@ +[ + { + "sample_id": "9ac9e56e4ac9c630d35ffca845a8114c-149", + "nav": "GO_LEFT", + "speed": 1.1893390244822897, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/9ac9e56e4ac9c630d35ffca845a8114c/149_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.29296875, + 0.106201171875 + ], + [ + 0.574951171875, + 0.232666015625 + ], + [ + 0.845703125, + 0.375244140625 + ], + [ + 1.101806640625, + 0.53271484375 + ], + [ + 1.343017578125, + 0.701416015625 + ] + ], + "fast_dvlm_sglang_v3": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"red car\", \"pedestrian\": \"none yet\", \"cyclist\": \" yet seen\", \"construction\": \"none yet\", \"traffic_element\": \"empty street\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none yet\", \"emergency_vehicle\": \"yet seen\", \"animal\": \"none seen\", \"special_vehicle\": \"yet seen\", \"conflicting_vehicle\": \"yet seen\", \"door_opening_vehicle\": \"yet seen\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a clear, empty street with an empty parking lot and a white brick wall. There no visible hazards, pedestrians, or cyclists. The traffic is minimal, and the weather is overcast. The driver's instruction is to go left, and the's current speed is 1.2 m/s. The driver should maintain their speed and keep their lane as they turn left. The driver should also be cautious any potential hazards that may appear in the next 5 seconds. The driver:\", \"navigation_command\": \"GO_LEFT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"turn left\"}, \"trajectory\": \"0.5s: forward=+00.5m, lateral=+00.3m\n1.0s: forward=+01.1m, lateral=+00.5m\n1.5s: forward=+01.7m, lateral=+01.3m\n2.0s: forward=+02.3m, lateral=+01.8m\n2.5s: forward=+02.9m, lateral=+02.4m\n3.0s: forward=+03.5m, lateral=+02.0m\n3.5s: forward=+04.1m, lateral=+02.6m\n4.0s: forward=+04.7m, lateral=+03.2m\n4.5s: forward=+05.3m, lateral=+03.8m\n5.0s: forward=+06.0m, lateral=+04.5m\"}", + "latency_s": 2.8598246574401855, + "algorithm": "mdm" + } + }, + { + "sample_id": "8f8d0a6d38a096c93301d38ec509e1fd-147", + "nav": "GO_RIGHT", + "speed": 2.825096784326252, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/8f8d0a6d38a096c93301d38ec509e1fd/147_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.70703125, + -0.140625 + ], + [ + 1.431640625, + -0.3427734375 + ], + [ + 2.1689453125, + -0.595703125 + ], + [ + 2.9287109375, + -0.9033203125 + ], + [ + 3.73046875, + -1.2685546875 + ] + ], + "fast_dvlm_sglang_v3": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white truck\", \"pedestrian\": \"none found\", \"cyclist\": \" found found\", \"construction\": \"construction zone\", \"traffic_element\": \"none found\", \"weather_condition\": \"scast\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none found\", \"animal\": \"none found\", \"special_vehicle\": \"none found\", \"conflicting_vehicle\": \"none found\", \"door_opening_vehicle\": \"none found\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a clear residential street with a few parked cars and a few palm trees. There are pedestrians and cyclists in the vicinity, but they are not interacting with the vehicle. The weather is sunny and clear, and there are no road hazards or construction zones The driver's instruction is to go right, and the current speed is 2.8 m/s. The vehicle is currently in the left lane, and the is no immediate hazard ahead The driver's action should be to maintain the current speed\", \"navigation_command\": \"GO_RIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"turn right\"}, \"trajectory\": \"0.5s: forward=+01.5m, lateral=+00.3m\n1.0s: forward=+02.8m, lateral=+00.5m\n1.5s: forward=+04.2m, lateral=+00.7m\n2.0s: forward=+05.6m, lateral=+00.9m\n2.5s: forward=+07.0m, lateral=+01.1m\n3.0s: forward=+09.4m, lateral=+01.3m\n3.5s: forward=+11.8m, lateral=+01.5m\n4.0s: forward=+14.2m, lateral=+01.7m\n4.5s: forward=+16.6m, lateral=+01.9m\n5.0s: forward=+19.0m, lateral=+02.1m\"}", + "latency_s": 3.0496411323547363, + "algorithm": "mdm" + } + }, + { + "sample_id": "f1dd0e328596edd3c4cd977a0d09aef5-148", + "nav": "GO_STRAIGHT", + "speed": 21.944717088411778, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/f1dd0e328596edd3c4cd977a0d09aef5/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 5.49462890625, + -0.0584716796875 + ], + [ + 10.9794921875, + -0.2078857421875 + ], + [ + 16.45751953125, + -0.44207763671875 + ], + [ + 21.927978515625, + -0.76922607421875 + ], + [ + 27.38916015625, + -1.18731689453125 + ] + ], + "fast_dvlm_sglang_v3": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic flow\", \"construction\": \"none zone\", \"traffic_element\": \"high flow\", \"weather_condition\": \"overcast\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a multi-lane highway with a clear path ahead. A white car is driving in lane 1, and there are no visible hazards or construction zones. The weather is overcast, and the traffic flow appears to be steady. The driver's is to go straight, and the current speed is 21.9 m/s, which is within the cruising range. The road is clear, and there no immediate hazards or conflicts. The driver's instruction is to go straight, so\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+21.9m, lateral=+00.0m\n1.0s: forward=+21.9m, lateral=+00.0m\n1.5s: forward=+21.9m, lateral=+00.0m\n2.0s: forward=+21.9m, lateral=+00.0m\n2.5s: forward=+21.9m, lateral=+00.0m\n3.0s: forward=+21.9m, lateral=+00.0m\n3.5s: forward=+21.9m, lateral=+00.0m\n4.0s: forward=+11.9m, lateral=+00.0m\n4.5s: forward=+11.9m, lateral=+00.0m\n5.0s: forward=+11.9m, lateral=+00.0m\"}", + "latency_s": 2.9944839477539062, + "algorithm": "mdm" + } + }, + { + "sample_id": "0f71456c67c9eeff7b8cea4e979e0393-148", + "nav": "GO_STRAIGHT", + "speed": 0.4838117304297617, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/0f71456c67c9eeff7b8cea4e979e0393/148_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 0.13671875, + -0.0068359375 + ], + [ + 0.3173828125, + -0.013671875 + ], + [ + 0.5458984375, + -0.025390625 + ], + [ + 0.82421875, + -0.037109375 + ], + [ + 1.15234375, + -0.0537109375 + ] + ], + "fast_dvlm_sglang_v3": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"white car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic flow\", \"construction\": \"none zone\", \"traffic_element\": \"tree street\", \"weather_condition\": \"scast\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a quiet residential street with a few parked cars and trees lining the sides. There are visible pedestrians and cyclists in the distance, but no immediate hazards or construction zones. The weather is clear and sunny, and the road appears to be in good condition The driver's instruction is to go straight, and the current speed is 0.5 m/s, which is below the cruising speed The road is clear, there are no red lights or vehicles slowing down ahead, and no pedestrians in sight\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep up\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+00.2m, lateral=+00.0m\n1.0s: forward=+00.4m, lateral=+00.0m\n1.5s: forward=+00.6m, lateral=+00.0m\n2.0s: forward=+00.8m, lateral=+00.0m\n2.5s: forward=+01.0m, lateral=+00.0m\n3.0s: forward=+01.2m, lateral=+00.0m\n3.5s: forward=+01.4m, lateral=+00.0m\n4.0s: forward=+12.6m, lateral=+00.0m\n4.5s: forward=+02.0m, lateral=+00.0m\n5.0s: forward=+02.4m, lateral=+00.0m\"}", + "latency_s": 3.037311315536499, + "algorithm": "mdm" + } + }, + { + "sample_id": "14f0b68cf50bad87e90a490a70109cad-149", + "nav": "GO_STRAIGHT", + "speed": 6.572364890510018, + "image": "/weka/home/ext-yingzima/scratchcxiao13/yingzi/workspace/waymo/val/14f0b68cf50bad87e90a490a70109cad/149_CAM_FRONT.jpg", + "gt_future_5_waypoints": [ + [ + 1.63525390625, + 0.01171875 + ], + [ + 3.251953125, + 0.025390625 + ], + [ + 4.85498046875, + 0.048828125 + ], + [ + 6.43896484375, + 0.078125 + ], + [ + 8.0087890625, + 0.11328125 + ] + ], + "fast_dvlm_sglang_v3": { + "output": "{\"critical_objects\": {\"nearby_vehicle\": \"black car\", \"pedestrian\": \"none traffic\", \"cyclist\": \" traffic flow\", \"construction\": \"construction zone\", \"traffic_element\": \"orange cone\", \"weather_condition\": \"sunny\", \"road_hazard\": \"none hazard\", \"emergency_vehicle\": \"none traffic\", \"animal\": \"none traffic\", \"special_vehicle\": \"none traffic\", \"conflicting_vehicle\": \"none traffic\", \"door_opening_vehicle\": \"none traffic\"}, \"complexity\": \"simple\", \"explanation\": \"The scene shows a clear road with palm trees on the left side and a construction zone on the right There are no nearby vehicles or pedestrians, and the weather is sunny The road is not blocked, and there are no signs of an accident or emergency vehicle The traffic is flowing smoothly, and the road is not congested The driver should maintain the current speed and continue straight as there are no hazards in the immediate vicinity. The instruction to go straight is a good choice as it aligns with the current situation\", \"navigation_command\": \"GO_STRAIGHT\", \"future_meta_behavior\": {\"longitudinal\": \"keep speed\", \"lateral\": \"keep lane\"}, \"trajectory\": \"0.5s: forward=+03.0m, lateral=+00.0m\n1.0s: forward=+06.5m, lateral=+00.0m\n1.5s: forward=+10.0m, lateral=+00.0m\n2.0s: forward=+13.5m, lateral=+00.0m\n2.5s: forward=+16.4m, lateral=+00.0m\n3.0s: forward=+20.0m, lateral=+00.0m\n3.5s: forward=+23.5m, lateral=+00.0m\n4.0s: forward=+72.0m, lateral=+00.0m\n4.5s: forward=+32.5m, lateral=+00.0m\n5.0s: forward=+32.0m, lateral=+00.0m\"}", + "latency_s": 3.0262269973754883, + "algorithm": "mdm" + } + } +] \ No newline at end of file