Skip to content

fix(tracker): grep verified_models.py for frontend support #99

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
juanmichelini merged 1 commit into from
May 21, 2026
+29 −10
Merged

fix(tracker): grep verified_models.py for frontend support #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 26 additions & 7 deletions scripts/track_llm_support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,10 +252,27 @@ def get_model_tier(model_id: str) -> int:
"infra": "All-Hands-AI/infra",
}

# Files to search for model support in each repo
# Files to search for model support in each repo.
#
# The "frontend" entry intentionally lists two paths:
#
# 1. ``frontend/src/utils/verified-models.ts`` — the legacy hand-maintained
# dropdown list inside ``OpenHands/OpenHands``. The file has since been
# deleted, but ``git log -G`` still finds historical adds, which keeps
# dates accurate for models that pre-date the migration.
#
# 2. ``openhands-sdk/openhands/sdk/llm/utils/verified_models.py`` — the
# current single source of truth inside the SDK repo. After the
# migration, ``OpenHands/OpenHands``'s ``openhands/app_server/utils/
# llm.py`` builds the frontend's openhands-provider "Verified" list
# directly from ``VERIFIED_OPENHANDS_MODELS`` in this file, so a model
# appearing here counts as frontend-supported.
SEARCH_PATHS = {
"sdk": ["openhands-sdk/openhands/sdk/llm/"],
"frontend": ["frontend/src/utils/verified-models.ts"],
"frontend": [
"frontend/src/utils/verified-models.ts",
"openhands-sdk/openhands/sdk/llm/utils/verified_models.py",
],
"index_results": ["results/"],
"eval_proxy": ["k8s/evaluation/litellm.yaml"],
"prod_proxy": ["k8s/production/litellm.yaml"],
Expand Down Expand Up @@ -528,14 +545,16 @@ def search_frontend_for_model(model_id: str) -> Optional[str]:
Searches two locations:

1. ``frontend/src/utils/verified-models.ts`` in the OpenHands/OpenHands
repo (legacy location).
2. ``openhands-sdk/openhands/sdk/llm/utils/model_features.py`` in the
repo (legacy hand-maintained list; file has since been deleted, but
``git log -G`` still finds the historical adds).
2. ``openhands-sdk/openhands/sdk/llm/utils/verified_models.py`` in the
OpenHands/software-agent-sdk repo, where ``VERIFIED_OPENHANDS_MODELS``
now lives after the "single source of truth" migration.

The frontend's model-selector renders the openhands-provider "Verified"
subsection from the SDK's ``VERIFIED_OPENHANDS_MODELS``, so a model
present there counts as frontend-supported.
subsection from the SDK's ``VERIFIED_OPENHANDS_MODELS`` (imported by
``openhands/app_server/utils/llm.py`` in ``OpenHands/OpenHands``), so a
model present there counts as frontend-supported.

Args:
model_id: The language model ID to search for
Expand Down Expand Up @@ -578,7 +597,7 @@ def _search_repo(temp_dir, search_path):
# 2. New location: VERIFIED_OPENHANDS_MODELS in the SDK repo
try:
cache = _get_sdk_repo()
_search_repo(cache["temp_dir"], "openhands-sdk/openhands/sdk/llm/utils/model_features.py")
_search_repo(cache["temp_dir"], "openhands-sdk/openhands/sdk/llm/utils/verified_models.py")
except Exception as e:
print(f"Warning: Error searching SDK for frontend models: {e}", file=sys.stderr)

Expand Down
6 changes: 3 additions & 3 deletions tests/test_track_llm_support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ def test_finds_model_in_legacy_verified_models_ts(self, mock_frontend, mock_sdk)
@patch("track_llm_support._get_sdk_repo")
@patch("track_llm_support._get_frontend_repo")
def test_finds_model_in_sdk_verified_openhands_models(self, mock_frontend, mock_sdk):
"""Model found only in SDK's model_features.py returns that timestamp.
"""Model found only in SDK's verified_models.py returns that timestamp.

After the 'single source of truth' migration, new models (e.g.
Nemotron-3-Super) are added to VERIFIED_OPENHANDS_MODELS in the SDK
Expand All @@ -478,7 +478,7 @@ def test_finds_model_in_sdk_verified_openhands_models(self, mock_frontend, mock_

with patch("subprocess.run") as mock_run:
mock_run.side_effect = lambda cmd, **kw: (
sdk_result if self._cmd_targets_path(cmd, "model_features.py") else frontend_result
sdk_result if self._cmd_targets_path(cmd, "verified_models.py") else frontend_result
)
result = search_frontend_for_model("Nemotron-3-Super")

Expand All @@ -496,7 +496,7 @@ def test_picks_earliest_across_both_repos(self, mock_frontend, mock_sdk):

with patch("subprocess.run") as mock_run:
mock_run.side_effect = lambda cmd, **kw: (
sdk_result if self._cmd_targets_path(cmd, "model_features.py") else frontend_result
sdk_result if self._cmd_targets_path(cmd, "verified_models.py") else frontend_result
)
result = search_frontend_for_model("some-model")

Expand Down
Loading