diff --git a/.gitignore b/.gitignore index a74c524..aac8096 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ node_modules/ *.log .DS_Store .env +.env.keys .env.local .vercel/ coverage/ diff --git a/models/anthropic/claude-3-5-haiku-20241022.yaml b/models/anthropic/claude-3-5-haiku-20241022.yaml deleted file mode 100644 index 96f8fa8..0000000 --- a/models/anthropic/claude-3-5-haiku-20241022.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-3-5-haiku-20241022 -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled diff --git a/models/anthropic/claude-3-5-haiku-latest.yaml b/models/anthropic/claude-3-5-haiku-latest.yaml deleted file mode 100644 index 05b9584..0000000 --- a/models/anthropic/claude-3-5-haiku-latest.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-3-5-haiku-latest -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled diff --git a/models/anthropic/claude-3-5-sonnet-20241022.yaml b/models/anthropic/claude-3-5-sonnet-20241022.yaml deleted file mode 100644 index 5696d46..0000000 --- a/models/anthropic/claude-3-5-sonnet-20241022.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-3-5-sonnet-20241022 -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled diff --git a/models/anthropic/claude-3-5-sonnet-latest.yaml b/models/anthropic/claude-3-5-sonnet-latest.yaml deleted file mode 100644 index e26bf93..0000000 --- a/models/anthropic/claude-3-5-sonnet-latest.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-3-5-sonnet-latest -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled diff --git a/models/anthropic/claude-3-opus-20240229.yaml b/models/anthropic/claude-3-opus-20240229.yaml deleted file mode 100644 index 9de2c57..0000000 --- a/models/anthropic/claude-3-opus-20240229.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-3-opus-20240229 -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled diff --git a/models/anthropic/claude-3-opus-latest.yaml b/models/anthropic/claude-3-opus-latest.yaml deleted file mode 100644 index 8ebe076..0000000 --- a/models/anthropic/claude-3-opus-latest.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-3-opus-latest -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled diff --git a/models/anthropic/claude-haiku-4-5-20251001.yaml b/models/anthropic/claude-haiku-4-5-20251001.yaml index 392ad3b..2738963 100644 --- a/models/anthropic/claude-haiku-4-5-20251001.yaml +++ b/models/anthropic/claude-haiku-4-5-20251001.yaml @@ -25,9 +25,10 @@ params: group: sampling applicability: except: - thinking.type: - - adaptive - - enabled + - thinking.type: + - enabled + - top_p: + not: null - path: top_p type: number label: Top P @@ -43,10 +44,9 @@ params: applicability: except: - thinking.type: - - adaptive - enabled - temperature: - not: 1 + not: null - path: top_k type: integer label: Top K @@ -58,7 +58,6 @@ params: applicability: except: thinking.type: - - adaptive - enabled - path: thinking.type type: enum diff --git a/models/anthropic/claude-haiku-4.yaml b/models/anthropic/claude-opus-4-1-20250805.yaml similarity index 92% rename from models/anthropic/claude-haiku-4.yaml rename to models/anthropic/claude-opus-4-1-20250805.yaml index d1ce197..d1bd2d7 100644 --- a/models/anthropic/claude-haiku-4.yaml +++ b/models/anthropic/claude-opus-4-1-20250805.yaml @@ -1,7 +1,7 @@ # yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json provider: anthropic authType: api_key -model: claude-haiku-4 +model: claude-opus-4-1-20250805 params: - path: max_tokens type: integer @@ -25,9 +25,10 @@ params: group: sampling applicability: except: - thinking.type: - - adaptive - - enabled + - thinking.type: + - enabled + - top_p: + not: null - path: top_p type: number label: Top P @@ -43,10 +44,9 @@ params: applicability: except: - thinking.type: - - adaptive - enabled - temperature: - not: 1 + not: null - path: top_k type: integer label: Top K @@ -58,7 +58,6 @@ params: applicability: except: thinking.type: - - adaptive - enabled - path: thinking.type type: enum diff --git a/models/anthropic/claude-haiku-4-5.yaml b/models/anthropic/claude-opus-4-20250514.yaml similarity index 92% rename from models/anthropic/claude-haiku-4-5.yaml rename to models/anthropic/claude-opus-4-20250514.yaml index 98ce128..f72d85b 100644 --- a/models/anthropic/claude-haiku-4-5.yaml +++ b/models/anthropic/claude-opus-4-20250514.yaml @@ -1,7 +1,7 @@ # yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json provider: anthropic authType: api_key -model: claude-haiku-4-5 +model: claude-opus-4-20250514 params: - path: max_tokens type: integer @@ -25,9 +25,10 @@ params: group: sampling applicability: except: - thinking.type: - - adaptive - - enabled + - thinking.type: + - enabled + - top_p: + not: null - path: top_p type: number label: Top P @@ -43,10 +44,9 @@ params: applicability: except: - thinking.type: - - adaptive - enabled - temperature: - not: 1 + not: null - path: top_k type: integer label: Top K @@ -58,7 +58,6 @@ params: applicability: except: thinking.type: - - adaptive - enabled - path: thinking.type type: enum diff --git a/models/anthropic/claude-3-7-sonnet-latest.yaml b/models/anthropic/claude-opus-4-5-20251101.yaml similarity index 92% rename from models/anthropic/claude-3-7-sonnet-latest.yaml rename to models/anthropic/claude-opus-4-5-20251101.yaml index 9a9bdb0..457e265 100644 --- a/models/anthropic/claude-3-7-sonnet-latest.yaml +++ b/models/anthropic/claude-opus-4-5-20251101.yaml @@ -1,7 +1,7 @@ # yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json provider: anthropic authType: api_key -model: claude-3-7-sonnet-latest +model: claude-opus-4-5-20251101 params: - path: max_tokens type: integer @@ -25,9 +25,10 @@ params: group: sampling applicability: except: - thinking.type: - - adaptive - - enabled + - thinking.type: + - enabled + - top_p: + not: null - path: top_p type: number label: Top P @@ -43,10 +44,9 @@ params: applicability: except: - thinking.type: - - adaptive - enabled - temperature: - not: 1 + not: null - path: top_k type: integer label: Top K @@ -58,7 +58,6 @@ params: applicability: except: thinking.type: - - adaptive - enabled - path: thinking.type type: enum diff --git a/models/anthropic/claude-opus-4-6.yaml b/models/anthropic/claude-opus-4-6.yaml index af30e0c..c7f567f 100644 --- a/models/anthropic/claude-opus-4-6.yaml +++ b/models/anthropic/claude-opus-4-6.yaml @@ -25,9 +25,10 @@ params: group: sampling applicability: except: - thinking.type: - - adaptive - - enabled + - thinking.type: + - enabled + - top_p: + not: null - path: top_p type: number label: Top P @@ -43,10 +44,9 @@ params: applicability: except: - thinking.type: - - adaptive - enabled - temperature: - not: 1 + not: null - path: top_k type: integer label: Top K @@ -58,7 +58,6 @@ params: applicability: except: thinking.type: - - adaptive - enabled - path: thinking.type type: enum @@ -67,7 +66,6 @@ params: default: disabled values: - disabled - - adaptive - enabled group: reasoning - path: thinking.budget_tokens diff --git a/models/anthropic/claude-opus-4-7.yaml b/models/anthropic/claude-opus-4-7.yaml index b78228d..414f3f6 100644 --- a/models/anthropic/claude-opus-4-7.yaml +++ b/models/anthropic/claude-opus-4-7.yaml @@ -11,55 +11,6 @@ params: range: min: 1 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - path: thinking.type type: enum label: Thinking mode @@ -67,5 +18,4 @@ params: default: disabled values: - disabled - - adaptive group: reasoning diff --git a/models/anthropic/claude-opus-4.yaml b/models/anthropic/claude-opus-4.yaml deleted file mode 100644 index 7b784ca..0000000 --- a/models/anthropic/claude-opus-4.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-opus-4 -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: thinking.type - type: enum - label: Thinking mode - description: Controls the Anthropic thinking mode values supported by this model. - default: disabled - values: - - disabled - - adaptive - - enabled - group: reasoning - - path: thinking.budget_tokens - type: integer - label: Budget tokens - description: >- - Maximum token budget Anthropic may use for extended thinking before producing the final - answer. - default: 4096 - range: - min: 1024 - group: reasoning - applicability: - only: - thinking.type: enabled diff --git a/models/anthropic/claude-3-7-sonnet-20250219.yaml b/models/anthropic/claude-sonnet-4-20250514.yaml similarity index 92% rename from models/anthropic/claude-3-7-sonnet-20250219.yaml rename to models/anthropic/claude-sonnet-4-20250514.yaml index 8c5f5ab..91bfb60 100644 --- a/models/anthropic/claude-3-7-sonnet-20250219.yaml +++ b/models/anthropic/claude-sonnet-4-20250514.yaml @@ -1,7 +1,7 @@ # yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json provider: anthropic authType: api_key -model: claude-3-7-sonnet-20250219 +model: claude-sonnet-4-20250514 params: - path: max_tokens type: integer @@ -25,9 +25,10 @@ params: group: sampling applicability: except: - thinking.type: - - adaptive - - enabled + - thinking.type: + - enabled + - top_p: + not: null - path: top_p type: number label: Top P @@ -43,10 +44,9 @@ params: applicability: except: - thinking.type: - - adaptive - enabled - temperature: - not: 1 + not: null - path: top_k type: integer label: Top K @@ -58,7 +58,6 @@ params: applicability: except: thinking.type: - - adaptive - enabled - path: thinking.type type: enum diff --git a/models/anthropic/claude-sonnet-4-5-20250929.yaml b/models/anthropic/claude-sonnet-4-5-20250929.yaml index 3b39817..6e1a141 100644 --- a/models/anthropic/claude-sonnet-4-5-20250929.yaml +++ b/models/anthropic/claude-sonnet-4-5-20250929.yaml @@ -25,9 +25,10 @@ params: group: sampling applicability: except: - thinking.type: - - adaptive - - enabled + - thinking.type: + - enabled + - top_p: + not: null - path: top_p type: number label: Top P @@ -43,10 +44,9 @@ params: applicability: except: - thinking.type: - - adaptive - enabled - temperature: - not: 1 + not: null - path: top_k type: integer label: Top K @@ -58,7 +58,6 @@ params: applicability: except: thinking.type: - - adaptive - enabled - path: thinking.type type: enum @@ -67,7 +66,6 @@ params: default: disabled values: - disabled - - adaptive - enabled group: reasoning - path: thinking.budget_tokens diff --git a/models/anthropic/claude-sonnet-4-5.yaml b/models/anthropic/claude-sonnet-4-5.yaml deleted file mode 100644 index d68d610..0000000 --- a/models/anthropic/claude-sonnet-4-5.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-sonnet-4-5 -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: thinking.type - type: enum - label: Thinking mode - description: Controls the Anthropic thinking mode values supported by this model. - default: disabled - values: - - disabled - - adaptive - - enabled - group: reasoning - - path: thinking.budget_tokens - type: integer - label: Budget tokens - description: >- - Maximum token budget Anthropic may use for extended thinking before producing the final - answer. - default: 4096 - range: - min: 1024 - group: reasoning - applicability: - only: - thinking.type: enabled diff --git a/models/anthropic/claude-sonnet-4-6.yaml b/models/anthropic/claude-sonnet-4-6.yaml index 1e35b85..5e65af2 100644 --- a/models/anthropic/claude-sonnet-4-6.yaml +++ b/models/anthropic/claude-sonnet-4-6.yaml @@ -25,9 +25,10 @@ params: group: sampling applicability: except: - thinking.type: - - adaptive - - enabled + - thinking.type: + - enabled + - top_p: + not: null - path: top_p type: number label: Top P @@ -43,10 +44,9 @@ params: applicability: except: - thinking.type: - - adaptive - enabled - temperature: - not: 1 + not: null - path: top_k type: integer label: Top K @@ -58,7 +58,6 @@ params: applicability: except: thinking.type: - - adaptive - enabled - path: thinking.type type: enum @@ -67,7 +66,6 @@ params: default: disabled values: - disabled - - adaptive - enabled group: reasoning - path: thinking.budget_tokens diff --git a/models/anthropic/claude-sonnet-4.yaml b/models/anthropic/claude-sonnet-4.yaml deleted file mode 100644 index cc35e74..0000000 --- a/models/anthropic/claude-sonnet-4.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: anthropic -authType: api_key -model: claude-sonnet-4 -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 1 - step: 0.1 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - applicability: - except: - - thinking.type: - - adaptive - - enabled - - temperature: - not: 1 - - path: top_k - type: integer - label: Top K - description: Limits token sampling to the top K most likely next tokens. - default: 0 - range: - min: 0 - group: sampling - applicability: - except: - thinking.type: - - adaptive - - enabled - - path: thinking.type - type: enum - label: Thinking mode - description: Controls the Anthropic thinking mode values supported by this model. - default: disabled - values: - - disabled - - adaptive - - enabled - group: reasoning - - path: thinking.budget_tokens - type: integer - label: Budget tokens - description: >- - Maximum token budget Anthropic may use for extended thinking before producing the final - answer. - default: 4096 - range: - min: 1024 - group: reasoning - applicability: - only: - thinking.type: enabled diff --git a/models/openai/chatgpt-4o-latest.yaml b/models/openai/chatgpt-4o-latest.yaml deleted file mode 100644 index 9022b5f..0000000 --- a/models/openai/chatgpt-4o-latest.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: openai -authType: api_key -model: chatgpt-4o-latest -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling diff --git a/models/openai/gpt-5-chat-latest.yaml b/models/openai/gpt-5-chat-latest.yaml index f4496a5..e0bd609 100644 --- a/models/openai/gpt-5-chat-latest.yaml +++ b/models/openai/gpt-5-chat-latest.yaml @@ -3,46 +3,11 @@ provider: openai authType: api_key model: gpt-5-chat-latest params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - - path: reasoning_effort - type: enum - label: Reasoning effort - description: Controls how much reasoning the model should perform before producing an answer. - default: medium - values: - - minimal - - low - - medium - - high - group: reasoning diff --git a/models/openai/gpt-5-mini.yaml b/models/openai/gpt-5-mini.yaml index 43da32c..7ab3f1c 100644 --- a/models/openai/gpt-5-mini.yaml +++ b/models/openai/gpt-5-mini.yaml @@ -3,38 +3,14 @@ provider: openai authType: api_key model: gpt-5-mini params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - path: reasoning_effort type: enum label: Reasoning effort diff --git a/models/openai/gpt-5-nano.yaml b/models/openai/gpt-5-nano.yaml index 13f9f62..bf6073a 100644 --- a/models/openai/gpt-5-nano.yaml +++ b/models/openai/gpt-5-nano.yaml @@ -3,38 +3,14 @@ provider: openai authType: api_key model: gpt-5-nano params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - path: reasoning_effort type: enum label: Reasoning effort diff --git a/models/openai/gpt-5.1.yaml b/models/openai/gpt-5.1.yaml index 2609d54..0c902bf 100644 --- a/models/openai/gpt-5.1.yaml +++ b/models/openai/gpt-5.1.yaml @@ -3,38 +3,14 @@ provider: openai authType: api_key model: gpt-5.1 params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - path: reasoning_effort type: enum label: Reasoning effort diff --git a/models/openai/gpt-5.2.yaml b/models/openai/gpt-5.2.yaml index b5ccd5b..299546a 100644 --- a/models/openai/gpt-5.2.yaml +++ b/models/openai/gpt-5.2.yaml @@ -3,45 +3,21 @@ provider: openai authType: api_key model: gpt-5.2 params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - path: reasoning_effort type: enum label: Reasoning effort description: Controls how much reasoning the model should perform before producing an answer. default: medium values: - - minimal + - none - low - medium - high diff --git a/models/openai/gpt-5.4-mini.yaml b/models/openai/gpt-5.4-mini.yaml index 366a318..1f87ffb 100644 --- a/models/openai/gpt-5.4-mini.yaml +++ b/models/openai/gpt-5.4-mini.yaml @@ -3,45 +3,21 @@ provider: openai authType: api_key model: gpt-5.4-mini params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - path: reasoning_effort type: enum label: Reasoning effort description: Controls how much reasoning the model should perform before producing an answer. default: medium values: - - minimal + - none - low - medium - high diff --git a/models/openai/gpt-5.4.yaml b/models/openai/gpt-5.4.yaml index adc236c..708fc18 100644 --- a/models/openai/gpt-5.4.yaml +++ b/models/openai/gpt-5.4.yaml @@ -3,45 +3,21 @@ provider: openai authType: api_key model: gpt-5.4 params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - path: reasoning_effort type: enum label: Reasoning effort description: Controls how much reasoning the model should perform before producing an answer. default: medium values: - - minimal + - none - low - medium - high diff --git a/models/openai/gpt-5.5.yaml b/models/openai/gpt-5.5.yaml index af1b65c..e5c6221 100644 --- a/models/openai/gpt-5.5.yaml +++ b/models/openai/gpt-5.5.yaml @@ -3,45 +3,21 @@ provider: openai authType: api_key model: gpt-5.5 params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - path: reasoning_effort type: enum label: Reasoning effort description: Controls how much reasoning the model should perform before producing an answer. default: medium values: - - minimal + - none - low - medium - high diff --git a/models/openai/gpt-5.yaml b/models/openai/gpt-5.yaml index 5d62f79..5a2c130 100644 --- a/models/openai/gpt-5.yaml +++ b/models/openai/gpt-5.yaml @@ -3,38 +3,14 @@ provider: openai authType: api_key model: gpt-5 params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - - path: temperature - type: number - label: Temperature - description: >- - Controls randomness. Lower values make outputs more focused; higher values make them more - varied. - default: 1 - range: - min: 0 - max: 2 - step: 0.1 - group: sampling - - path: top_p - type: number - label: Top P - description: >- - Controls nucleus sampling by limiting generation to tokens whose cumulative probability - reaches this value. - default: 1 - range: - min: 0 - max: 1 - step: 0.01 - group: sampling - path: reasoning_effort type: enum label: Reasoning effort diff --git a/models/openai/o1-mini.yaml b/models/openai/o1-mini.yaml deleted file mode 100644 index cbbd35c..0000000 --- a/models/openai/o1-mini.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: openai -authType: api_key -model: o1-mini -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: reasoning_effort - type: enum - label: Reasoning effort - description: Controls how much reasoning the model should perform before producing an answer. - default: medium - values: - - minimal - - low - - medium - - high - group: reasoning diff --git a/models/openai/o1-preview.yaml b/models/openai/o1-preview.yaml deleted file mode 100644 index 566f621..0000000 --- a/models/openai/o1-preview.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://modelparameters.dev/api/v1/schema.json -provider: openai -authType: api_key -model: o1-preview -params: - - path: max_tokens - type: integer - label: Max tokens - description: Maximum number of output tokens the model may generate. - default: 4096 - range: - min: 1 - group: generation_length - - path: reasoning_effort - type: enum - label: Reasoning effort - description: Controls how much reasoning the model should perform before producing an answer. - default: medium - values: - - minimal - - low - - medium - - high - group: reasoning diff --git a/models/openai/o1.yaml b/models/openai/o1.yaml index 35ec80c..7866237 100644 --- a/models/openai/o1.yaml +++ b/models/openai/o1.yaml @@ -3,13 +3,13 @@ provider: openai authType: api_key model: o1 params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - path: reasoning_effort type: enum @@ -17,8 +17,8 @@ params: description: Controls how much reasoning the model should perform before producing an answer. default: medium values: - - minimal - low - medium - high + - xhigh group: reasoning diff --git a/models/openai/o3-mini.yaml b/models/openai/o3-mini.yaml index beb1f43..367d746 100644 --- a/models/openai/o3-mini.yaml +++ b/models/openai/o3-mini.yaml @@ -3,13 +3,13 @@ provider: openai authType: api_key model: o3-mini params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - path: reasoning_effort type: enum @@ -17,8 +17,8 @@ params: description: Controls how much reasoning the model should perform before producing an answer. default: medium values: - - minimal - low - medium - high + - xhigh group: reasoning diff --git a/models/openai/o3.yaml b/models/openai/o3.yaml index c2d5af6..27597b3 100644 --- a/models/openai/o3.yaml +++ b/models/openai/o3.yaml @@ -3,13 +3,13 @@ provider: openai authType: api_key model: o3 params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - path: reasoning_effort type: enum @@ -17,8 +17,8 @@ params: description: Controls how much reasoning the model should perform before producing an answer. default: medium values: - - minimal - low - medium - high + - xhigh group: reasoning diff --git a/models/openai/o4-mini.yaml b/models/openai/o4-mini.yaml index e676838..7bdbc41 100644 --- a/models/openai/o4-mini.yaml +++ b/models/openai/o4-mini.yaml @@ -3,13 +3,13 @@ provider: openai authType: api_key model: o4-mini params: - - path: max_tokens + - path: max_completion_tokens type: integer label: Max tokens description: Maximum number of output tokens the model may generate. default: 4096 range: - min: 1 + min: 16 group: generation_length - path: reasoning_effort type: enum @@ -17,8 +17,8 @@ params: description: Controls how much reasoning the model should perform before producing an answer. default: medium values: - - minimal - low - medium - high + - xhigh group: reasoning diff --git a/package.json b/package.json index af08806..3d48465 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "build": "tsx src/build/build.ts", "dev": "tsx watch src/server/dev.ts", "validate": "tsx src/data/validate.ts", + "smoke:params": "tsx src/smoke/provider-params.ts", "typecheck": "tsc --noEmit", "lint": "eslint . --ext .ts,.tsx", "format": "prettier --write .", diff --git a/src/smoke/provider-params.ts b/src/smoke/provider-params.ts new file mode 100644 index 0000000..f33702c --- /dev/null +++ b/src/smoke/provider-params.ts @@ -0,0 +1,636 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadAllModels } from "../data/load.js"; +import type { AuthType, Model, Parameter, ApplicabilityRule } from "../schema/model.js"; + +type JsonPrimitive = string | number | boolean | null; +type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; +type JsonRecord = Record; + +type Expected = "success" | "failure"; + +interface SmokeCase { + name: string; + params: JsonRecord; + expected: Expected; +} + +interface Options { + provider?: string; + model?: string; + authType?: AuthType; + keysFile?: string; + includeNegative: boolean; + limit?: number; +} + +interface ProviderAdapter { + envNames: readonly string[]; + keyAliases: readonly string[]; + run(model: Model, params: JsonRecord, apiKey: string): Promise; +} + +interface ProviderResult { + ok: boolean; + status: number; + message: string; +} + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, "../.."); +const DEFAULT_KEYS_FILE = path.resolve(REPO_ROOT, ".env.keys"); +const REQUEST_TIMEOUT_MS = 45_000; + +const PROVIDERS: Record = { + anthropic: { + envNames: ["ANTHROPIC_API_KEY"], + keyAliases: ["anthropic"], + run: runAnthropic, + }, + deepseek: { + envNames: ["DEEPSEEK_API_KEY"], + keyAliases: ["deepseek"], + run: (model, params, apiKey) => + runOpenAiCompatible("https://api.deepseek.com/chat/completions", model, params, apiKey, true), + }, + gemini: { + envNames: ["GEMINI_API_KEY", "GOOGLE_API_KEY"], + keyAliases: ["gemini", "google"], + run: runGemini, + }, + google: { + envNames: ["GEMINI_API_KEY", "GOOGLE_API_KEY"], + keyAliases: ["gemini", "google"], + run: runGemini, + }, + groq: { + envNames: ["GROQ_API_KEY"], + keyAliases: ["groq"], + run: (model, params, apiKey) => + runOpenAiCompatible( + "https://api.groq.com/openai/v1/chat/completions", + model, + params, + apiKey, + true, + ), + }, + mistral: { + envNames: ["MISTRAL_API_KEY"], + keyAliases: ["mistral"], + run: (model, params, apiKey) => + runOpenAiCompatible( + "https://api.mistral.ai/v1/chat/completions", + model, + params, + apiKey, + true, + ), + }, + openai: { + envNames: ["OPENAI_API_KEY"], + keyAliases: ["openai"], + run: (model, params, apiKey) => + runOpenAiCompatible( + "https://api.openai.com/v1/chat/completions", + model, + params, + apiKey, + false, + ), + }, +}; + +function parseOptions(argv: string[]): Options { + const options: Options = { + includeNegative: true, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]!; + const readValue = (): string => { + const next = argv[i + 1]; + if (!next) throw new Error(`${arg} requires a value`); + i += 1; + return next; + }; + + if (arg === "--provider") options.provider = readValue().toLowerCase(); + else if (arg === "--model") options.model = readValue(); + else if (arg === "--auth-type") options.authType = readValue() as AuthType; + else if (arg === "--keys-file") options.keysFile = path.resolve(readValue()); + else if (arg === "--positive-only") options.includeNegative = false; + else if (arg === "--limit") options.limit = Number.parseInt(readValue(), 10); + else if (arg === "--help" || arg === "-h") printHelpAndExit(); + else throw new Error(`Unknown option: ${arg}`); + } + + return options; +} + +function printHelpAndExit(): never { + console.log(`Usage: npm run smoke:params -- [options] + +Options: + --provider Limit to one provider, e.g. anthropic + --model Limit to one model id + --auth-type Limit to api_key or subscription + --keys-file Free-form key file. Defaults to ./.env.keys when present + --positive-only Skip intentionally invalid applicability cases + --limit Stop after n requests + +The smoke test sends minimal live requests and never prints API keys or request bodies.`); + process.exit(0); +} + +async function loadKeys(keysFile?: string): Promise> { + const keys = new Map(); + for (const [provider, adapter] of Object.entries(PROVIDERS)) { + for (const envName of adapter.envNames) { + const value = process.env[envName]; + if (value) keys.set(provider, value); + } + } + + const file = keysFile ?? ((await exists(DEFAULT_KEYS_FILE)) ? DEFAULT_KEYS_FILE : undefined); + if (!file) return keys; + + const text = await fs.readFile(file, "utf8"); + for (const line of text.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.toLowerCase() === "subscription") continue; + + for (const [provider, adapter] of Object.entries(PROVIDERS)) { + if (keys.has(provider)) continue; + if (!lineMatchesAnyAlias(trimmed, adapter.keyAliases)) continue; + if (provider === "anthropic" && /\btoken\b/i.test(trimmed)) continue; + + const candidate = trimmed.split(/\s+/).at(-1); + if (candidate && looksLikeApiKey(candidate)) { + keys.set(provider, candidate); + } + } + } + + return keys; +} + +function lineMatchesAnyAlias(line: string, aliases: readonly string[]): boolean { + const lower = line.toLowerCase(); + return aliases.some((alias) => lower.startsWith(alias.toLowerCase())); +} + +function looksLikeApiKey(value: string): boolean { + return value.length >= 12 && !value.includes("@"); +} + +async function exists(file: string): Promise { + try { + await fs.access(file); + return true; + } catch { + return false; + } +} + +function buildCases(model: Model): SmokeCase[] { + const cases: SmokeCase[] = []; + for (const param of model.params) { + for (const value of sampleValues(param)) { + const params = applyValidContext(model, defaultParams(model)); + setPath(params, param.path, value); + cases.push({ + name: `${param.path}=${formatValue(value)}`, + params: applyValidContext(model, params), + expected: "success", + }); + } + + if (param.applicability) { + cases.push(...buildNegativeApplicabilityCases(model, param)); + } + } + return dedupeCases(cases); +} + +function sampleValues(param: Parameter): JsonValue[] { + if (param.type === "boolean") return [true, false]; + if (param.type === "enum") return param.values as JsonValue[]; + if (param.type === "string") return param.default !== undefined ? [param.default] : ["test"]; + if (param.type === "integer" || param.type === "number") { + const values: JsonValue[] = []; + if (param.default !== undefined) values.push(param.default); + if (param.range?.min !== undefined) values.push(param.range.min); + if (param.range?.max !== undefined) values.push(param.range.max); + return dedupeValues(values.length > 0 ? values : [1]); + } + return []; +} + +function defaultParams(model: Model): JsonRecord { + const params: JsonRecord = {}; + for (const param of model.params) { + if (param.default === undefined) continue; + if (!paramIsApplicable(param, params)) continue; + setPath(params, param.path, param.default); + } + return params; +} + +function applyValidContext(model: Model, input: JsonRecord): JsonRecord { + const params = cloneRecord(input); + let changed = true; + while (changed) { + changed = false; + for (const param of model.params) { + if (param.default === undefined || hasPath(params, param.path)) continue; + if (!paramIsApplicable(param, params)) continue; + setPath(params, param.path, param.default); + changed = true; + } + } + + for (const param of model.params) { + if (!hasPath(params, param.path)) continue; + if (!paramIsApplicable(param, params)) deletePath(params, param.path); + } + + ensureProviderMinimums(model, params); + return params; +} + +function ensureProviderMinimums(model: Model, params: JsonRecord): void { + if (model.provider === "anthropic" && getPath(params, "thinking.type") === "enabled") { + const budget = getPath(params, "thinking.budget_tokens"); + if (typeof budget === "number") { + const maxTokens = getPath(params, "max_tokens"); + if (typeof maxTokens !== "number" || maxTokens <= budget) { + setPath(params, "max_tokens", Math.ceil(budget) + 1); + } + } + } +} + +function buildNegativeApplicabilityCases(model: Model, param: Parameter): SmokeCase[] { + const cases: SmokeCase[] = []; + const value = sampleValues(param)[0]; + if (value === undefined) return cases; + + for (const params of inapplicableContexts(param)) { + const next = applyValidContext(model, defaultParams(model)); + mergeRecord(next, params); + setPath(next, param.path, value); + ensureProviderMinimums(model, next); + cases.push({ + name: `${param.path} forbidden with ${Object.keys(params).join("+")}`, + params: next, + expected: "failure", + }); + } + + return cases; +} + +function inapplicableContexts(param: Parameter): JsonRecord[] { + const contexts: JsonRecord[] = []; + const applicability = param.applicability; + if (!applicability) return contexts; + + if (applicability.except) { + for (const match of ruleList(applicability.except)) { + contexts.push(matchContext(match, "match")); + } + } + if (applicability.only) { + for (const match of ruleList(applicability.only)) { + contexts.push(matchContext(match, "miss")); + } + } + + return contexts; +} + +function ruleList(rule: ApplicabilityRule | ApplicabilityRule[]): ApplicabilityRule[] { + return Array.isArray(rule) ? rule : [rule]; +} + +function matchContext(rule: ApplicabilityRule, mode: "match" | "miss"): JsonRecord { + const params: JsonRecord = {}; + for (const [pathKey, value] of Object.entries(rule)) { + setPath(params, pathKey, mode === "match" ? matchingValue(value) : nonMatchingValue(value)); + } + return params; +} + +function matchingValue(value: ApplicabilityRule[string]): JsonValue { + if (isRecord(value) && "not" in value) return nonMatchingValue(value.not); + if (Array.isArray(value)) return value[0] as JsonValue; + return value as JsonValue; +} + +function nonMatchingValue(value: ApplicabilityRule[string]): JsonValue { + if (isRecord(value) && "not" in value) return matchingValue(value.not); + const blocked = new Set(Array.isArray(value) ? value : [value]); + for (const candidate of ["disabled", "enabled", "adaptive", 0, 1, true, false, null]) { + if (!blocked.has(candidate)) return candidate as JsonValue; + } + return "__mps_non_matching_value__"; +} + +function paramIsApplicable(param: Parameter, params: JsonRecord): boolean { + const applicability = param.applicability; + if (!applicability) return true; + if (applicability.only && !ruleMatches(applicability.only, params)) return false; + if (applicability.except && ruleMatches(applicability.except, params)) return false; + return true; +} + +function ruleMatches(rule: ApplicabilityRule | ApplicabilityRule[], params: JsonRecord): boolean { + return ruleList(rule).some((match) => + Object.entries(match).every(([pathKey, expected]) => + valueMatches(getPath(params, pathKey), expected), + ), + ); +} + +function valueMatches(actual: unknown, expected: ApplicabilityRule[string]): boolean { + if (actual === undefined) return false; + if (isRecord(expected) && "not" in expected) return !valueMatches(actual, expected.not); + if (Array.isArray(expected)) return expected.some((value) => jsonEqual(actual, value)); + return jsonEqual(actual, expected); +} + +async function runAnthropic( + model: Model, + params: JsonRecord, + apiKey: string, +): Promise { + const body: JsonRecord = { + model: model.model, + max_tokens: 16, + messages: [{ role: "user", content: "Reply with ok." }], + ...params, + }; + return postJson("https://api.anthropic.com/v1/messages", body, { + "anthropic-version": "2023-06-01", + "content-type": "application/json", + "x-api-key": apiKey, + }); +} + +async function runGemini( + model: Model, + params: JsonRecord, + apiKey: string, +): Promise { + const url = new URL( + `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent( + model.model, + )}:generateContent`, + ); + url.searchParams.set("key", apiKey); + const body: JsonRecord = { + contents: [{ role: "user", parts: [{ text: "Reply with ok." }] }], + generationConfig: params, + }; + return postJson(url.toString(), body, { + "content-type": "application/json", + }); +} + +async function runOpenAiCompatible( + endpoint: string, + model: Model, + params: JsonRecord, + apiKey: string, + addDefaultTokenCap: boolean, +): Promise { + const body: JsonRecord = { + model: model.model, + messages: [{ role: "user", content: "Reply with ok." }], + ...params, + }; + if ( + addDefaultTokenCap && + getPath(params, "max_tokens") === undefined && + getPath(params, "max_completion_tokens") === undefined + ) { + body.max_tokens = 1; + } + return postJson(endpoint, body, { + authorization: `Bearer ${apiKey}`, + "content-type": "application/json", + }); +} + +async function postJson( + url: string, + body: JsonRecord, + headers: Record, +): Promise { + let last: ProviderResult | undefined; + for (let attempt = 0; attempt < 3; attempt += 1) { + const response = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + const text = await response.text(); + last = { + ok: response.ok, + status: response.status, + message: summarizeProviderMessage(text), + }; + if (!isTransientStatus(response.status)) return last; + await delay(750 * (attempt + 1)); + } + return last!; +} + +function isTransientStatus(status: number): boolean { + return status === 429 || status === 500 || status === 502 || status === 503 || status === 529; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function summarizeProviderMessage(text: string): string { + if (!text) return ""; + try { + const parsed = JSON.parse(text) as unknown; + if (isRecord(parsed)) { + const error = parsed.error; + if (isRecord(error)) { + const message = error.message; + if (typeof message === "string") return message.slice(0, 220); + } + const message = parsed.message; + if (typeof message === "string") return message.slice(0, 220); + } + } catch { + // Fall through to raw text summary. + } + return text.replace(/\s+/g, " ").slice(0, 220); +} + +function dedupeCases(cases: SmokeCase[]): SmokeCase[] { + const seen = new Set(); + return cases.filter((testCase) => { + const key = `${testCase.expected}:${testCase.name}:${JSON.stringify(testCase.params)}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function dedupeValues(values: JsonValue[]): JsonValue[] { + const seen = new Set(); + return values.filter((value) => { + const key = JSON.stringify(value); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function hasPath(record: JsonRecord, dottedPath: string): boolean { + return getPath(record, dottedPath) !== undefined; +} + +function getPath(record: JsonRecord, dottedPath: string): JsonValue | undefined { + let current: unknown = record; + for (const segment of dottedPath.split(".")) { + if (!isRecord(current) || !(segment in current)) return undefined; + current = current[segment]; + } + return current as JsonValue; +} + +function setPath(record: JsonRecord, dottedPath: string, value: JsonValue): void { + const segments = dottedPath.split("."); + let current = record; + for (const segment of segments.slice(0, -1)) { + const next = current[segment]; + if (!isRecord(next)) { + current[segment] = {}; + } + current = current[segment] as JsonRecord; + } + current[segments.at(-1)!] = value; +} + +function deletePath(record: JsonRecord, dottedPath: string): void { + const segments = dottedPath.split("."); + let current: JsonRecord = record; + for (const segment of segments.slice(0, -1)) { + const next = current[segment]; + if (!isRecord(next)) return; + current = next; + } + delete current[segments.at(-1)!]; +} + +function mergeRecord(target: JsonRecord, source: JsonRecord): void { + for (const [key, value] of Object.entries(source)) { + if (isRecord(value) && isRecord(target[key])) { + mergeRecord(target[key], value); + } else { + target[key] = value; + } + } +} + +function cloneRecord(record: JsonRecord): JsonRecord { + return structuredClone(record) as JsonRecord; +} + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function jsonEqual(a: unknown, b: unknown): boolean { + return JSON.stringify(a) === JSON.stringify(b); +} + +function formatValue(value: JsonValue): string { + return JSON.stringify(value); +} + +function modelMatches(model: Model, options: Options): boolean { + if (options.provider && model.provider !== options.provider) return false; + if (options.model && model.model !== options.model) return false; + if (options.authType && model.authType !== options.authType) return false; + return true; +} + +async function main(): Promise { + const options = parseOptions(process.argv.slice(2)); + const { models, issues } = await loadAllModels(); + if (issues.length > 0) { + throw new Error( + `Catalog has ${issues.length} validation issue(s). Run npm run validate first.`, + ); + } + + const keys = await loadKeys(options.keysFile); + const runnable = models.filter((model) => modelMatches(model, options)); + let passed = 0; + let failed = 0; + let skipped = 0; + let sent = 0; + + for (const model of runnable) { + const adapter = PROVIDERS[model.provider]; + const key = keys.get(model.provider); + if (model.authType !== "api_key") { + skipped += 1; + console.log(`- ${model.provider}/${model.model} (${model.authType}) skipped: subscription`); + continue; + } + if (!adapter) { + skipped += 1; + console.log(`- ${model.provider}/${model.model} skipped: no smoke adapter`); + continue; + } + if (!key) { + skipped += 1; + console.log(`- ${model.provider}/${model.model} skipped: no API key`); + continue; + } + + const cases = buildCases(model).filter((testCase) => + options.includeNegative ? true : testCase.expected === "success", + ); + for (const testCase of cases) { + if (options.limit !== undefined && sent >= options.limit) break; + sent += 1; + const result = await adapter.run(model, testCase.params, key); + const ok = + testCase.expected === "success" + ? result.ok + : result.status === 400 || result.status === 422; + if (ok) { + passed += 1; + const verb = testCase.expected === "success" ? "accepted" : "rejected"; + console.log(`✓ ${model.provider}/${model.model} ${testCase.name} ${verb}`); + } else { + failed += 1; + console.log( + `✗ ${model.provider}/${model.model} ${testCase.name} expected ${testCase.expected}, got ${result.status}: ${result.message}`, + ); + } + } + if (options.limit !== undefined && sent >= options.limit) break; + } + + console.log(`\nSummary: ${passed} passed, ${failed} failed, ${skipped} skipped, ${sent} sent.`); + if (failed > 0) process.exit(1); +} + +main().catch((err) => { + console.error((err as Error).message); + process.exit(2); +});