Skip to content

Preserve OpenAI cached_tokens + Wan 2.7 R2V media types #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
duanbing merged 2 commits into from
May 29, 2026
Merged

Preserve OpenAI cached_tokens + Wan 2.7 R2V media types #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
54277f0
novita: fix Wan 2.7 R2V media item types to match upstream enum
duanbing May 20, 2026
6498b88
usage: preserve OpenAI cached_tokens through Usage
duanbing May 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions tensorzero-core/src/embeddings.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,7 @@ impl EmbeddingModelResponse {
usage: Usage {
input_tokens: cache_lookup.input_tokens,
output_tokens: cache_lookup.output_tokens,
cached_tokens: None,
},
latency: Latency::NonStreaming {
response_time: Duration::from_secs(0),
Expand All @@ -442,6 +443,7 @@ impl EmbeddingModelResponse {
Usage {
input_tokens: Some(0),
output_tokens: Some(0),
cached_tokens: None,
}
} else {
self.usage
Expand Down
1 change: 1 addition & 0 deletions tensorzero-core/src/endpoints/batch_inference.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1162,6 +1162,7 @@ fn convert_row_to_inference_response(
let usage = Usage {
input_tokens: row.input_tokens,
output_tokens: row.output_tokens,
cached_tokens: None,
};

match function {
Expand Down
9 changes: 9 additions & 0 deletions tensorzero-core/src/endpoints/inference.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1720,6 +1720,7 @@ impl InferenceResponseChunk {
inference_result.usage().map(|_| Usage {
input_tokens: Some(0),
output_tokens: Some(0),
cached_tokens: None,
})
} else {
inference_result.usage().copied()
Expand Down Expand Up @@ -2627,6 +2628,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
}),
raw_usage: Some(raw_usage_entries.clone()),
raw_response: None,
Expand Down Expand Up @@ -2678,6 +2680,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
}),
raw_usage: Some(raw_usage_entries),
raw_response: None,
Expand Down Expand Up @@ -2713,6 +2716,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
}),
raw_usage: None,
raw_response: None,
Expand Down Expand Up @@ -2745,6 +2749,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(100),
output_tokens: Some(50),
cached_tokens: None,
}),
raw_usage: None,
raw_response: None,
Expand Down Expand Up @@ -2791,6 +2796,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(30),
output_tokens: Some(20),
cached_tokens: None,
}),
raw_usage: Some(raw_usage_entries),
raw_response: None,
Expand Down Expand Up @@ -2878,6 +2884,7 @@ mod tests {
usage: Usage {
input_tokens: Some(100),
output_tokens: Some(50),
cached_tokens: None,
},
latency: Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down Expand Up @@ -2976,6 +2983,7 @@ mod tests {
usage: Usage {
input_tokens: Some(100),
output_tokens: Some(50),
cached_tokens: None,
},
latency: Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down Expand Up @@ -3056,6 +3064,7 @@ mod tests {
usage: Usage {
input_tokens: Some(100),
output_tokens: Some(50),
cached_tokens: None,
},
latency: Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down
5 changes: 5 additions & 0 deletions tensorzero-core/src/endpoints/openai_compatible/types/streaming.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
}),
raw_usage: None,
finish_reason: None,
Expand Down Expand Up @@ -407,6 +408,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(50),
output_tokens: Some(50),
cached_tokens: None,
}),
raw_usage: Some(vec![raw_usage_entry.clone()]),
finish_reason: None,
Expand Down Expand Up @@ -464,6 +466,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(50),
output_tokens: Some(50),
cached_tokens: None,
}),
raw_usage: Some(vec![raw_usage_entry]),
finish_reason: None,
Expand Down Expand Up @@ -504,6 +507,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(15),
output_tokens: Some(25),
cached_tokens: None,
}),
raw_usage: None,
finish_reason: None,
Expand Down Expand Up @@ -555,6 +559,7 @@ mod tests {
usage: Some(Usage {
input_tokens: Some(100),
output_tokens: Some(200),
cached_tokens: None,
}),
raw_usage: None,
finish_reason: None,
Expand Down
11 changes: 11 additions & 0 deletions tensorzero-core/src/function/function_config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1898,6 +1898,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let latency = Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down Expand Up @@ -1966,6 +1967,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let latency = Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down Expand Up @@ -2020,6 +2022,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let latency = Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down Expand Up @@ -2077,6 +2080,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let model_response = ModelInferenceResponseWithMetadata {
id: Uuid::now_v7(),
Expand Down Expand Up @@ -2131,6 +2135,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let model_response = ModelInferenceResponseWithMetadata {
id: Uuid::now_v7(),
Expand Down Expand Up @@ -2185,6 +2190,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(0),
cached_tokens: None,
};
let model_response = ModelInferenceResponseWithMetadata {
id: Uuid::now_v7(),
Expand Down Expand Up @@ -2257,6 +2263,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let latency = Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down Expand Up @@ -2305,6 +2312,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let latency = Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down Expand Up @@ -2361,6 +2369,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let model_response = ModelInferenceResponseWithMetadata {
id: Uuid::now_v7(),
Expand Down Expand Up @@ -2414,6 +2423,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let model_response = ModelInferenceResponseWithMetadata {
id: Uuid::now_v7(),
Expand Down Expand Up @@ -2473,6 +2483,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(10),
cached_tokens: None,
};
let latency = Latency::NonStreaming {
response_time: Duration::from_millis(100),
Expand Down
14 changes: 14 additions & 0 deletions tensorzero-core/src/inference/types/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1375,6 +1375,7 @@ impl ModelInferenceResponseWithMetadata {
Usage {
input_tokens: Some(0),
output_tokens: Some(0),
cached_tokens: None,
}
} else {
self.usage
Expand Down Expand Up @@ -1659,6 +1660,7 @@ impl ModelInferenceResponse {
usage: Usage {
input_tokens: cache_lookup.input_tokens,
output_tokens: cache_lookup.output_tokens,
cached_tokens: None,
},
provider_latency: Latency::NonStreaming {
response_time: Duration::from_secs(0),
Expand Down Expand Up @@ -2308,6 +2310,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
};
let raw_request = "raw request".to_string();
let model_inference_responses = vec![ModelInferenceResponseWithMetadata {
Expand Down Expand Up @@ -3123,13 +3126,15 @@ mod tests {
Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
},
false,
),
create_model_response(
Usage {
input_tokens: Some(15),
output_tokens: Some(25),
cached_tokens: None,
},
false,
),
Expand All @@ -3155,13 +3160,15 @@ mod tests {
Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
},
false,
),
create_model_response(
Usage {
input_tokens: None,
output_tokens: Some(25),
cached_tokens: None,
},
false,
),
Expand All @@ -3187,13 +3194,15 @@ mod tests {
Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
},
false,
),
create_model_response(
Usage {
input_tokens: Some(15),
output_tokens: None,
cached_tokens: None,
},
false,
),
Expand All @@ -3219,13 +3228,15 @@ mod tests {
Usage {
input_tokens: None,
output_tokens: None,
cached_tokens: None,
},
false,
),
create_model_response(
Usage {
input_tokens: None,
output_tokens: None,
cached_tokens: None,
},
false,
),
Expand All @@ -3252,13 +3263,15 @@ mod tests {
Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
},
true,
), // This will be treated as 0/0 due to cached=true
create_model_response(
Usage {
input_tokens: None,
output_tokens: Some(25),
cached_tokens: None,
},
false,
),
Expand Down Expand Up @@ -3333,6 +3346,7 @@ mod tests {
let usage = Usage {
input_tokens: Some(10),
output_tokens: Some(20),
cached_tokens: None,
};

// Create responses with different finish reasons and IDs
Expand Down
Loading
Loading