From 5a5977d35db2bf73ee47d20b3abc0e71734be0a1 Mon Sep 17 00:00:00 2001 From: Ryan Ward Date: Thu, 16 Jul 2026 10:42:19 -0500 Subject: [PATCH] Use LiteLLM /model/info for accurate model metadata discovery The previous implementation only used the standard OpenAI /models endpoint which often lacks context window and capability metadata, causing a blind 200K fallback for all models. Now fetch_openai_models_from_endpoint tries LiteLLM's /model/info endpoint first, which returns rich metadata: - max_input_tokens (e.g. 1,000,000 for Sonnet 4.6) - max_output_tokens (e.g. 128,000 for max models) - supports_vision - supports_function_calling - underlying model path (for provider detection) Falls back to /models if /model/info is unavailable (e.g. non-LiteLLM OpenAI-compatible endpoints). This ensures the model picker and context window configuration reflect the actual capabilities of the configured models. --- app/src/ai/llms.rs | 368 ++++++++++++++++++++++++++++++++------------- 1 file changed, 260 insertions(+), 108 deletions(-) diff --git a/app/src/ai/llms.rs b/app/src/ai/llms.rs index 344c7da9..1e460426 100644 --- a/app/src/ai/llms.rs +++ b/app/src/ai/llms.rs @@ -1089,8 +1089,12 @@ impl LLMPreferences { self.openai_provider_routing.get(model_id) } - /// Fetches available models from the configured OpenAI-compatible /models endpoint - /// and stores them in memory. Called at startup and when the user clicks "Fetch Models". + /// Fetches available models from the configured OpenAI-compatible endpoint. + /// + /// Tries LiteLLM's `/model/info` first (which returns rich metadata including + /// accurate `max_input_tokens`, `max_output_tokens`, `supports_vision`, and + /// `supports_function_calling`). Falls back to the standard OpenAI `/models` + /// endpoint if `/model/info` is unavailable. #[cfg(not(target_family = "wasm"))] pub fn fetch_openai_models_from_endpoint(&mut self, ctx: &mut ModelContext) { let settings = AISettings::as_ref(ctx); @@ -1114,118 +1118,21 @@ impl LLMPreferences { let _ = ctx.spawn( async move { - let url = format!("{}/models", base_url.trim_end_matches('/')); + let base = base_url.trim_end_matches('/'); let client = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(10)) .build() .unwrap_or_default(); - let mut request = client.get(&url); - if let Some(ref key) = api_key { - request = request.header("Authorization", format!("Bearer {key}")); + + // Try LiteLLM /model/info first for rich metadata + if let Some(models) = + fetch_from_litellm_model_info(base, api_key.as_deref(), &client).await + { + return models; } - let response = match request.send().await { - Ok(r) => r, - Err(e) => { - log::warn!("[openai/litellm] Failed to fetch models from endpoint: {e}"); - return Vec::new(); - } - }; - - if !response.status().is_success() { - log::warn!( - "[openai/litellm] Model fetch returned HTTP {}", - response.status() - ); - return Vec::new(); - } - - let body: serde_json::Value = match response.json().await { - Ok(v) => v, - Err(e) => { - log::warn!("[openai/litellm] Failed to parse models response: {e}"); - return Vec::new(); - } - }; - - fn u32_from_any(value: &serde_json::Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| value[*key].as_u64()) - .and_then(|value| u32::try_from(value).ok()) - } - - let models: Vec = body["data"] - .as_array() - .map(Vec::as_slice) - .unwrap_or_default() - .iter() - .filter_map(|m| { - let id = m["id"].as_str()?; - let max_input_tokens = u32_from_any( - m, - &["max_input_tokens", "input_token_limit", "max_prompt_tokens"], - ); - let context_size = - u32_from_any(m, &["max_model_len", "context_window", "token_size"]) - .or(max_input_tokens) - .unwrap_or(200_000); - let max_output_tokens = u32_from_any( - m, - &[ - "max_output_tokens", - "output_token_limit", - "max_completion_tokens", - "max_tokens", - ], - ); - - let display_name = id - .split('/') - .next_back() - .unwrap_or(id) - .replace(['-', '_'], " "); - let display_name = display_name - .split_whitespace() - .map(|word| { - let mut chars = word.chars(); - match chars.next() { - None => String::new(), - Some(c) => c.to_uppercase().to_string() + chars.as_str(), - } - }) - .collect::>() - .join(" "); - - let provider = if id.contains("claude") || id.contains("anthropic") { - Some("anthropic".to_string()) - } else if id.contains("gpt") || id.contains("o1") || id.contains("o3") { - Some("openai".to_string()) - } else if id.contains("gemini") { - Some("google".to_string()) - } else { - None - }; - - Some(OpenAIModelConfig { - model_id: id.to_string(), - display_name, - vision_supported: m["supports_vision"] - .as_bool() - .or_else(|| m["vision_support"].as_bool()) - .unwrap_or(false), - context_size, - max_input_tokens, - max_output_tokens, - provider, - }) - }) - .collect(); - - log::info!( - "[openai/litellm] Fetched {} model(s) from endpoint", - models.len() - ); - models + // Fallback to standard OpenAI /models endpoint + fetch_from_openai_models(base, api_key.as_deref(), &client).await }, |me, models, ctx| { if !models.is_empty() { @@ -2136,6 +2043,251 @@ fn custom_llm_info_from(endpoint: &CustomEndpoint, model: &CustomEndpointModel) } } +/// Fetches model metadata from LiteLLM's `/model/info` endpoint which returns rich +/// metadata including accurate context window sizes, output token limits, and +/// capability flags (vision, function calling). +/// +/// Returns `None` if the endpoint is unavailable or doesn't return valid data, +/// allowing the caller to fall back to the standard `/models` endpoint. +#[cfg(not(target_family = "wasm"))] +async fn fetch_from_litellm_model_info( + base_url: &str, + api_key: Option<&str>, + client: &reqwest::Client, +) -> Option> { + let url = format!("{base_url}/model/info"); + let mut request = client.get(&url); + if let Some(key) = api_key { + request = request.header("Authorization", format!("Bearer {key}")); + } + + let response = match request.send().await { + Ok(r) => r, + Err(e) => { + log::info!("[openai/litellm] /model/info not available ({e}), falling back to /models"); + return None; + } + }; + + if !response.status().is_success() { + log::info!( + "[openai/litellm] /model/info returned HTTP {}, falling back to /models", + response.status() + ); + return None; + } + + let body: serde_json::Value = match response.json().await { + Ok(v) => v, + Err(e) => { + log::warn!("[openai/litellm] Failed to parse /model/info response: {e}"); + return None; + } + }; + + let data = body["data"].as_array()?; + if data.is_empty() { + return None; + } + + let models: Vec = data + .iter() + .filter_map(|entry| { + let model_name = entry["model_name"].as_str()?; + let model_info = &entry["model_info"]; + + let max_input_tokens = model_info["max_input_tokens"] + .as_u64() + .and_then(|v| u32::try_from(v).ok()); + let max_output_tokens = model_info["max_output_tokens"] + .as_u64() + .and_then(|v| u32::try_from(v).ok()); + let context_size = max_input_tokens.unwrap_or(200_000); + + let vision_supported = model_info["supports_vision"].as_bool().unwrap_or(false); + + let display_name = model_name.replace(['-', '_'], " "); + let display_name = display_name + .split_whitespace() + .map(|word| { + let mut chars = word.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().to_string() + chars.as_str(), + } + }) + .collect::>() + .join(" "); + + // Detect provider from the underlying model path if available + let litellm_model = entry["litellm_params"]["model"] + .as_str() + .unwrap_or(model_name); + let provider = if litellm_model.contains("claude") + || litellm_model.contains("anthropic") + || litellm_model.contains("bedrock") + { + Some("anthropic".to_string()) + } else if litellm_model.contains("gpt") + || litellm_model.contains("o1") + || litellm_model.contains("o3") + { + Some("openai".to_string()) + } else if litellm_model.contains("gemini") { + Some("google".to_string()) + } else { + None + }; + + log::info!( + "[openai/litellm] Discovered model '{}': context={}, max_output={}, vision={}", + model_name, + context_size, + max_output_tokens.unwrap_or(0), + vision_supported, + ); + + Some(OpenAIModelConfig { + model_id: model_name.to_string(), + display_name, + vision_supported, + context_size, + max_input_tokens, + max_output_tokens, + provider, + }) + }) + .collect(); + + if models.is_empty() { + return None; + } + + log::info!( + "[openai/litellm] Fetched {} model(s) from /model/info endpoint", + models.len() + ); + Some(models) +} + +/// Fetches models from the standard OpenAI-compatible `/models` endpoint. +/// Used as a fallback when `/model/info` is unavailable. +#[cfg(not(target_family = "wasm"))] +async fn fetch_from_openai_models( + base_url: &str, + api_key: Option<&str>, + client: &reqwest::Client, +) -> Vec { + let url = format!("{base_url}/models"); + let mut request = client.get(&url); + if let Some(key) = api_key { + request = request.header("Authorization", format!("Bearer {key}")); + } + + let response = match request.send().await { + Ok(r) => r, + Err(e) => { + log::warn!("[openai/litellm] Failed to fetch models from /models endpoint: {e}"); + return Vec::new(); + } + }; + + if !response.status().is_success() { + log::warn!( + "[openai/litellm] /models returned HTTP {}", + response.status() + ); + return Vec::new(); + } + + let body: serde_json::Value = match response.json().await { + Ok(v) => v, + Err(e) => { + log::warn!("[openai/litellm] Failed to parse /models response: {e}"); + return Vec::new(); + } + }; + + fn u32_from_any(value: &serde_json::Value, keys: &[&str]) -> Option { + keys.iter() + .find_map(|key| value[*key].as_u64()) + .and_then(|value| u32::try_from(value).ok()) + } + + let models: Vec = body["data"] + .as_array() + .map(Vec::as_slice) + .unwrap_or_default() + .iter() + .filter_map(|m| { + let id = m["id"].as_str()?; + let max_input_tokens = u32_from_any( + m, + &["max_input_tokens", "input_token_limit", "max_prompt_tokens"], + ); + let context_size = + u32_from_any(m, &["max_model_len", "context_window", "token_size"]) + .or(max_input_tokens) + .unwrap_or(200_000); + let max_output_tokens = u32_from_any( + m, + &[ + "max_output_tokens", + "output_token_limit", + "max_completion_tokens", + "max_tokens", + ], + ); + + let display_name = id + .split('/') + .next_back() + .unwrap_or(id) + .replace(['-', '_'], " "); + let display_name = display_name + .split_whitespace() + .map(|word| { + let mut chars = word.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().to_string() + chars.as_str(), + } + }) + .collect::>() + .join(" "); + + let provider = if id.contains("claude") || id.contains("anthropic") { + Some("anthropic".to_string()) + } else if id.contains("gpt") || id.contains("o1") || id.contains("o3") { + Some("openai".to_string()) + } else if id.contains("gemini") { + Some("google".to_string()) + } else { + None + }; + + Some(OpenAIModelConfig { + model_id: id.to_string(), + display_name, + vision_supported: m["supports_vision"] + .as_bool() + .or_else(|| m["vision_support"].as_bool()) + .unwrap_or(false), + context_size, + max_input_tokens, + max_output_tokens, + provider, + }) + }) + .collect(); + + log::info!( + "[openai/litellm] Fetched {} model(s) from /models endpoint", + models.len() + ); + models +} + /// Gets the last cached LLM metadata. /// Disabled — Galaxy uses only locally configured providers. No server-fetched models /// are cached or restored. The model list is built exclusively from Bedrock/LiteLLM